19_HyperoptExecutor_NotbookForm - GermEvalRandom.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import os\n",
    "import time\n",
    "import logging\n",
    "from hyperopt.plotting import *\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials, base, rand\n",
    "from data.data_loader import Dataset\n",
    "from misc.preferences import PREFERENCES\n",
    "from misc.run_configuration import from_hyperopt, OutputLayerType, LearningSchedulerType, OptimizerType, default_params\n",
    "from misc import utils\n",
    "from misc.hyperopt_space import *\n",
    "\n",
    "from optimizer import get_optimizer\n",
    "from criterion import NllLoss, LossCombiner\n",
    "from models.transformer.encoder import TransformerEncoder\n",
    "from models.jointAspectTagger import JointAspectTagger\n",
    "from trainer.train import Trainer\n",
    "import pprint\n",
    "import argparse\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "POSSIBLE_DATASET_VALUES = ['germeval', 'organic', 'amazon']\n",
    "\n",
    "\n",
    "def load_model(dataset, rc, experiment_name):\n",
    "    loss = LossCombiner(4, dataset.class_weights, NllLoss)\n",
    "    transformer = TransformerEncoder(dataset.source_embedding,\n",
    "                                     hyperparameters=rc)\n",
    "    model = JointAspectTagger(transformer, rc, 4, 20, dataset.target_names)\n",
    "    optimizer = get_optimizer(model, rc)\n",
    "    trainer = Trainer(\n",
    "                        model,\n",
    "                        loss,\n",
    "                        optimizer,\n",
    "                        rc,\n",
    "                        dataset,\n",
    "                        experiment_name,\n",
    "                        enable_tensorboard=False,\n",
    "                        verbose=False)\n",
    "    return trainer\n",
    "\n",
    "def load_dataset(rc, logger, task):\n",
    "    dataset = Dataset(\n",
    "            task,\n",
    "            logger,\n",
    "            rc,\n",
    "            source_index=PREFERENCES.source_index,\n",
    "            target_vocab_index=PREFERENCES.target_vocab_index,\n",
    "            data_path=PREFERENCES.data_root,\n",
    "            train_file=PREFERENCES.data_train,\n",
    "            valid_file=PREFERENCES.data_validation,\n",
    "            test_file=PREFERENCES.data_test,\n",
    "            file_format=PREFERENCES.file_format,\n",
    "            init_token=None,\n",
    "            eos_token=None\n",
    "        )\n",
    "    dataset.load_data(dsl, verbose=False)\n",
    "    return dataset\n",
    "\n",
    "def objective(parameters):\n",
    "    run_time = time.time()\n",
    "\n",
    "    utils.reset_loggers()\n",
    "    experiment_name = utils.create_loggers(experiment_name=main_experiment_name)\n",
    "    logger = logging.getLogger(__name__)\n",
    "    dataset_logger = logging.getLogger('data_loader')\n",
    "\n",
    "    # generate hp's from parameters\n",
    "    try:\n",
    "        rc = from_hyperopt(parameters, use_cuda, model_size=300, early_stopping=5, num_epochs=35, log_every_xth_iteration=-1, language=PREFERENCES.language)\n",
    "    except Exception as err:\n",
    "        print('Could not convert params: ' + str(err))\n",
    "        logger.exception(\"Could not load parameters from hyperopt configuration: \" + parameters)\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "    logger.info('New Params:')\n",
    "    logger.info(rc)\n",
    "    print('\\n\\n#########################################################################')\n",
    "    print(rc)\n",
    "\n",
    "    logger.debug('Load dataset')\n",
    "    try:\n",
    "        dataset = load_dataset(rc, dataset_logger, rc.task)\n",
    "    except Exception as err:\n",
    "        print('Could not load dataset: ' + str(err))\n",
    "        logger.exception(\"Could not load dataset\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "    logger.debug('dataset loaded')\n",
    "    logger.debug('Load model')\n",
    "\n",
    "    try:\n",
    "        trainer = load_model(dataset, rc, experiment_name)\n",
    "    except Exception as err:\n",
    "        print('Could not load model: ' + str(err))\n",
    "        logger.exception(\"Could not load model\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "\n",
    "    logger.debug('model loaded')\n",
    "\n",
    "    logger.debug('Begin training')\n",
    "    model = None\n",
    "    try:\n",
    "        result = trainer.train(use_cuda=rc.use_cuda, perform_evaluation=False)\n",
    "        model = result['model']\n",
    "    except Exception as err:\n",
    "        print('Exception while training: ' + str(err))\n",
    "        logger.exception(\"Could not complete iteration\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    if math.isnan(trainer.get_best_loss()):\n",
    "        print('Loss is nan')\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    # perform evaluation and log results\n",
    "    result = None\n",
    "    try:\n",
    "        result = trainer.perform_final_evaluation(use_test_set=True, verbose=False)\n",
    "    except Exception as err:\n",
    "        logger.exception(\"Could not complete iteration evaluation.\")\n",
    "        print('Could not complete iteration evaluation: ' + str(err))\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "    print(f'VAL f1\\t{trainer.get_best_f1()} - ({result[1][1]})')\n",
    "    print(f'VAL loss\\t{trainer.get_best_loss()}')\n",
    "\n",
    "    print(f\"       .---.\\n \\\n",
    "         /     \\\\\\n\\\n",
    "          \\\\.@-@./\\n\\\n",
    "          /`\\\\_/`\\\\\\n\\\n",
    "         //  _  \\\\\\\\\\tLoss: {trainer.get_best_loss()}\\n\\\n",
    "        | \\\\     )|_\\tf1: {trainer.get_best_f1()}\\n\\\n",
    "       /`\\\\_`>  <_/ \\\\\\n\\\n",
    "       \\\\__/'---'\\\\__/\\n\")\n",
    "\n",
    "    return {\n",
    "            'loss': result[1][0],\n",
    "            'status': STATUS_OK,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1(),\n",
    "            'sample_iterations': trainer.get_num_samples_seen(),\n",
    "            'iterations': trainer.get_num_iterations(),\n",
    "            'rc': rc,\n",
    "            'results': {\n",
    "                'train': {\n",
    "                    'loss': result[0][0],\n",
    "                    'f1': result[0][1]\n",
    "                },\n",
    "                'validation': {\n",
    "                    'loss': result[1][0],\n",
    "                    'f1': result[1][1]\n",
    "                },\n",
    "                'test': {\n",
    "                    'loss': result[2][0],\n",
    "                    'f1': result[2][1]\n",
    "                }\n",
    "            }\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset_choice = 'germeval'\n",
    "runs = 100\n",
    "main_experiment_name = 'GermEvalHyperoptRandom'\n",
    "use_cuda = True\n",
    "description = 'GermEval random Hyperopt run'\n",
    "\n",
    "if dataset_choice not in POSSIBLE_DATASET_VALUES:\n",
    "    raise Error()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if dataset_choice == POSSIBLE_DATASET_VALUES[0]:\n",
    "    PREFERENCES.defaults(\n",
    "        data_root='./data/data/germeval2017',\n",
    "        data_train='train_v1.4.tsv',    \n",
    "        data_validation='dev_v1.4.tsv',\n",
    "        data_test='test_TIMESTAMP1.tsv',\n",
    "        source_index=0,\n",
    "        target_vocab_index=2,\n",
    "        file_format='csv',\n",
    "        language='de'\n",
    "    )\n",
    "    from data.germeval2017 import germeval2017_dataset as dsl\n",
    "\n",
    "    search_space = {\n",
    "        'batch_size': hp.quniform('batch_size', 10, 100, 1),\n",
    "        'num_encoder_blocks': hp.quniform('num_encoder_blocks', 1, 8, 1),\n",
    "        'pointwise_layer_size': hp.quniform('pointwise_layer_size', 32, 256, 1),\n",
    "        'clip_comments_to': hp.quniform('clip_comments_to', 10, 250, 1),\n",
    "        'dropout_rate': hp.uniform('dropout_rate', 0.0, 0.8),\n",
    "        'output_dropout_rate': hp.uniform('last_layer_dropout', 0.0, 0.8),\n",
    "        'num_heads': hp.choice('num_heads', [1, 2, 3, 4, 5]),\n",
    "        'transformer_use_bias': hp_bool('transformer_use_bias'),\n",
    "        'output_layer': hp.choice('output_layer', [\n",
    "            {\n",
    "                'type': OutputLayerType.Convolutions,\n",
    "                'output_conv_num_filters': hp.quniform('output_conv_num_filters', 1, 400, 1),\n",
    "                'output_conv_kernel_size': hp.quniform('output_conv_kernel_size', 1, 10, 1),\n",
    "                'output_conv_stride': hp.quniform('output_conv_stride', 1, 10, 1),\n",
    "                'output_conv_padding': hp.quniform('output_conv_padding', 0, 5, 1),\n",
    "            },\n",
    "            {\n",
    "                'type': OutputLayerType.LinearSum\n",
    "            }\n",
    "        ]),\n",
    "        'learning_rate_scheduler': hp.choice('learning_rate_scheduler', [\n",
    "            {\n",
    "                'type': LearningSchedulerType.Noam,\n",
    "                'noam_learning_rate_warmup': hp.quniform('noam_learning_rate_warmup', 1000, 9000, 1),\n",
    "                'noam_learning_rate_factor': hp.uniform('noam_learning_rate_factor', 0.01, 4)\n",
    "            }\n",
    "        ]),\n",
    "        'optimizer': hp.choice('optimizer', [\n",
    "            {\n",
    "                'type': OptimizerType.Adam,\n",
    "                'adam_beta1': hp.uniform('adam_beta1', 0.7, 0.999),\n",
    "                'adam_beta2': hp.uniform('adam_beta2', 0.7, 0.999),\n",
    "                'adam_eps': hp.loguniform('adam_eps', np.log(1e-10), np.log(1)),\n",
    "                'learning_rate': hp.lognormal('adam_learning_rate', np.log(0.01), np.log(10)),\n",
    "                'adam_weight_decay': 1*10**hp.quniform('adam_weight_decay', -8, -3, 1)\n",
    "            },\n",
    "            #{\n",
    "            #    'type': OptimizerType.SGD,\n",
    "            #    'sgd_momentum': hp.uniform('sgd_momentum', 0.4, 1),\n",
    "            #    'sgd_weight_decay': hp.loguniform('sgd_weight_decay', np.log(1e-4), np.log(1)),\n",
    "            #    'sgd_nesterov': hp_bool('sgd_nesterov'),\n",
    "            #    'learning_rate': hp.lognormal('sgd_learning_rate', np.log(0.01), np.log(10))\n",
    "        ]),\n",
    "        'replace_url_tokens': hp_bool('replace_url_tokens'),\n",
    "        'harmonize_bahn': hp_bool('harmonize_bahn'),\n",
    "        'embedding_type': hp.choice('embedding_type', ['fasttext', 'glove']),\n",
    "        'embedding_name': hp.choice('embedding_name', ['6B']),\n",
    "        'embedding_dim': hp.choice('embedding_dim', [300]),\n",
    "        'use_stop_words': hp_bool('use_stop_words'),\n",
    "        'use_spell_checker': hp_bool('use_spell_checker'),\n",
    "        'embedding_type': hp.choice('embedding_type', ['fasttext', 'glove']),\n",
    "        'task': 'germeval'\n",
    "    }\n",
    "\n",
    "elif dataset_choice == POSSIBLE_DATASET_VALUES[1]:\n",
    "     from data.organic2019 import organic_dataset as dsl\n",
    "     from data.organic2019 import ORGANIC_TASK_ALL, ORGANIC_TASK_ENTITIES, ORGANIC_TASK_ATTRIBUTES, ORGANIC_TASK_ENTITIES_COMBINE, ORGANIC_TASK_COARSE\n",
    "     PREFERENCES.defaults(\n",
    "        data_root='./data/data/organic2019',\n",
    "        data_train='train.csv',    \n",
    "        data_validation='validation.csv',\n",
    "        data_test='test.csv',\n",
    "        source_index=0,\n",
    "        target_vocab_index=1,\n",
    "        file_format='csv',\n",
    "        language='en'\n",
    "     )\n",
    "\n",
    "     search_space = {\n",
    "        'batch_size': hp.quniform('batch_size', 10, 64, 1),\n",
    "        'num_encoder_blocks': hp.quniform('num_encoder_blocks', 1, 4, 1),\n",
    "        'pointwise_layer_size': hp.quniform('pointwise_layer_size', 32, 350, 1),\n",
    "        'clip_comments_to': hp.quniform('clip_comments_to', 45, 180, 1),\n",
    "        'dropout_rate': hp.uniform('dropout_rate', 0.0, 0.8),\n",
    "        'output_dropout_rate': hp.uniform('last_layer_dropout', 0.0, 0.8),\n",
    "        'num_heads': hp.choice('num_heads', [1, 2, 3, 4, 5]),\n",
    "        'transformer_use_bias': hp_bool('transformer_use_bias'),\n",
    "        'output_layer': hp.choice('output_layer', [\n",
    "            {\n",
    "                'type': OutputLayerType.Convolutions,\n",
    "                'output_conv_num_filters': hp.quniform('output_conv_num_filters', 10, 400, 1),\n",
    "                'output_conv_kernel_size': hp.quniform('output_conv_kernel_size', 1, 10, 1),\n",
    "                'output_conv_stride': hp.quniform('output_conv_stride', 1, 10, 1),\n",
    "                'output_conv_padding': hp.quniform('output_conv_padding', 0, 5, 1),\n",
    "            },\n",
    "            {\n",
    "                'type': OutputLayerType.LinearSum\n",
    "            }\n",
    "        ]),\n",
    "        'learning_rate_scheduler': hp.choice('learning_rate_scheduler', [\n",
    "            {\n",
    "                'type': LearningSchedulerType.Noam,\n",
    "                'noam_learning_rate_warmup': hp.quniform('noam_learning_rate_warmup', 1000, 9000, 1),\n",
    "                'noam_learning_rate_factor': hp.uniform('noam_learning_rate_factor', 0.01, 4)\n",
    "            }\n",
    "        ]),\n",
    "        'optimizer': hp.choice('optimizer', [\n",
    "            {\n",
    "                'type': OptimizerType.Adam,\n",
    "                'adam_beta1': hp.uniform('adam_beta1', 0.7, 0.999),\n",
    "                'adam_beta2': hp.uniform('adam_beta2', 0.7, 0.999),\n",
    "                'adam_eps': hp.loguniform('adam_eps', np.log(1e-10), np.log(1)),\n",
    "                'learning_rate': hp.lognormal('adam_learning_rate', np.log(0.01), np.log(10)),\n",
    "                'adam_weight_decay': 1*10**hp.quniform('adam_weight_decay', -8, -3, 1)\n",
    "            },\n",
    "            #{\n",
    "            #    'type': OptimizerType.SGD,\n",
    "            #    'sgd_momentum': hp.uniform('sgd_momentum', 0.4, 1),\n",
    "            #    'sgd_weight_decay': hp.loguniform('sgd_weight_decay', np.log(1e-4), np.log(1)),\n",
    "            #    'sgd_nesterov': hp_bool('sgd_nesterov'),\n",
    "            #    'learning_rate': hp.lognormal('sgd_learning_rate', np.log(0.01), np.log(10))\n",
    "        ]),\n",
    "        'task': hp.choice('task', [\n",
    "            ORGANIC_TASK_ENTITIES,\n",
    "            ORGANIC_TASK_ENTITIES_COMBINE\n",
    "        ]),\n",
    "        'use_stop_words': hp_bool('use_stop_words'),\n",
    "        'use_spell_checker': hp_bool('use_spell_checker'),\n",
    "        'embedding_type': hp.choice('embedding_type', ['fasttext', 'glove'])\n",
    "    }\n",
    "else:\n",
    "    PREFERENCES.defaults(\n",
    "        data_root='./data/data/amazon/splits',\n",
    "        data_train='train.pkl',    \n",
    "        data_validation='val.pkl',\n",
    "        data_test='test.pkl',\n",
    "        source_index=0,\n",
    "        target_vocab_index=1,\n",
    "        file_format='pkl',\n",
    "        language='en'\n",
    "    )\n",
    "    from data.amazon import amazon_dataset as dsl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\0\n"
     ]
    }
   ],
   "source": [
    "experiment_name = utils.create_loggers(experiment_name=main_experiment_name)\n",
    "logger = logging.getLogger(__name__)\n",
    "dataset_logger = logging.getLogger('data_loader')\n",
    "logger.info('Run hyper parameter random grid search for experiment with name ' + main_experiment_name)\n",
    "logger.info('num_optim_iterations: ' + str(runs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current commit: b'57a7aff'\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\1 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 66.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         66                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.00586084465357032                |\n",
      "|  noam_learning_rate_warmup   |                        3595                       |\n",
      "|  noam_learning_rate_factor   |                 1.5938738141770603                |\n",
      "|          adam_beta1          |                 0.9333966954452615                |\n",
      "|          adam_beta2          |                 0.9293016047853316                |\n",
      "|           adam_eps           |               0.00032249356653622855              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.25873167624068566                |\n",
      "|     pointwise_layer_size     |                         71                        |\n",
      "|      last_layer_dropout      |                 0.744069009696656                 |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         38                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  0%|                                                                            | 0/100 [00:00<?, ?it/s, best loss: ?]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68de06d106524bed947ee405b89c5965",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.30\t\t0.22\t\t0.234\t\t0.896\t\t0.84m - 0.8m / 0.0m                                                                    \n",
      "2\t34k\t0.23\t\t0.20\t\t0.272\t\t0.912\t\t0.83m - 1.7m / 29.5m                                                                   \n",
      "3\t51k\t0.21\t\t0.20\t\t0.211\t\t0.863\t\t0.78m - 2.5m / 29.1m                                                                   \n",
      "4\t68k\t0.20\t\t0.19\t\t0.199\t\t0.848\t\t0.84m - 3.4m / 27.4m                                                                   \n",
      "5\t85k\t0.18\t\t0.19\t\t0.256\t\t0.894\t\t0.82m - 4.2m / 29.3m                                                                   \n",
      "6\t103k\t0.17\t\t0.20\t\t0.217\t\t0.870\t\t0.83m - 5.0m / 28.8m                                                                  \n",
      "7\t120k\t0.15\t\t0.22\t\t0.331\t\t0.929\t\t0.80m - 5.8m / 29.1m                                                                  \n",
      "8\t137k\t0.14\t\t0.22\t\t0.220\t\t0.869\t\t0.82m - 6.7m / 28.2m                                                                  \n",
      "9\t154k\t0.12\t\t0.25\t\t0.165\t\t0.845\t\t0.83m - 7.5m / 28.8m                                                                  \n",
      "10\t171k\t0.11\t\t0.24\t\t0.277\t\t0.909\t\t0.83m - 8.4m / 29.1m                                                                 \n",
      "11\t188k\t0.11\t\t0.26\t\t0.262\t\t0.903\t\t0.83m - 9.2m / 29.2m                                                                 \n",
      "12\t205k\t0.10\t\t0.28\t\t0.203\t\t0.865\t\t0.82m - 10.1m / 29.2m                                                                \n",
      "VAL f1\t0.33144535347178516 - (0.33144535347178516)                                                                     \n",
      "VAL loss\t0.18956038028453337                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.18956038028453337\n",
      "        | \\     )|_\tf1: 0.33144535347178516\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\2 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 79.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         79                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.008542833833933339               |\n",
      "|  noam_learning_rate_warmup   |                        1640                       |\n",
      "|  noam_learning_rate_factor   |                 1.675045452251482                 |\n",
      "|          adam_beta1          |                 0.8242403503980521                |\n",
      "|          adam_beta2          |                 0.8824302104883239                |\n",
      "|           adam_eps           |                 0.7621548477363127                |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.4660615883295896                |\n",
      "|     pointwise_layer_size     |                         70                        |\n",
      "|      last_layer_dropout      |                0.31313886157023774                |\n",
      "|   output_conv_num_filters    |                        164                        |\n",
      "|   output_conv_kernel_size    |                         1                         |\n",
      "|      output_conv_stride      |                         2                         |\n",
      "|     output_conv_padding      |                         4                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         11                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  1%|▍                                             | 1/100 [10:57<18:04:51, 657.49s/it, best loss: 0.22460714185779745]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6dae06f71a146029c28a6b7feafc672",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exception while training: size mismatch, m1: [12956 x 3], m2: [164 x 4] at c:\\a\\w\\1\\s\\tmp_conda_3.6_105809\\conda\\conda-bld\\pytorch_1544094150554\\work\\aten\\src\\thc\\generic/THCTensorMathBlas.cu:266\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\3 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 85.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         85                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.04728314975172574                |\n",
      "|  noam_learning_rate_warmup   |                        7416                       |\n",
      "|  noam_learning_rate_factor   |                 2.032298442174665                 |\n",
      "|          adam_beta1          |                 0.8910847283362151                |\n",
      "|          adam_beta2          |                 0.7373027988375062                |\n",
      "|           adam_eps           |               7.187359306241272e-09               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.7598036574173083                |\n",
      "|     pointwise_layer_size     |                        148                        |\n",
      "|      last_layer_dropout      |                0.40908813951394046                |\n",
      "|   output_conv_num_filters    |                        280                        |\n",
      "|   output_conv_kernel_size    |                         9                         |\n",
      "|      output_conv_stride      |                         1                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        136                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  2%|▉                                             | 2/100 [11:22<12:43:54, 467.70s/it, best loss: 0.22460714185779745]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7812b923aa7c41719ea9a57fffed1574",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exception while training: size mismatch, m1: [23800 x 7], m2: [280 x 4] at c:\\a\\w\\1\\s\\tmp_conda_3.6_105809\\conda\\conda-bld\\pytorch_1544094150554\\work\\aten\\src\\thc\\generic/THCTensorMathBlas.cu:266\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\4 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 59.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         59                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0006357255208244108               |\n",
      "|  noam_learning_rate_warmup   |                        6804                       |\n",
      "|  noam_learning_rate_factor   |                 2.6759329806403716                |\n",
      "|          adam_beta1          |                 0.7517039435730589                |\n",
      "|          adam_beta2          |                 0.7445216921102901                |\n",
      "|           adam_eps           |                6.57394083516952e-08               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.30003705890640237                |\n",
      "|     pointwise_layer_size     |                         82                        |\n",
      "|      last_layer_dropout      |                 0.2814179509350761                |\n",
      "|   output_conv_num_filters    |                        344                        |\n",
      "|   output_conv_kernel_size    |                         6                         |\n",
      "|      output_conv_stride      |                         9                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         39                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  3%|█▍                                             | 3/100 [11:38<8:56:57, 332.14s/it, best loss: 0.22460714185779745]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "656d17bcbd5842fe92a97944f7f8f9b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.27\t\t0.25\t\t0.135\t\t0.832\t\t1.40m - 1.4m / 0.0m                                                                    \n",
      "2\t34k\t0.25\t\t0.24\t\t0.135\t\t0.800\t\t1.52m - 3.0m / 49.2m                                                                   \n",
      "3\t51k\t0.26\t\t0.24\t\t0.169\t\t0.803\t\t1.51m - 4.6m / 53.3m                                                                   \n",
      "4\t68k\t0.25\t\t0.23\t\t0.196\t\t0.843\t\t1.39m - 6.0m / 52.8m                                                                   \n",
      "5\t85k\t0.25\t\t0.25\t\t0.138\t\t0.752\t\t1.50m - 7.5m / 49.1m                                                                   \n",
      "6\t102k\t0.23\t\t0.24\t\t0.193\t\t0.864\t\t1.50m - 9.0m / 52.7m                                                                  \n",
      "7\t119k\t0.24\t\t0.27\t\t0.109\t\t0.735\t\t1.53m - 10.6m / 52.5m                                                                 \n",
      "8\t136k\t0.24\t\t0.22\t\t0.183\t\t0.854\t\t1.52m - 12.1m / 53.3m                                                                 \n",
      "9\t153k\t0.24\t\t0.26\t\t0.177\t\t0.810\t\t1.59m - 13.7m / 53.3m                                                                 \n",
      "VAL f1\t0.19567812798471823 - (0.19567812798471823)                                                                     \n",
      "VAL loss\t0.22028145247452485                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.22028145247452485\n",
      "        | \\     )|_\tf1: 0.19567812798471823\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\5 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 99.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         99                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.00207309923891512                |\n",
      "|  noam_learning_rate_warmup   |                        1400                       |\n",
      "|  noam_learning_rate_factor   |                 1.9910251479122478                |\n",
      "|          adam_beta1          |                 0.7084320880209571                |\n",
      "|          adam_beta2          |                 0.7820896174128037                |\n",
      "|           adam_eps           |               9.200445482726313e-08               |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.20443562946701263                |\n",
      "|     pointwise_layer_size     |                        177                        |\n",
      "|      last_layer_dropout      |                0.45737138601545824                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         62                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  4%|█▊                                            | 4/100 [26:30<13:20:06, 500.07s/it, best loss: 0.22460714185779745]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7861d4a6e6224a139eb74e937d1b953e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.20\t\t0.15\t\t0.208\t\t0.864\t\t0.69m - 0.7m / 0.0m                                                                    \n",
      "2\t34k\t0.16\t\t0.15\t\t0.307\t\t0.925\t\t0.68m - 1.4m / 24.3m                                                                   \n",
      "3\t51k\t0.15\t\t0.15\t\t0.233\t\t0.879\t\t0.68m - 2.1m / 24.0m                                                                   \n",
      "4\t69k\t0.15\t\t0.16\t\t0.191\t\t0.844\t\t0.69m - 2.8m / 23.8m                                                                   \n",
      "5\t86k\t0.15\t\t0.15\t\t0.230\t\t0.880\t\t0.67m - 3.5m / 24.1m                                                                   \n",
      "6\t103k\t0.15\t\t0.14\t\t0.225\t\t0.874\t\t0.67m - 4.2m / 23.7m                                                                  \n",
      "7\t120k\t0.15\t\t0.15\t\t0.222\t\t0.866\t\t0.64m - 4.8m / 23.5m                                                                  \n",
      "VAL f1\t0.3072042570609906 - (0.3072042570609906)                                                                       \n",
      "VAL loss\t0.1432683138734965                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.1432683138734965\n",
      "        | \\     )|_\tf1: 0.3072042570609906\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\6 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 39.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         39                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.02600518843729271                |\n",
      "|  noam_learning_rate_warmup   |                        1545                       |\n",
      "|  noam_learning_rate_factor   |                 0.4287227656909603                |\n",
      "|          adam_beta1          |                 0.9680100973269881                |\n",
      "|          adam_beta2          |                 0.880923521713613                 |\n",
      "|           adam_eps           |               0.0013874038822283465               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.793714789363686                 |\n",
      "|     pointwise_layer_size     |                        112                        |\n",
      "|      last_layer_dropout      |                0.15702113768169026                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         25                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  5%|██▎                                           | 5/100 [32:06<11:53:58, 450.93s/it, best loss: 0.14858898811468774]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3dbc464fa364a2dabc8200229992daa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.45\t\t0.40\t\t0.230\t\t0.893\t\t0.92m - 0.9m / 0.0m                                                                    \n",
      "2\t34k\t0.35\t\t0.37\t\t0.205\t\t0.858\t\t0.93m - 1.9m / 32.1m                                                                   \n",
      "3\t51k\t0.34\t\t0.59\t\t0.098\t\t0.705\t\t0.93m - 2.8m / 32.7m                                                                   \n",
      "4\t68k\t0.32\t\t0.57\t\t0.106\t\t0.684\t\t0.93m - 3.8m / 32.6m                                                                   \n",
      "5\t85k\t0.31\t\t0.64\t\t0.100\t\t0.672\t\t0.93m - 4.7m / 32.5m                                                                   \n",
      "6\t102k\t0.29\t\t0.59\t\t0.108\t\t0.697\t\t0.94m - 5.6m / 32.6m                                                                  \n",
      "VAL f1\t0.23000824402308326 - (0.23000824402308326)                                                                     \n",
      "VAL loss\t0.3717025903093682                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3717025903093682\n",
      "        | \\     )|_\tf1: 0.23000824402308326\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\7 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 66.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         66                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0006781946033925557               |\n",
      "|  noam_learning_rate_warmup   |                        4234                       |\n",
      "|  noam_learning_rate_factor   |                 2.5317625179595398                |\n",
      "|          adam_beta1          |                 0.9614980309480116                |\n",
      "|          adam_beta2          |                 0.992434812545663                 |\n",
      "|           adam_eps           |               3.5077684363994236e-05              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.6827516647621432                |\n",
      "|     pointwise_layer_size     |                        116                        |\n",
      "|      last_layer_dropout      |                 0.6625171190707642                |\n",
      "|   output_conv_num_filters    |                         35                        |\n",
      "|   output_conv_kernel_size    |                         8                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        242                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  6%|██▊                                           | 6/100 [38:47<11:22:56, 435.92s/it, best loss: 0.14858898811468774]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5638794b88fe45c7b556da6c425398f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.28\t\t0.32\t\t0.089\t\t0.592\t\t2.61m - 2.6m / 0.0m                                                                    \n",
      "2\t34k\t0.26\t\t0.32\t\t0.076\t\t0.561\t\t2.59m - 5.2m / 91.2m                                                                   \n",
      "3\t51k\t0.24\t\t0.29\t\t0.094\t\t0.655\t\t2.58m - 7.8m / 90.7m                                                                   \n",
      "4\t68k\t0.23\t\t0.26\t\t0.158\t\t0.811\t\t2.59m - 10.4m / 90.5m                                                                  \n",
      "5\t85k\t0.23\t\t0.29\t\t0.101\t\t0.683\t\t2.59m - 13.1m / 90.9m                                                                  \n",
      "6\t103k\t0.22\t\t0.33\t\t0.083\t\t0.604\t\t2.58m - 15.7m / 90.7m                                                                 \n",
      "7\t120k\t0.23\t\t0.31\t\t0.111\t\t0.714\t\t2.60m - 18.3m / 90.6m                                                                 \n",
      "8\t137k\t0.24\t\t0.31\t\t0.098\t\t0.674\t\t2.58m - 20.9m / 90.9m                                                                 \n",
      "9\t154k\t0.25\t\t0.33\t\t0.147\t\t0.817\t\t2.62m - 23.5m / 90.6m                                                                 \n",
      "VAL f1\t0.15780748109004247 - (0.15780748109004247)                                                                     \n",
      "VAL loss\t0.26173195875052246                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.26173195875052246\n",
      "        | \\     )|_\tf1: 0.15780748109004247\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\8 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 93.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         93                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.12494709387822975                |\n",
      "|  noam_learning_rate_warmup   |                        8252                       |\n",
      "|  noam_learning_rate_factor   |                 2.502932361433983                 |\n",
      "|          adam_beta1          |                 0.7104527990830158                |\n",
      "|          adam_beta2          |                 0.8370326584026015                |\n",
      "|           adam_eps           |               1.7161246673807278e-08              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |               0.0016600846437419216               |\n",
      "|     pointwise_layer_size     |                         61                        |\n",
      "|      last_layer_dropout      |                 0.2193704317063274                |\n",
      "|   output_conv_num_filters    |                        263                        |\n",
      "|   output_conv_kernel_size    |                         7                         |\n",
      "|      output_conv_stride      |                         9                         |\n",
      "|     output_conv_padding      |                         4                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        111                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  7%|███                                         | 7/100 [1:04:29<19:49:59, 767.74s/it, best loss: 0.14858898811468774]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24f220c41c8642b5a5f595b830eaa899",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.18\t\t0.13\t\t0.253\t\t0.890\t\t1.78m - 1.8m / 0.0m                                                                    \n",
      "2\t34k\t0.13\t\t0.11\t\t0.289\t\t0.906\t\t1.79m - 3.6m / 62.2m                                                                   \n",
      "3\t51k\t0.12\t\t0.11\t\t0.277\t\t0.901\t\t1.79m - 5.4m / 62.7m                                                                   \n",
      "4\t68k\t0.11\t\t0.11\t\t0.329\t\t0.925\t\t1.77m - 7.2m / 62.6m                                                                   \n",
      "5\t86k\t0.11\t\t0.12\t\t0.228\t\t0.872\t\t1.79m - 9.0m / 62.2m                                                                   \n",
      "6\t103k\t0.10\t\t0.13\t\t0.253\t\t0.885\t\t1.79m - 10.9m / 62.8m                                                                 \n",
      "7\t120k\t0.09\t\t0.12\t\t0.276\t\t0.906\t\t1.79m - 12.6m / 62.7m                                                                 \n",
      "8\t137k\t0.09\t\t0.11\t\t0.317\t\t0.921\t\t1.80m - 14.5m / 62.7m                                                                 \n",
      "9\t154k\t0.08\t\t0.13\t\t0.300\t\t0.913\t\t1.80m - 16.3m / 63.0m                                                                 \n",
      "VAL f1\t0.3289315726290516 - (0.3289315726290516)                                                                       \n",
      "VAL loss\t0.10995660924978375                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.10995660924978375\n",
      "        | \\     )|_\tf1: 0.3289315726290516\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\9 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 82.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         82                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0009973320877254012               |\n",
      "|  noam_learning_rate_warmup   |                        8090                       |\n",
      "|  noam_learning_rate_factor   |                 2.988021855896275                 |\n",
      "|          adam_beta1          |                 0.9972492639440775                |\n",
      "|          adam_beta2          |                 0.8541364893730953                |\n",
      "|           adam_eps           |                0.028615775206241957               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.6343996941585556                |\n",
      "|     pointwise_layer_size     |                         35                        |\n",
      "|      last_layer_dropout      |                 0.6699322382173344                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        231                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  8%|███▌                                        | 8/100 [1:22:12<21:52:57, 856.27s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f630ce3f3cca46bc92fa4b01a3ab420e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.26\t\t0.20\t\t0.192\t\t0.868\t\t2.86m - 2.9m / 0.0m                                                                    \n",
      "2\t34k\t0.20\t\t0.20\t\t0.382\t\t0.959\t\t2.80m - 5.7m / 100.1m                                                                  \n",
      "3\t51k\t0.20\t\t0.19\t\t0.199\t\t0.869\t\t2.78m - 8.5m / 98.1m                                                                   \n",
      "4\t68k\t0.19\t\t0.18\t\t0.382\t\t0.959\t\t2.79m - 11.3m / 97.5m                                                                  \n",
      "5\t85k\t0.19\t\t0.19\t\t0.140\t\t0.778\t\t2.81m - 14.1m / 97.7m                                                                  \n",
      "6\t102k\t0.19\t\t0.19\t\t0.244\t\t0.912\t\t2.85m - 17.0m / 98.5m                                                                 \n",
      "7\t119k\t0.19\t\t0.18\t\t0.265\t\t0.916\t\t2.82m - 19.8m / 99.6m                                                                 \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.18401191269479145                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.18401191269479145\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\10\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 48.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         48                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.026723704256167755               |\n",
      "|  noam_learning_rate_warmup   |                        7065                       |\n",
      "|  noam_learning_rate_factor   |                 3.716551875762824                 |\n",
      "|          adam_beta1          |                 0.8131684654785292                |\n",
      "|          adam_beta2          |                 0.9358326184860551                |\n",
      "|           adam_eps           |               6.897158282647107e-10               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.46793096123675293                |\n",
      "|     pointwise_layer_size     |                        149                        |\n",
      "|      last_layer_dropout      |                0.006329369812271501               |\n",
      "|   output_conv_num_filters    |                        276                        |\n",
      "|   output_conv_kernel_size    |                         1                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         27                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  9%|███▉                                        | 9/100 [1:44:23<25:14:52, 998.82s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48b78be267824e7a9b42ec72b16ce065",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.34\t\t0.27\t\t0.206\t\t0.875\t\t1.71m - 1.7m / 0.0m                                                                    \n",
      "2\t34k\t0.27\t\t0.25\t\t0.225\t\t0.872\t\t1.71m - 3.4m / 59.8m                                                                   \n",
      "3\t51k\t0.26\t\t0.24\t\t0.280\t\t0.917\t\t1.65m - 5.1m / 59.9m                                                                   \n",
      "4\t68k\t0.27\t\t0.25\t\t0.179\t\t0.834\t\t1.68m - 6.8m / 58.1m                                                                   \n",
      "5\t85k\t0.27\t\t0.28\t\t0.164\t\t0.807\t\t1.69m - 8.5m / 59.1m                                                                   \n",
      "6\t103k\t0.30\t\t0.32\t\t0.318\t\t0.943\t\t1.69m - 10.2m / 59.1m                                                                 \n",
      "7\t120k\t0.32\t\t0.31\t\t0.252\t\t0.910\t\t1.65m - 11.9m / 59.4m                                                                 \n",
      "8\t137k\t0.32\t\t0.30\t\t0.382\t\t0.959\t\t1.61m - 13.5m / 58.0m                                                                 \n",
      "9\t154k\t0.32\t\t0.33\t\t0.382\t\t0.959\t\t1.65m - 15.2m / 57.0m                                                                 \n",
      "10\t171k\t0.32\t\t0.32\t\t0.382\t\t0.959\t\t1.67m - 16.9m / 58.1m                                                                \n",
      "11\t188k\t0.32\t\t0.32\t\t0.382\t\t0.959\t\t1.68m - 18.6m / 58.7m                                                                \n",
      "12\t205k\t0.33\t\t0.31\t\t0.382\t\t0.959\t\t1.69m - 20.3m / 59.0m                                                                \n",
      "13\t222k\t0.32\t\t0.32\t\t0.382\t\t0.959\t\t1.69m - 22.0m / 59.1m                                                                \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.24355522791544595                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.24355522791544595\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\11\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 31.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         31                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0001617801985222442               |\n",
      "|  noam_learning_rate_warmup   |                        3827                       |\n",
      "|  noam_learning_rate_factor   |                 2.856818734523311                 |\n",
      "|          adam_beta1          |                 0.7596489446304899                |\n",
      "|          adam_beta2          |                  0.74948281692768                 |\n",
      "|           adam_eps           |                0.003594717013703679               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.4856793714618126                |\n",
      "|     pointwise_layer_size     |                         74                        |\n",
      "|      last_layer_dropout      |                 0.559726122843919                 |\n",
      "|   output_conv_num_filters    |                         56                        |\n",
      "|   output_conv_kernel_size    |                         5                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         0                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         31                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 10%|████▏                                     | 10/100 [2:07:42<27:58:16, 1118.85s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a43afe6e785a4ce2a04bea844dc3f8d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.46\t\t0.37\t\t0.263\t\t0.908\t\t1.34m - 1.3m / 0.0m                                                                    \n",
      "2\t34k\t0.48\t\t0.40\t\t0.256\t\t0.919\t\t1.34m - 2.7m / 47.1m                                                                   \n",
      "3\t51k\t0.51\t\t0.44\t\t0.279\t\t0.909\t\t1.35m - 4.1m / 46.8m                                                                   \n",
      "4\t68k\t0.55\t\t0.53\t\t0.308\t\t0.928\t\t1.32m - 5.4m / 47.4m                                                                   \n",
      "5\t85k\t0.64\t\t0.48\t\t0.263\t\t0.903\t\t1.32m - 6.8m / 46.3m                                                                   \n",
      "6\t102k\t0.65\t\t0.50\t\t0.167\t\t0.821\t\t1.33m - 8.1m / 46.3m                                                                  \n",
      "7\t119k\t0.56\t\t0.48\t\t0.120\t\t0.699\t\t1.31m - 9.4m / 46.6m                                                                  \n",
      "8\t136k\t0.41\t\t0.35\t\t0.239\t\t0.884\t\t1.28m - 10.7m / 46.0m                                                                 \n",
      "9\t153k\t0.39\t\t0.34\t\t0.273\t\t0.908\t\t1.28m - 12.0m / 45.2m                                                                 \n",
      "VAL f1\t0.3084566596194503 - (0.3084566596194503)                                                                       \n",
      "VAL loss\t0.33635250490103197                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.33635250490103197\n",
      "        | \\     )|_\tf1: 0.3084566596194503\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\12\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 68.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         68                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.03785868189569093                |\n",
      "|  noam_learning_rate_warmup   |                        6335                       |\n",
      "|  noam_learning_rate_factor   |                 3.056424140585657                 |\n",
      "|          adam_beta1          |                 0.7231889041445163                |\n",
      "|          adam_beta2          |                  0.99289416082121                 |\n",
      "|           adam_eps           |               0.00032555937399574985              |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         8                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3570731953971101                |\n",
      "|     pointwise_layer_size     |                        240                        |\n",
      "|      last_layer_dropout      |                 0.5748618569377213                |\n",
      "|   output_conv_num_filters    |                         99                        |\n",
      "|   output_conv_kernel_size    |                         9                         |\n",
      "|      output_conv_stride      |                         4                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         95                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 11%|████▌                                     | 11/100 [2:20:50<25:12:34, 1019.72s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "364fe372e0ae461b9716ab62623d5641",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.25\t\t0.27\t\t0.143\t\t0.803\t\t2.42m - 2.4m / 0.0m                                                                    \n",
      "2\t34k\t0.22\t\t0.25\t\t0.162\t\t0.827\t\t2.44m - 4.9m / 84.6m                                                                   \n",
      "3\t51k\t0.21\t\t0.25\t\t0.161\t\t0.813\t\t2.45m - 7.4m / 85.4m                                                                   \n",
      "4\t68k\t0.21\t\t0.26\t\t0.174\t\t0.816\t\t2.46m - 9.8m / 85.9m                                                                   \n",
      "5\t85k\t0.21\t\t0.26\t\t0.247\t\t0.900\t\t2.47m - 12.3m / 86.0m                                                                  \n",
      "6\t102k\t0.21\t\t0.27\t\t0.299\t\t0.930\t\t2.48m - 14.9m / 86.5m                                                                 \n",
      "7\t119k\t0.21\t\t0.27\t\t0.223\t\t0.889\t\t2.46m - 17.4m / 86.7m                                                                 \n",
      "8\t137k\t0.23\t\t0.29\t\t0.267\t\t0.917\t\t2.46m - 19.8m / 86.2m                                                                 \n",
      "9\t154k\t0.23\t\t0.27\t\t0.225\t\t0.893\t\t2.43m - 22.3m / 86.3m                                                                 \n",
      "10\t171k\t0.23\t\t0.29\t\t0.234\t\t0.894\t\t2.44m - 24.7m / 85.3m                                                                \n",
      "11\t188k\t0.23\t\t0.31\t\t0.226\t\t0.885\t\t2.46m - 27.2m / 85.8m                                                                \n",
      "VAL f1\t0.29851737109980087 - (0.29851737109980087)                                                                     \n",
      "VAL loss\t0.2486100830208418                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2486100830208418\n",
      "        | \\     )|_\tf1: 0.29851737109980087\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\13\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 49.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         49                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.03723325297084338                |\n",
      "|  noam_learning_rate_warmup   |                        6604                       |\n",
      "|  noam_learning_rate_factor   |                  3.15745492096616                 |\n",
      "|          adam_beta1          |                 0.8630559599481448                |\n",
      "|          adam_beta2          |                 0.8671975374496907                |\n",
      "|           adam_eps           |               2.6804216615588038e-09              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.7721644877450281                |\n",
      "|     pointwise_layer_size     |                         64                        |\n",
      "|      last_layer_dropout      |                 0.5419973533024681                |\n",
      "|   output_conv_num_filters    |                         63                        |\n",
      "|   output_conv_kernel_size    |                         7                         |\n",
      "|      output_conv_stride      |                         9                         |\n",
      "|     output_conv_padding      |                         1                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        152                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 12%|█████                                     | 12/100 [2:50:03<30:18:02, 1239.58s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58c6374925e447fbb4d1e7cff459a5c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.34\t\t0.51\t\t0.054\t\t0.293\t\t2.44m - 2.4m / 0.0m                                                                    \n",
      "2\t34k\t0.31\t\t0.62\t\t0.056\t\t0.283\t\t2.42m - 4.9m / 85.3m                                                                   \n",
      "3\t51k\t0.32\t\t0.64\t\t0.023\t\t0.309\t\t2.40m - 7.3m / 84.7m                                                                   \n",
      "4\t68k\t0.33\t\t0.41\t\t0.065\t\t0.612\t\t2.40m - 9.7m / 84.3m                                                                   \n",
      "5\t85k\t0.33\t\t0.36\t\t0.075\t\t0.677\t\t2.24m - 12.0m / 84.2m                                                                  \n",
      "6\t102k\t0.33\t\t0.34\t\t0.098\t\t0.715\t\t2.20m - 14.2m / 79.2m                                                                 \n",
      "7\t119k\t0.33\t\t0.39\t\t0.048\t\t0.563\t\t2.27m - 16.7m / 78.2m                                                                 \n",
      "8\t136k\t0.36\t\t0.40\t\t0.067\t\t0.561\t\t2.40m - 19.1m / 80.2m                                                                 \n",
      "9\t153k\t0.37\t\t0.37\t\t0.082\t\t0.649\t\t2.25m - 21.4m / 83.9m                                                                 \n",
      "10\t171k\t0.39\t\t0.42\t\t0.058\t\t0.542\t\t2.15m - 23.6m / 80.0m                                                                \n",
      "11\t188k\t0.41\t\t0.40\t\t0.074\t\t0.660\t\t2.17m - 25.8m / 77.4m                                                                \n",
      "VAL f1\t0.09812947537629695 - (0.09812947537629695)                                                                     \n",
      "VAL loss\t0.3436466041876346                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3436466041876346\n",
      "        | \\     )|_\tf1: 0.09812947537629695\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\14\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 58.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         58                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.003061004884739858               |\n",
      "|  noam_learning_rate_warmup   |                        2876                       |\n",
      "|  noam_learning_rate_factor   |                 3.0638499942515662                |\n",
      "|          adam_beta1          |                 0.7405110681907038                |\n",
      "|          adam_beta2          |                 0.7475656306575296                |\n",
      "|           adam_eps           |               1.8517825599212247e-06              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.1378665508845102                |\n",
      "|     pointwise_layer_size     |                        150                        |\n",
      "|      last_layer_dropout      |                0.22978148236123053                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         21                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 13%|█████▍                                    | 13/100 [3:17:48<33:02:15, 1367.07s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b0a2ddafd8246f2b94ce3d520c2b701",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.27\t\t0.20\t\t0.328\t\t0.936\t\t1.02m - 1.0m / 0.0m                                                                    \n",
      "2\t34k\t0.23\t\t0.21\t\t0.266\t\t0.904\t\t1.03m - 2.1m / 35.8m                                                                   \n",
      "3\t51k\t0.23\t\t0.24\t\t0.320\t\t0.932\t\t1.03m - 3.1m / 36.0m                                                                   \n",
      "4\t68k\t0.24\t\t0.23\t\t0.286\t\t0.922\t\t1.10m - 4.2m / 36.0m                                                                   \n",
      "5\t85k\t0.24\t\t0.24\t\t0.218\t\t0.865\t\t1.03m - 5.3m / 38.3m                                                                   \n",
      "6\t102k\t0.25\t\t0.26\t\t0.243\t\t0.891\t\t1.04m - 6.3m / 36.3m                                                                  \n",
      "VAL f1\t0.32824948782153424 - (0.32824948782153424)                                                                     \n",
      "VAL loss\t0.19624762345547878                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.19624762345547878\n",
      "        | \\     )|_\tf1: 0.32824948782153424\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\15\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 27.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         27                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0039087102652701836               |\n",
      "|  noam_learning_rate_warmup   |                        5597                       |\n",
      "|  noam_learning_rate_factor   |                 0.758650389870651                 |\n",
      "|          adam_beta1          |                 0.7229606303748571                |\n",
      "|          adam_beta2          |                 0.8745124809218658                |\n",
      "|           adam_eps           |               1.0405338445104391e-09              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.7940070280082968                |\n",
      "|     pointwise_layer_size     |                        223                        |\n",
      "|      last_layer_dropout      |                 0.7698009877702902                |\n",
      "|   output_conv_num_filters    |                        349                        |\n",
      "|   output_conv_kernel_size    |                         7                         |\n",
      "|      output_conv_stride      |                         5                         |\n",
      "|     output_conv_padding      |                         0                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        193                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 14%|█████▉                                    | 14/100 [3:25:19<26:05:35, 1092.28s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eec24101707d47a2b548da8e41434cdb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.51\t\t2.08\t\t0.017\t\t0.069\t\t3.87m - 3.9m / 0.0m                                                                    \n",
      "2\t34k\t0.56\t\t2.98\t\t0.015\t\t0.065\t\t4.01m - 8.0m / 135.5m                                                                  \n",
      "3\t51k\t0.65\t\t1.27\t\t0.015\t\t0.221\t\t4.28m - 12.3m / 140.2m                                                                 \n",
      "4\t68k\t0.71\t\t1.33\t\t0.017\t\t0.410\t\t4.18m - 16.5m / 149.4m                                                                 \n",
      "5\t85k\t0.77\t\t3.33\t\t0.018\t\t0.410\t\t3.96m - 20.5m / 146.2m                                                                 \n",
      "6\t102k\t0.88\t\t5.25\t\t0.010\t\t0.159\t\t4.04m - 24.5m / 139.2m                                                                \n",
      "7\t119k\t0.94\t\t1.06\t\t0.025\t\t0.366\t\t4.03m - 28.6m / 141.8m                                                                \n",
      "8\t137k\t1.09\t\t1.39\t\t0.020\t\t0.263\t\t3.96m - 32.6m / 141.5m                                                                \n",
      "9\t154k\t1.40\t\t2.95\t\t0.048\t\t0.188\t\t4.07m - 36.7m / 139.5m                                                                \n",
      "10\t171k\t1.84\t\t5.21\t\t0.047\t\t0.141\t\t4.20m - 40.9m / 142.6m                                                               \n",
      "11\t188k\t2.26\t\t6.27\t\t0.044\t\t0.093\t\t4.35m - 45.3m / 145.9m                                                               \n",
      "12\t205k\t2.73\t\t8.06\t\t0.045\t\t0.229\t\t4.29m - 49.6m / 149.6m                                                               \n",
      "13\t222k\t2.97\t\t7.40\t\t0.043\t\t0.092\t\t4.14m - 53.8m / 148.3m                                                               \n",
      "14\t239k\t3.21\t\t6.48\t\t0.050\t\t0.327\t\t4.02m - 57.8m / 144.9m                                                               \n",
      "15\t256k\t3.27\t\t10.67\t\t0.043\t\t0.092\t\t3.96m - 61.8m / 142.3m                                                              \n",
      "16\t273k\t3.40\t\t9.23\t\t0.041\t\t0.178\t\t3.97m - 65.8m / 141.0m                                                               \n",
      "17\t290k\t3.42\t\t9.42\t\t0.042\t\t0.180\t\t3.90m - 69.7m / 141.4m                                                               \n",
      "18\t307k\t3.37\t\t11.86\t\t0.038\t\t0.220\t\t3.94m - 73.7m / 139.9m                                                              \n",
      "19\t324k\t3.33\t\t12.59\t\t0.037\t\t0.219\t\t3.96m - 77.7m / 140.6m                                                              \n",
      "VAL f1\t0.05049820375516844 - (0.05049820375516844)                                                                     \n",
      "VAL loss\t1.0622889944452059                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 1.0622889944452059\n",
      "        | \\     )|_\tf1: 0.05049820375516844\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\16\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 34.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         34                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0031524856338611745               |\n",
      "|  noam_learning_rate_warmup   |                        7521                       |\n",
      "|  noam_learning_rate_factor   |                 1.753288795494817                 |\n",
      "|          adam_beta1          |                 0.8378339309167363                |\n",
      "|          adam_beta2          |                 0.7996293478324605                |\n",
      "|           adam_eps           |                0.17158258955881336                |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.28796899187970676                |\n",
      "|     pointwise_layer_size     |                        186                        |\n",
      "|      last_layer_dropout      |                 0.3970214206443492                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        173                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 15%|██████▎                                   | 15/100 [4:45:57<52:19:34, 2216.17s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5829384070b247db947af022447c0c17",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.50\t\t0.35\t\t0.307\t\t0.934\t\t1.78m - 1.8m / 0.0m                                                                    \n",
      "2\t34k\t0.37\t\t0.32\t\t0.272\t\t0.906\t\t1.77m - 3.6m / 62.5m                                                                   \n",
      "3\t51k\t0.32\t\t0.28\t\t0.278\t\t0.906\t\t1.83m - 5.4m / 62.1m                                                                   \n",
      "4\t68k\t0.29\t\t0.29\t\t0.192\t\t0.853\t\t1.79m - 7.2m / 64.0m                                                                   \n",
      "5\t85k\t0.26\t\t0.24\t\t0.304\t\t0.915\t\t1.81m - 9.1m / 62.9m                                                                   \n",
      "6\t102k\t0.25\t\t0.29\t\t0.207\t\t0.863\t\t1.79m - 10.9m / 63.2m                                                                 \n",
      "VAL f1\t0.30671296296296297 - (0.30671296296296297)                                                                     \n",
      "VAL loss\t0.2389094498486275                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2389094498486275\n",
      "        | \\     )|_\tf1: 0.30671296296296297\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\17\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 93.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         93                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.009656352559471478               |\n",
      "|  noam_learning_rate_warmup   |                        2374                       |\n",
      "|  noam_learning_rate_factor   |                 3.189143511649288                 |\n",
      "|          adam_beta1          |                 0.9175750640301038                |\n",
      "|          adam_beta2          |                 0.9403647576832646                |\n",
      "|           adam_eps           |               0.00010262669675726182              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.5315452978155136                |\n",
      "|     pointwise_layer_size     |                        252                        |\n",
      "|      last_layer_dropout      |                 0.6470491381722728                |\n",
      "|   output_conv_num_filters    |                        173                        |\n",
      "|   output_conv_kernel_size    |                         9                         |\n",
      "|      output_conv_stride      |                         9                         |\n",
      "|     output_conv_padding      |                         1                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        243                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 16%|██████▋                                   | 16/100 [4:58:13<41:21:04, 1772.20s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e06d934ee81340e8b4c4d75d96f9b315",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.19\t\t0.24\t\t0.045\t\t0.513\t\t2.37m - 2.4m / 0.0m                                                                    \n",
      "2\t34k\t0.18\t\t0.20\t\t0.105\t\t0.680\t\t2.39m - 4.8m / 82.9m                                                                   \n",
      "3\t51k\t0.18\t\t0.21\t\t0.108\t\t0.664\t\t2.46m - 7.3m / 83.7m                                                                   \n",
      "4\t68k\t0.18\t\t0.19\t\t0.184\t\t0.847\t\t2.48m - 9.8m / 86.2m                                                                   \n",
      "5\t86k\t0.18\t\t0.21\t\t0.154\t\t0.794\t\t2.47m - 12.3m / 86.6m                                                                  \n",
      "6\t103k\t0.18\t\t0.24\t\t0.081\t\t0.558\t\t2.48m - 14.8m / 86.5m                                                                 \n",
      "7\t120k\t0.19\t\t0.21\t\t0.099\t\t0.643\t\t2.49m - 17.3m / 86.7m                                                                 \n",
      "8\t137k\t0.20\t\t0.25\t\t0.074\t\t0.570\t\t2.49m - 19.8m / 87.1m                                                                 \n",
      "9\t154k\t0.21\t\t0.25\t\t0.067\t\t0.565\t\t2.47m - 22.3m / 87.1m                                                                 \n",
      "VAL f1\t0.18357369836225862 - (0.18357369836225862)                                                                     \n",
      "VAL loss\t0.18772084136720343                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.18772084136720343\n",
      "        | \\     )|_\tf1: 0.18357369836225862\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\18\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 71.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         71                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.006087647785782573               |\n",
      "|  noam_learning_rate_warmup   |                        8825                       |\n",
      "|  noam_learning_rate_factor   |                 1.4115544921993006                |\n",
      "|          adam_beta1          |                 0.8397919210325919                |\n",
      "|          adam_beta2          |                 0.9747291864176111                |\n",
      "|           adam_eps           |               1.2302839619615785e-08              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.6168968385089014                |\n",
      "|     pointwise_layer_size     |                         36                        |\n",
      "|      last_layer_dropout      |                 0.6178784550309249                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        163                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 17%|███████▏                                  | 17/100 [5:22:40<38:44:28, 1680.35s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5176c51f350420b99ca7e2acfdb040d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.31\t\t0.21\t\t0.241\t\t0.903\t\t1.60m - 1.6m / 0.0m                                                                    \n",
      "2\t34k\t0.22\t\t0.21\t\t0.210\t\t0.871\t\t1.59m - 3.2m / 55.9m                                                                   \n",
      "3\t51k\t0.21\t\t0.18\t\t0.240\t\t0.887\t\t1.60m - 4.8m / 55.8m                                                                   \n",
      "4\t68k\t0.18\t\t0.21\t\t0.136\t\t0.779\t\t1.61m - 6.5m / 56.0m                                                                   \n",
      "5\t86k\t0.17\t\t0.28\t\t0.098\t\t0.683\t\t1.57m - 8.1m / 56.4m                                                                   \n",
      "6\t103k\t0.16\t\t0.24\t\t0.121\t\t0.731\t\t1.57m - 9.6m / 55.3m                                                                  \n",
      "VAL f1\t0.2414651493598862 - (0.2414651493598862)                                                                       \n",
      "VAL loss\t0.18423130744982252                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.18423130744982252\n",
      "        | \\     )|_\tf1: 0.2414651493598862\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\19\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 86.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         86                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.014889909052037794               |\n",
      "|  noam_learning_rate_warmup   |                        4461                       |\n",
      "|  noam_learning_rate_factor   |                 3.262351776702605                 |\n",
      "|          adam_beta1          |                 0.7788722247719246                |\n",
      "|          adam_beta2          |                 0.8873375366783246                |\n",
      "|           adam_eps           |               1.026826438126101e-06               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.2818659819165862                |\n",
      "|     pointwise_layer_size     |                        183                        |\n",
      "|      last_layer_dropout      |                 0.4814375074498509                |\n",
      "|   output_conv_num_filters    |                        280                        |\n",
      "|   output_conv_kernel_size    |                         3                         |\n",
      "|      output_conv_stride      |                         7                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         13                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 18%|███████▌                                  | 18/100 [5:33:47<31:21:10, 1376.47s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f6fd707c1104fd884f01693ae709bef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.19\t\t0.16\t\t0.205\t\t0.851\t\t0.67m - 0.7m / 0.0m                                                                    \n",
      "2\t34k\t0.16\t\t0.16\t\t0.149\t\t0.797\t\t0.67m - 1.4m / 23.6m                                                                   \n",
      "3\t51k\t0.15\t\t0.15\t\t0.270\t\t0.899\t\t0.65m - 2.0m / 23.4m                                                                   \n",
      "4\t68k\t0.15\t\t0.16\t\t0.261\t\t0.896\t\t0.65m - 2.8m / 22.9m                                                                   \n",
      "5\t86k\t0.15\t\t0.14\t\t0.296\t\t0.912\t\t0.65m - 3.5m / 23.0m                                                                   \n",
      "6\t103k\t0.15\t\t0.14\t\t0.233\t\t0.876\t\t0.65m - 4.1m / 23.0m                                                                  \n",
      "7\t120k\t0.15\t\t0.16\t\t0.246\t\t0.887\t\t0.65m - 4.8m / 22.9m                                                                  \n",
      "8\t137k\t0.15\t\t0.15\t\t0.324\t\t0.927\t\t0.65m - 5.4m / 22.9m                                                                  \n",
      "9\t154k\t0.16\t\t0.17\t\t0.201\t\t0.847\t\t0.66m - 6.1m / 22.9m                                                                  \n",
      "10\t171k\t0.16\t\t0.15\t\t0.282\t\t0.907\t\t0.66m - 6.8m / 23.3m                                                                 \n",
      "11\t188k\t0.17\t\t0.16\t\t0.164\t\t0.801\t\t0.66m - 7.5m / 23.2m                                                                 \n",
      "12\t205k\t0.18\t\t0.18\t\t0.151\t\t0.782\t\t0.66m - 8.1m / 23.2m                                                                 \n",
      "13\t222k\t0.18\t\t0.17\t\t0.145\t\t0.759\t\t0.66m - 8.8m / 23.3m                                                                 \n",
      "VAL f1\t0.3235294117647059 - (0.3235294117647059)                                                                       \n",
      "VAL loss\t0.1387040615081787                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.1387040615081787\n",
      "        | \\     )|_\tf1: 0.3235294117647059\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\20\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 90.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         90                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.3526516851824026                |\n",
      "|  noam_learning_rate_warmup   |                        8832                       |\n",
      "|  noam_learning_rate_factor   |                0.22208571663921559                |\n",
      "|          adam_beta1          |                 0.8166790577422153                |\n",
      "|          adam_beta2          |                 0.916246312706396                 |\n",
      "|           adam_eps           |               0.0013456080478003757               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.5659550544879457                |\n",
      "|     pointwise_layer_size     |                        146                        |\n",
      "|      last_layer_dropout      |                 0.6282279390173042                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        103                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 19%|███████▉                                  | 19/100 [5:43:29<25:36:24, 1138.08s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf1f5e64278240f38f329d1b99d6bc24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.30\t\t0.24\t\t0.109\t\t0.720\t\t1.37m - 1.4m / 0.0m                                                                    \n",
      "2\t34k\t0.22\t\t0.17\t\t0.189\t\t0.870\t\t1.38m - 2.8m / 48.1m                                                                   \n",
      "3\t51k\t0.19\t\t0.17\t\t0.182\t\t0.860\t\t1.38m - 4.2m / 48.2m                                                                   \n",
      "4\t68k\t0.18\t\t0.17\t\t0.219\t\t0.889\t\t1.39m - 5.6m / 48.3m                                                                   \n",
      "5\t86k\t0.18\t\t0.17\t\t0.188\t\t0.860\t\t1.40m - 7.0m / 48.6m                                                                   \n",
      "6\t103k\t0.18\t\t0.17\t\t0.176\t\t0.841\t\t1.40m - 8.4m / 49.1m                                                                  \n",
      "7\t120k\t0.17\t\t0.19\t\t0.109\t\t0.701\t\t1.43m - 9.9m / 49.0m                                                                  \n",
      "8\t137k\t0.17\t\t0.21\t\t0.092\t\t0.694\t\t1.38m - 11.3m / 49.9m                                                                 \n",
      "9\t154k\t0.16\t\t0.27\t\t0.054\t\t0.576\t\t1.39m - 12.7m / 48.6m                                                                 \n",
      "VAL f1\t0.2192289078883691 - (0.2192289078883691)                                                                       \n",
      "VAL loss\t0.16702494229671458                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.16702494229671458\n",
      "        | \\     )|_\tf1: 0.2192289078883691\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\21\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 87.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         87                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.001557106598156508               |\n",
      "|  noam_learning_rate_warmup   |                        8851                       |\n",
      "|  noam_learning_rate_factor   |                 2.5572027163227875                |\n",
      "|          adam_beta1          |                 0.9155019261698589                |\n",
      "|          adam_beta2          |                 0.9300790172896248                |\n",
      "|           adam_eps           |               2.013849949088891e-10               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.001023888927194072               |\n",
      "|     pointwise_layer_size     |                         39                        |\n",
      "|      last_layer_dropout      |                 0.2026795958317137                |\n",
      "|   output_conv_num_filters    |                        322                        |\n",
      "|   output_conv_kernel_size    |                         7                         |\n",
      "|      output_conv_stride      |                         6                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         18                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 20%|████████▍                                 | 20/100 [5:57:20<23:14:47, 1046.10s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef30997ace1544de9627303e8e829c16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.20\t\t0.16\t\t0.263\t\t0.905\t\t1.08m - 1.1m / 0.0m                                                                    \n",
      "2\t34k\t0.15\t\t0.15\t\t0.270\t\t0.903\t\t1.09m - 2.2m / 37.7m                                                                   \n",
      "3\t51k\t0.12\t\t0.15\t\t0.265\t\t0.901\t\t1.06m - 3.3m / 38.3m                                                                   \n",
      "4\t68k\t0.09\t\t0.15\t\t0.262\t\t0.899\t\t1.07m - 4.4m / 37.3m                                                                   \n",
      "5\t85k\t0.06\t\t0.17\t\t0.277\t\t0.915\t\t1.06m - 5.5m / 37.4m                                                                   \n",
      "6\t102k\t0.04\t\t0.18\t\t0.243\t\t0.903\t\t1.09m - 6.6m / 37.4m                                                                  \n",
      "7\t119k\t0.03\t\t0.23\t\t0.326\t\t0.933\t\t1.10m - 7.7m / 38.3m                                                                  \n",
      "8\t136k\t0.03\t\t0.28\t\t0.382\t\t0.950\t\t1.06m - 8.8m / 38.4m                                                                  \n",
      "9\t153k\t0.02\t\t0.27\t\t0.326\t\t0.936\t\t1.10m - 9.9m / 37.5m                                                                  \n",
      "10\t171k\t0.02\t\t0.34\t\t0.319\t\t0.933\t\t1.10m - 11.0m / 38.6m                                                                \n",
      "11\t188k\t0.02\t\t0.37\t\t0.350\t\t0.944\t\t1.09m - 12.1m / 38.5m                                                                \n",
      "12\t205k\t0.02\t\t0.40\t\t0.343\t\t0.944\t\t1.09m - 13.2m / 38.2m                                                                \n",
      "13\t222k\t0.03\t\t0.41\t\t0.323\t\t0.936\t\t1.08m - 14.3m / 38.2m                                                                \n",
      "VAL f1\t0.3820436775411162 - (0.3820436775411162)                                                                       \n",
      "VAL loss\t0.14693108075423258                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.14693108075423258\n",
      "        | \\     )|_\tf1: 0.3820436775411162\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\22\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 39.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         39                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.1734069442119519                |\n",
      "|  noam_learning_rate_warmup   |                        7155                       |\n",
      "|  noam_learning_rate_factor   |                 2.3731370057171577                |\n",
      "|          adam_beta1          |                 0.9568912098581941                |\n",
      "|          adam_beta2          |                 0.9927936997028899                |\n",
      "|           adam_eps           |               1.4894687609501842e-08              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.3913510612398751                |\n",
      "|     pointwise_layer_size     |                        164                        |\n",
      "|      last_layer_dropout      |                 0.2323477084071814                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         33                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 21%|████████▊                                 | 21/100 [6:12:26<22:01:56, 1004.01s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b67cd80f78284804826e08e0c35a1c95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.44\t\t0.30\t\t0.332\t\t0.940\t\t0.85m - 0.9m / 0.0m                                                                    \n",
      "2\t34k\t0.31\t\t0.26\t\t0.318\t\t0.923\t\t0.87m - 1.7m / 29.9m                                                                   \n",
      "3\t51k\t0.26\t\t0.25\t\t0.345\t\t0.932\t\t0.89m - 2.6m / 30.3m                                                                   \n",
      "4\t68k\t0.23\t\t0.26\t\t0.239\t\t0.877\t\t0.84m - 3.5m / 31.1m                                                                   \n",
      "5\t85k\t0.20\t\t0.25\t\t0.297\t\t0.908\t\t0.85m - 4.4m / 29.6m                                                                   \n",
      "6\t102k\t0.18\t\t0.26\t\t0.292\t\t0.906\t\t0.84m - 5.2m / 29.9m                                                                  \n",
      "7\t119k\t0.15\t\t0.26\t\t0.394\t\t0.942\t\t0.85m - 6.1m / 29.7m                                                                  \n",
      "8\t136k\t0.13\t\t0.29\t\t0.358\t\t0.933\t\t0.86m - 7.0m / 29.8m                                                                  \n",
      "9\t153k\t0.11\t\t0.33\t\t0.352\t\t0.934\t\t0.88m - 7.8m / 30.1m                                                                  \n",
      "10\t170k\t0.10\t\t0.39\t\t0.408\t\t0.947\t\t0.86m - 8.7m / 30.7m                                                                 \n",
      "11\t187k\t0.08\t\t0.43\t\t0.402\t\t0.949\t\t0.84m - 9.6m / 30.2m                                                                 \n",
      "12\t205k\t0.07\t\t0.44\t\t0.402\t\t0.949\t\t0.84m - 10.4m / 29.8m                                                                \n",
      "13\t222k\t0.06\t\t0.51\t\t0.406\t\t0.951\t\t0.84m - 11.3m / 29.8m                                                                \n",
      "14\t239k\t0.06\t\t0.53\t\t0.397\t\t0.947\t\t0.85m - 12.1m / 29.7m                                                                \n",
      "15\t256k\t0.06\t\t0.56\t\t0.434\t\t0.958\t\t0.84m - 13.0m / 29.9m                                                                \n",
      "16\t273k\t0.05\t\t0.56\t\t0.394\t\t0.947\t\t0.85m - 13.9m / 29.9m                                                                \n",
      "17\t290k\t0.04\t\t0.61\t\t0.399\t\t0.950\t\t0.85m - 14.7m / 30.0m                                                                \n",
      "18\t307k\t0.04\t\t0.63\t\t0.372\t\t0.946\t\t0.85m - 15.6m / 30.1m                                                                \n",
      "19\t324k\t0.03\t\t0.70\t\t0.429\t\t0.957\t\t0.85m - 16.5m / 30.1m                                                                \n",
      "20\t341k\t0.03\t\t0.73\t\t0.441\t\t0.958\t\t0.86m - 17.3m / 30.1m                                                                \n",
      "21\t358k\t0.03\t\t0.78\t\t0.435\t\t0.958\t\t0.85m - 18.2m / 30.2m                                                                \n",
      "22\t375k\t0.03\t\t0.75\t\t0.433\t\t0.957\t\t0.85m - 19.1m / 30.1m                                                                \n",
      "23\t392k\t0.02\t\t0.84\t\t0.445\t\t0.961\t\t0.85m - 19.9m / 30.1m                                                                \n",
      "24\t409k\t0.02\t\t0.84\t\t0.441\t\t0.959\t\t0.85m - 20.8m / 30.1m                                                                \n",
      "25\t426k\t0.02\t\t0.87\t\t0.426\t\t0.958\t\t0.85m - 21.7m / 30.1m                                                                \n",
      "26\t443k\t0.02\t\t0.94\t\t0.442\t\t0.960\t\t0.85m - 22.5m / 30.2m                                                                \n",
      "27\t460k\t0.02\t\t0.88\t\t0.436\t\t0.959\t\t0.85m - 23.4m / 30.1m                                                                \n",
      "28\t477k\t0.02\t\t0.98\t\t0.448\t\t0.961\t\t0.85m - 24.2m / 30.2m                                                                \n",
      "29\t494k\t0.02\t\t0.98\t\t0.447\t\t0.961\t\t0.88m - 25.1m / 30.2m                                                                \n",
      "30\t511k\t0.02\t\t0.97\t\t0.445\t\t0.960\t\t0.88m - 26.0m / 30.4m                                                                \n",
      "31\t528k\t0.02\t\t0.97\t\t0.450\t\t0.961\t\t0.85m - 26.9m / 30.4m                                                                \n",
      "32\t545k\t0.02\t\t0.93\t\t0.442\t\t0.959\t\t0.85m - 27.8m / 30.3m                                                                \n",
      "33\t562k\t0.02\t\t1.00\t\t0.437\t\t0.959\t\t0.86m - 28.6m / 30.3m                                                                \n",
      "34\t579k\t0.02\t\t1.08\t\t0.451\t\t0.962\t\t0.85m - 29.5m / 30.4m                                                                \n",
      "35\t597k\t0.02\t\t1.05\t\t0.440\t\t0.961\t\t0.86m - 30.4m / 30.4m                                                                \n",
      "VAL f1\t0.45082938388625593 - (0.45082938388625593)                                                                     \n",
      "VAL loss\t0.24528811135958747                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.24528811135958747\n",
      "        | \\     )|_\tf1: 0.45082938388625593\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\23\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 89.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         89                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.03767102695634418                |\n",
      "|  noam_learning_rate_warmup   |                        1955                       |\n",
      "|  noam_learning_rate_factor   |                 0.7000613834025997                |\n",
      "|          adam_beta1          |                 0.7501423478926437                |\n",
      "|          adam_beta2          |                 0.743620253361487                 |\n",
      "|           adam_eps           |               2.485633168435825e-08               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.2567800961381143                |\n",
      "|     pointwise_layer_size     |                         90                        |\n",
      "|      last_layer_dropout      |                 0.6569817782771766                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        172                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------+---------------------------------------------------+\n",
      " 22%|█████████▏                                | 22/100 [6:43:51<27:28:55, 1268.40s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db399e0b51064cc0a8cb45d235828195",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.21\t\t0.16\t\t0.240\t\t0.899\t\t1.40m - 1.4m / 0.0m                                                                    \n",
      "2\t34k\t0.16\t\t0.13\t\t0.211\t\t0.868\t\t1.39m - 2.8m / 48.9m                                                                   \n",
      "3\t51k\t0.13\t\t0.14\t\t0.180\t\t0.821\t\t1.39m - 4.2m / 48.8m                                                                   \n",
      "4\t68k\t0.12\t\t0.14\t\t0.181\t\t0.840\t\t1.39m - 5.6m / 48.8m                                                                   \n",
      "5\t85k\t0.12\t\t0.14\t\t0.201\t\t0.842\t\t1.40m - 7.0m / 48.7m                                                                   \n",
      "6\t103k\t0.11\t\t0.13\t\t0.230\t\t0.862\t\t1.42m - 8.5m / 49.1m                                                                  \n",
      "VAL f1\t0.24014336917562723 - (0.24014336917562723)                                                                     \n",
      "VAL loss\t0.13256469067562834                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.13256469067562834\n",
      "        | \\     )|_\tf1: 0.24014336917562723\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\24\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 74.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         74                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.028082669147584418               |\n",
      "|  noam_learning_rate_warmup   |                        2761                       |\n",
      "|  noam_learning_rate_factor   |                 0.8839029905774988                |\n",
      "|          adam_beta1          |                 0.7134529319485204                |\n",
      "|          adam_beta2          |                 0.751712427111727                 |\n",
      "|           adam_eps           |               3.519919083688092e-07               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.024331296975042174               |\n",
      "|     pointwise_layer_size     |                        109                        |\n",
      "|      last_layer_dropout      |                0.08995045094748147                |\n",
      "|   output_conv_num_filters    |                         4                         |\n",
      "|   output_conv_kernel_size    |                         5                         |\n",
      "|      output_conv_stride      |                         5                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         31                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 23%|█████████▋                                | 23/100 [6:53:42<22:47:01, 1065.22s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a2e6cb789da4d09b69f17043e76412a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.30\t\t0.25\t\t0.104\t\t0.702\t\t1.15m - 1.1m / 0.0m                                                                    \n",
      "2\t34k\t0.25\t\t0.24\t\t0.114\t\t0.723\t\t1.15m - 2.3m / 40.2m                                                                   \n",
      "3\t51k\t0.24\t\t0.23\t\t0.129\t\t0.777\t\t1.12m - 3.5m / 40.3m                                                                   \n",
      "4\t68k\t0.23\t\t0.22\t\t0.132\t\t0.781\t\t1.14m - 4.6m / 39.5m                                                                   \n",
      "5\t85k\t0.22\t\t0.22\t\t0.193\t\t0.859\t\t1.12m - 5.8m / 39.9m                                                                   \n",
      "6\t103k\t0.21\t\t0.20\t\t0.217\t\t0.873\t\t1.14m - 6.9m / 39.5m                                                                  \n",
      "7\t120k\t0.20\t\t0.20\t\t0.226\t\t0.884\t\t1.11m - 8.1m / 40.0m                                                                  \n",
      "8\t137k\t0.20\t\t0.19\t\t0.239\t\t0.896\t\t1.13m - 9.2m / 39.3m                                                                  \n",
      "9\t154k\t0.19\t\t0.18\t\t0.325\t\t0.935\t\t1.11m - 10.4m / 39.9m                                                                 \n",
      "10\t171k\t0.19\t\t0.20\t\t0.264\t\t0.909\t\t1.14m - 11.5m / 39.4m                                                                \n",
      "11\t188k\t0.20\t\t0.19\t\t0.349\t\t0.949\t\t1.13m - 12.7m / 40.0m                                                                \n",
      "12\t205k\t0.20\t\t0.21\t\t0.185\t\t0.856\t\t1.11m - 13.8m / 39.9m                                                                \n",
      "13\t222k\t0.20\t\t0.20\t\t0.330\t\t0.946\t\t1.11m - 14.9m / 39.2m                                                                \n",
      "14\t239k\t0.21\t\t0.20\t\t0.226\t\t0.889\t\t1.11m - 16.1m / 39.3m                                                                \n",
      "15\t256k\t0.21\t\t0.21\t\t0.330\t\t0.946\t\t1.11m - 17.2m / 39.5m                                                                \n",
      "16\t274k\t0.21\t\t0.20\t\t0.275\t\t0.925\t\t1.11m - 18.3m / 39.3m                                                                \n",
      "VAL f1\t0.3493844049247606 - (0.3493844049247606)                                                                       \n",
      "VAL loss\t0.18111076111038682                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.18111076111038682\n",
      "        | \\     )|_\tf1: 0.3493844049247606\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\25\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 24%|██████████                                | 24/100 [7:13:01<23:04:41, 1093.18s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57ddbf5ec42d4f59b3d37da1c47f24a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.55\t\t0.37\t\t0.247\t\t0.888\t\t4.51m - 4.5m / 0.0m                                                                    \n",
      "2\t34k\t0.38\t\t0.36\t\t0.232\t\t0.879\t\t4.55m - 9.1m / 158.0m                                                                  \n",
      "3\t51k\t0.36\t\t0.37\t\t0.217\t\t0.869\t\t4.58m - 13.7m / 159.4m                                                                 \n",
      "4\t68k\t0.35\t\t0.37\t\t0.182\t\t0.865\t\t4.55m - 18.3m / 160.2m                                                                 \n",
      "5\t85k\t0.35\t\t0.34\t\t0.294\t\t0.910\t\t4.54m - 22.8m / 159.4m                                                                 \n",
      "6\t102k\t0.35\t\t0.35\t\t0.341\t\t0.930\t\t4.58m - 27.4m / 159.1m                                                                \n",
      "7\t119k\t0.35\t\t0.34\t\t0.334\t\t0.932\t\t4.55m - 32.0m / 160.3m                                                                \n",
      "8\t136k\t0.36\t\t0.33\t\t0.311\t\t0.925\t\t4.53m - 36.5m / 159.3m                                                                \n",
      "9\t154k\t0.35\t\t0.40\t\t0.343\t\t0.934\t\t4.61m - 41.2m / 159.0m                                                                \n",
      "10\t171k\t0.34\t\t0.37\t\t0.345\t\t0.929\t\t4.53m - 45.7m / 161.0m                                                               \n",
      "11\t188k\t0.33\t\t0.43\t\t0.378\t\t0.945\t\t4.52m - 50.3m / 158.9m                                                               \n",
      "12\t205k\t0.32\t\t0.46\t\t0.269\t\t0.917\t\t4.56m - 54.9m / 158.8m                                                               \n",
      "13\t222k\t0.30\t\t0.36\t\t0.376\t\t0.941\t\t4.54m - 59.4m / 159.8m                                                               \n",
      "14\t239k\t0.29\t\t0.46\t\t0.421\t\t0.952\t\t4.52m - 63.9m / 159.3m                                                               \n",
      "15\t256k\t0.28\t\t0.42\t\t0.374\t\t0.943\t\t4.52m - 68.5m / 158.9m                                                               \n",
      "16\t273k\t0.27\t\t0.44\t\t0.406\t\t0.952\t\t4.53m - 73.0m / 159.0m                                                               \n",
      "17\t290k\t0.25\t\t0.44\t\t0.393\t\t0.947\t\t4.52m - 77.6m / 159.1m                                                               \n",
      "18\t307k\t0.24\t\t0.56\t\t0.390\t\t0.949\t\t4.52m - 82.1m / 159.0m                                                               \n",
      "19\t324k\t0.23\t\t0.57\t\t0.433\t\t0.958\t\t4.52m - 86.6m / 159.0m                                                               \n",
      "20\t341k\t0.23\t\t0.46\t\t0.382\t\t0.944\t\t4.49m - 91.2m / 159.0m                                                               \n",
      "21\t358k\t0.22\t\t0.54\t\t0.442\t\t0.957\t\t4.50m - 95.7m / 158.6m                                                               \n",
      "22\t375k\t0.21\t\t0.56\t\t0.367\t\t0.946\t\t4.49m - 100.2m / 158.6m                                                              \n",
      "23\t392k\t0.20\t\t0.68\t\t0.454\t\t0.960\t\t4.49m - 104.7m / 158.6m                                                              \n",
      "24\t409k\t0.20\t\t0.66\t\t0.442\t\t0.959\t\t4.49m - 109.2m / 158.7m                                                              \n",
      "25\t426k\t0.19\t\t0.60\t\t0.374\t\t0.948\t\t4.57m - 113.8m / 158.6m                                                              \n",
      "26\t443k\t0.19\t\t0.63\t\t0.425\t\t0.955\t\t4.51m - 118.3m / 159.5m                                                              \n",
      "27\t461k\t0.19\t\t0.61\t\t0.444\t\t0.958\t\t4.53m - 122.9m / 158.9m                                                              \n",
      "28\t478k\t0.18\t\t0.59\t\t0.377\t\t0.946\t\t4.51m - 127.4m / 159.1m                                                              \n",
      "VAL f1\t0.4536741214057508 - (0.4536741214057508)                                                                       \n",
      "VAL loss\t0.3319024578273703                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3319024578273703\n",
      "        | \\     )|_\tf1: 0.4536741214057508\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\26\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 71.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         71                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.015652031912205898               |\n",
      "|  noam_learning_rate_warmup   |                        5046                       |\n",
      "|  noam_learning_rate_factor   |                0.14787708893178422                |\n",
      "|          adam_beta1          |                 0.8956588431412156                |\n",
      "|          adam_beta2          |                 0.8061947273795329                |\n",
      "|           adam_eps           |               4.5799143863503846e-05              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.5950348720595402                |\n",
      "|     pointwise_layer_size     |                         56                        |\n",
      "|      last_layer_dropout      |                 0.5189430739425561                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        154                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------+---------------------------------------------------+\n",
      " 25%|██████████▌                               | 25/100 [9:23:22<64:49:21, 3111.49s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d94a94e241aa48eca3501233fe7a488b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.37\t\t0.25\t\t0.133\t\t0.787\t\t1.04m - 1.0m / 0.0m                                                                    \n",
      "2\t34k\t0.24\t\t0.22\t\t0.318\t\t0.941\t\t1.06m - 2.1m / 36.4m                                                                   \n",
      "3\t51k\t0.22\t\t0.21\t\t0.296\t\t0.934\t\t1.03m - 3.2m / 37.2m                                                                   \n",
      "4\t68k\t0.21\t\t0.21\t\t0.286\t\t0.926\t\t1.03m - 4.2m / 36.3m                                                                   \n",
      "5\t86k\t0.21\t\t0.20\t\t0.234\t\t0.883\t\t1.03m - 5.3m / 36.2m                                                                   \n",
      "6\t103k\t0.20\t\t0.20\t\t0.188\t\t0.835\t\t1.06m - 6.4m / 36.3m                                                                  \n",
      "7\t120k\t0.19\t\t0.19\t\t0.204\t\t0.848\t\t1.03m - 7.4m / 37.0m                                                                  \n",
      "VAL f1\t0.31843434343434346 - (0.31843434343434346)                                                                     \n",
      "VAL loss\t0.1882128615932826                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.1882128615932826\n",
      "        | \\     )|_\tf1: 0.31843434343434346\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\27\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 80.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         80                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.004480339695933948               |\n",
      "|  noam_learning_rate_warmup   |                        3714                       |\n",
      "|  noam_learning_rate_factor   |                 1.165859327043268                 |\n",
      "|          adam_beta1          |                 0.9490985652032606                |\n",
      "|          adam_beta2          |                 0.7696869869548763                |\n",
      "|           adam_eps           |                0.02520636577748654                |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.7121775404236956                |\n",
      "|     pointwise_layer_size     |                        174                        |\n",
      "|      last_layer_dropout      |                 0.1028533869337486                |\n",
      "|   output_conv_num_filters    |                        199                        |\n",
      "|   output_conv_kernel_size    |                         7                         |\n",
      "|      output_conv_stride      |                         2                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         61                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 26%|██████████▉                               | 26/100 [9:31:48<47:53:33, 2329.92s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7866b56423bb4785bf00b50e6e87132a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exception while training: size mismatch, m1: [15920 x 2], m2: [199 x 4] at c:\\a\\w\\1\\s\\tmp_conda_3.6_105809\\conda\\conda-bld\\pytorch_1544094150554\\work\\aten\\src\\thc\\generic/THCTensorMathBlas.cu:266\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\28\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 46.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         46                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0020339997565108816               |\n",
      "|  noam_learning_rate_warmup   |                        4362                       |\n",
      "|  noam_learning_rate_factor   |                 3.6631650685836696                |\n",
      "|          adam_beta1          |                 0.9487681739793696                |\n",
      "|          adam_beta2          |                 0.8611279389547384                |\n",
      "|           adam_eps           |                0.00944947701472517                |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.0903292018045888                |\n",
      "|     pointwise_layer_size     |                        249                        |\n",
      "|      last_layer_dropout      |                 0.6186602973414121                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        140                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 27%|███████████▎                              | 27/100 [9:32:03<33:09:57, 1635.58s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "434790af43aa43349dbb9ba2b3f39150",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.34\t\t0.26\t\t0.214\t\t0.861\t\t1.37m - 1.4m / 0.0m                                                                    \n",
      "2\t34k\t0.25\t\t0.22\t\t0.246\t\t0.884\t\t1.39m - 2.8m / 48.1m                                                                   \n",
      "3\t51k\t0.24\t\t0.23\t\t0.392\t\t0.950\t\t1.36m - 4.2m / 48.6m                                                                   \n",
      "4\t68k\t0.24\t\t0.22\t\t0.267\t\t0.895\t\t1.37m - 5.6m / 47.9m                                                                   \n",
      "5\t85k\t0.23\t\t0.23\t\t0.280\t\t0.904\t\t1.37m - 7.0m / 48.2m                                                                   \n",
      "6\t102k\t0.22\t\t0.24\t\t0.230\t\t0.883\t\t1.37m - 8.3m / 48.2m                                                                  \n",
      "7\t119k\t0.21\t\t0.24\t\t0.290\t\t0.909\t\t1.37m - 9.7m / 48.2m                                                                  \n",
      "8\t137k\t0.20\t\t0.22\t\t0.347\t\t0.932\t\t1.37m - 11.1m / 48.2m                                                                 \n",
      "VAL f1\t0.3921011874032008 - (0.3921011874032008)                                                                       \n",
      "VAL loss\t0.21923767794733456                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.21923767794733456\n",
      "        | \\     )|_\tf1: 0.3921011874032008\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\29\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 11.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         11                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.014836120359227128               |\n",
      "|  noam_learning_rate_warmup   |                        3305                       |\n",
      "|  noam_learning_rate_factor   |                 2.3237434861167867                |\n",
      "|          adam_beta1          |                 0.7228055099443881                |\n",
      "|          adam_beta2          |                 0.7795233682218415                |\n",
      "|           adam_eps           |               6.107894303072867e-08               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.16514505372582233                |\n",
      "|     pointwise_layer_size     |                        125                        |\n",
      "|      last_layer_dropout      |                0.15845872383954102                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         19                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 28%|███████████▊                              | 28/100 [9:44:15<27:17:04, 1364.22s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6d395466f9840cc931f720ad81113f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.96\t\t0.95\t\t0.382\t\t0.959\t\t5.52m - 5.5m / 0.0m                                                                    \n",
      "2\t34k\t1.00\t\t0.89\t\t0.265\t\t0.917\t\t5.67m - 11.2m / 193.4m                                                                 \n",
      "3\t51k\t0.92\t\t0.86\t\t0.382\t\t0.959\t\t5.72m - 17.0m / 198.3m                                                                 \n",
      "4\t68k\t0.88\t\t0.82\t\t0.382\t\t0.959\t\t5.63m - 22.6m / 200.1m                                                                 \n",
      "5\t85k\t0.87\t\t0.86\t\t0.382\t\t0.959\t\t5.62m - 28.2m / 197.0m                                                                 \n",
      "6\t102k\t0.93\t\t0.87\t\t0.382\t\t0.959\t\t5.57m - 33.8m / 196.8m                                                                \n",
      "VAL f1\t0.3821757056771329 - (0.3821757056771329)                                                                       \n",
      "VAL loss\t0.8244816759690882                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.8244816759690882\n",
      "        | \\     )|_\tf1: 0.3821757056771329\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\30\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 77.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         77                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.022601426594520947               |\n",
      "|  noam_learning_rate_warmup   |                        5840                       |\n",
      "|  noam_learning_rate_factor   |                 2.023544350596826                 |\n",
      "|          adam_beta1          |                 0.9349986447370091                |\n",
      "|          adam_beta2          |                 0.9491886040550691                |\n",
      "|           adam_eps           |               4.063788685108895e-07               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.7686073482652587                |\n",
      "|     pointwise_layer_size     |                         32                        |\n",
      "|      last_layer_dropout      |                0.12757307166562545                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         46                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 29%|███████████▉                             | 29/100 [10:22:11<32:18:03, 1637.79s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e4789ae777244a0a1c420150fbf17a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.28\t\t0.22\t\t0.319\t\t0.942\t\t1.13m - 1.1m / 0.0m                                                                    \n",
      "2\t34k\t0.21\t\t0.24\t\t0.218\t\t0.888\t\t1.11m - 2.3m / 39.6m                                                                   \n",
      "3\t51k\t0.21\t\t0.32\t\t0.120\t\t0.758\t\t1.11m - 3.4m / 39.0m                                                                   \n",
      "4\t68k\t0.20\t\t0.24\t\t0.310\t\t0.939\t\t1.10m - 4.5m / 39.0m                                                                   \n",
      "5\t85k\t0.20\t\t0.25\t\t0.203\t\t0.875\t\t1.13m - 5.7m / 38.7m                                                                   \n",
      "6\t103k\t0.20\t\t0.29\t\t0.118\t\t0.743\t\t1.07m - 6.7m / 39.5m                                                                  \n",
      "VAL f1\t0.31937984496124033 - (0.31937984496124033)                                                                     \n",
      "VAL loss\t0.22277622729783567                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.22277622729783567\n",
      "        | \\     )|_\tf1: 0.31937984496124033\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\31\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 79.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         79                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.003565147027398163               |\n",
      "|  noam_learning_rate_warmup   |                        8634                       |\n",
      "|  noam_learning_rate_factor   |                 2.5530013342461815                |\n",
      "|          adam_beta1          |                 0.8856019880375663                |\n",
      "|          adam_beta2          |                 0.727144748530047                 |\n",
      "|           adam_eps           |                 0.4434391039621775                |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.465056360812745                 |\n",
      "|     pointwise_layer_size     |                        188                        |\n",
      "|      last_layer_dropout      |                0.18655244642157573                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        171                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 30%|████████████▎                            | 30/100 [10:29:50<24:58:21, 1284.31s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35a038a20fa54933abaa7b2299e087c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.32\t\t0.20\t\t0.270\t\t0.922\t\t0.97m - 1.0m / 0.0m                                                                    \n",
      "2\t34k\t0.21\t\t0.19\t\t0.283\t\t0.925\t\t0.97m - 2.0m / 34.0m                                                                   \n",
      "3\t51k\t0.20\t\t0.19\t\t0.325\t\t0.943\t\t0.97m - 3.0m / 34.0m                                                                   \n",
      "4\t68k\t0.20\t\t0.19\t\t0.286\t\t0.926\t\t0.97m - 4.0m / 34.1m                                                                   \n",
      "5\t85k\t0.20\t\t0.19\t\t0.305\t\t0.934\t\t0.97m - 5.0m / 34.0m                                                                   \n",
      "6\t102k\t0.20\t\t0.18\t\t0.287\t\t0.927\t\t0.97m - 5.9m / 34.1m                                                                  \n",
      "7\t119k\t0.20\t\t0.18\t\t0.304\t\t0.933\t\t0.99m - 6.9m / 34.1m                                                                  \n",
      "8\t137k\t0.20\t\t0.18\t\t0.285\t\t0.923\t\t1.00m - 7.9m / 34.6m                                                                  \n",
      "VAL f1\t0.3247052793439262 - (0.3247052793439262)                                                                       \n",
      "VAL loss\t0.18414371828489665                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.18414371828489665\n",
      "        | \\     )|_\tf1: 0.3247052793439262\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\32\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 89.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         89                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.5911215488410878                |\n",
      "|  noam_learning_rate_warmup   |                        3481                       |\n",
      "|  noam_learning_rate_factor   |                 2.480851484161066                 |\n",
      "|          adam_beta1          |                 0.984258886564946                 |\n",
      "|          adam_beta2          |                 0.8546806263752986                |\n",
      "|           adam_eps           |                0.045251639104696297               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.6838714614113256                |\n",
      "|     pointwise_layer_size     |                        170                        |\n",
      "|      last_layer_dropout      |                 0.738046416060631                 |\n",
      "|   output_conv_num_filters    |                        247                        |\n",
      "|   output_conv_kernel_size    |                         9                         |\n",
      "|      output_conv_stride      |                         2                         |\n",
      "|     output_conv_padding      |                         5                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        136                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 31%|████████████▋                            | 31/100 [10:38:48<20:19:15, 1060.22s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb43308db25d46f0a0d55eb81b4d4caa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exception while training: size mismatch, m1: [21983 x 3], m2: [247 x 4] at c:\\a\\w\\1\\s\\tmp_conda_3.6_105809\\conda\\conda-bld\\pytorch_1544094150554\\work\\aten\\src\\thc\\generic/THCTensorMathBlas.cu:266\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\33\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 44.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         44                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.003795886648209607               |\n",
      "|  noam_learning_rate_warmup   |                        5115                       |\n",
      "|  noam_learning_rate_factor   |                 2.6140224303685864                |\n",
      "|          adam_beta1          |                 0.7490932219770139                |\n",
      "|          adam_beta2          |                 0.7812964865139868                |\n",
      "|           adam_eps           |               1.5136718098328003e-09              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         8                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.07360510823955649                |\n",
      "|     pointwise_layer_size     |                         72                        |\n",
      "|      last_layer_dropout      |                0.35061999896836227                |\n",
      "|   output_conv_num_filters    |                        176                        |\n",
      "|   output_conv_kernel_size    |                         9                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         61                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 32%|█████████████▍                            | 32/100 [10:39:02<14:06:11, 746.65s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d36e51140074da4b4cb3cdf59028987",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.35\t\t0.34\t\t0.148\t\t0.809\t\t2.41m - 2.4m / 0.0m                                                                    \n",
      "2\t34k\t0.35\t\t0.32\t\t0.157\t\t0.788\t\t2.57m - 5.0m / 84.2m                                                                   \n",
      "3\t51k\t0.34\t\t0.36\t\t0.209\t\t0.869\t\t2.39m - 7.4m / 89.8m                                                                   \n",
      "4\t68k\t0.34\t\t0.34\t\t0.172\t\t0.818\t\t2.57m - 10.0m / 83.9m                                                                  \n",
      "5\t85k\t0.36\t\t0.34\t\t0.197\t\t0.874\t\t2.61m - 12.6m / 89.7m                                                                  \n",
      "6\t102k\t0.36\t\t0.35\t\t0.265\t\t0.916\t\t2.63m - 15.3m / 90.9m                                                                 \n",
      "7\t120k\t0.39\t\t0.35\t\t0.265\t\t0.916\t\t2.49m - 17.8m / 91.6m                                                                 \n",
      "8\t137k\t0.37\t\t0.36\t\t0.265\t\t0.916\t\t2.47m - 20.3m / 87.6m                                                                 \n",
      "9\t154k\t0.36\t\t0.38\t\t0.382\t\t0.959\t\t2.40m - 22.7m / 87.0m                                                                 \n",
      "10\t171k\t0.38\t\t0.36\t\t0.265\t\t0.916\t\t2.57m - 25.3m / 85.2m                                                                \n",
      "11\t188k\t0.36\t\t0.34\t\t0.382\t\t0.959\t\t2.57m - 27.9m / 89.6m                                                                \n",
      "12\t205k\t0.36\t\t0.32\t\t0.382\t\t0.959\t\t2.56m - 30.5m / 89.6m                                                                \n",
      "13\t222k\t0.36\t\t0.31\t\t0.382\t\t0.959\t\t2.56m - 33.1m / 89.5m                                                                \n",
      "14\t239k\t0.36\t\t0.32\t\t0.382\t\t0.959\t\t2.57m - 35.6m / 89.5m                                                                \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.3122934919261379                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3122934919261379\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\34\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 54.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         54                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.029478403695529433               |\n",
      "|  noam_learning_rate_warmup   |                        1479                       |\n",
      "|  noam_learning_rate_factor   |                 2.3374992578102622                |\n",
      "|          adam_beta1          |                 0.7531464162801571                |\n",
      "|          adam_beta2          |                 0.7327857241757614                |\n",
      "|           adam_eps           |               0.0031545368493449432               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.3875089029465299                |\n",
      "|     pointwise_layer_size     |                        150                        |\n",
      "|      last_layer_dropout      |                 0.7972922539718585                |\n",
      "|   output_conv_num_filters    |                         46                        |\n",
      "|   output_conv_kernel_size    |                         4                         |\n",
      "|      output_conv_stride      |                         9                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         42                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 33%|█████████████▌                           | 33/100 [11:16:06<22:08:30, 1189.71s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bec5d7c7261d43bf82fc47868854623b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.34\t\t0.25\t\t0.244\t\t0.885\t\t0.96m - 1.0m / 0.0m                                                                    \n",
      "2\t34k\t0.33\t\t0.32\t\t0.191\t\t0.841\t\t0.95m - 1.9m / 33.6m                                                                   \n",
      "3\t51k\t0.35\t\t0.28\t\t0.143\t\t0.754\t\t0.96m - 2.9m / 33.3m                                                                   \n",
      "4\t68k\t0.35\t\t0.27\t\t0.151\t\t0.776\t\t0.96m - 3.9m / 33.6m                                                                   \n",
      "5\t85k\t0.34\t\t0.28\t\t0.130\t\t0.723\t\t0.95m - 4.8m / 33.5m                                                                   \n",
      "6\t102k\t0.32\t\t0.27\t\t0.141\t\t0.746\t\t0.95m - 5.8m / 33.5m                                                                  \n",
      "VAL f1\t0.24414266527383974 - (0.24414266527383974)                                                                     \n",
      "VAL loss\t0.2501223031540364                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2501223031540364\n",
      "        | \\     )|_\tf1: 0.24414266527383974\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\35\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 67.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         67                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0006721393906862822               |\n",
      "|  noam_learning_rate_warmup   |                        8977                       |\n",
      "|  noam_learning_rate_factor   |                 1.466130337335041                 |\n",
      "|          adam_beta1          |                 0.9324237524426078                |\n",
      "|          adam_beta2          |                 0.7510057970905955                |\n",
      "|           adam_eps           |               2.1443542795335077e-10              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.4676300339055528                |\n",
      "|     pointwise_layer_size     |                         92                        |\n",
      "|      last_layer_dropout      |                 0.5817947117287748                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         65                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 34%|██████████████▎                           | 34/100 [11:22:49<17:28:54, 953.55s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "422b537b03154ec5a83d55807b20a24a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.32\t\t0.21\t\t0.234\t\t0.895\t\t0.97m - 1.0m / 0.0m                                                                    \n",
      "2\t34k\t0.23\t\t0.21\t\t0.206\t\t0.860\t\t0.98m - 2.0m / 34.1m                                                                   \n",
      "3\t51k\t0.21\t\t0.20\t\t0.205\t\t0.859\t\t0.98m - 3.0m / 34.2m                                                                   \n",
      "4\t68k\t0.18\t\t0.21\t\t0.250\t\t0.885\t\t0.98m - 4.0m / 34.2m                                                                   \n",
      "5\t85k\t0.14\t\t0.23\t\t0.191\t\t0.842\t\t0.97m - 4.9m / 34.3m                                                                   \n",
      "6\t103k\t0.11\t\t0.27\t\t0.292\t\t0.908\t\t0.98m - 5.9m / 34.1m                                                                  \n",
      "7\t120k\t0.10\t\t0.26\t\t0.281\t\t0.904\t\t1.01m - 7.0m / 34.3m                                                                  \n",
      "8\t137k\t0.09\t\t0.24\t\t0.286\t\t0.909\t\t0.98m - 8.0m / 35.2m                                                                  \n",
      "9\t154k\t0.08\t\t0.30\t\t0.298\t\t0.911\t\t0.98m - 8.9m / 34.5m                                                                  \n",
      "10\t171k\t0.07\t\t0.28\t\t0.364\t\t0.936\t\t0.97m - 9.9m / 34.3m                                                                 \n",
      "11\t188k\t0.07\t\t0.33\t\t0.352\t\t0.931\t\t0.97m - 10.9m / 34.3m                                                                \n",
      "12\t205k\t0.07\t\t0.35\t\t0.352\t\t0.933\t\t0.97m - 11.9m / 34.3m                                                                \n",
      "13\t222k\t0.07\t\t0.36\t\t0.412\t\t0.951\t\t0.98m - 12.9m / 34.3m                                                                \n",
      "14\t239k\t0.07\t\t0.34\t\t0.397\t\t0.948\t\t0.97m - 13.9m / 34.5m                                                                \n",
      "15\t256k\t0.07\t\t0.33\t\t0.368\t\t0.940\t\t1.01m - 14.9m / 34.4m                                                                \n",
      "16\t273k\t0.07\t\t0.39\t\t0.425\t\t0.953\t\t1.01m - 16.0m / 35.1m                                                                \n",
      "17\t290k\t0.07\t\t0.32\t\t0.393\t\t0.947\t\t1.00m - 17.0m / 35.2m                                                                \n",
      "18\t308k\t0.07\t\t0.35\t\t0.393\t\t0.944\t\t0.98m - 18.0m / 35.0m                                                                \n",
      "19\t325k\t0.07\t\t0.34\t\t0.362\t\t0.937\t\t0.98m - 19.0m / 34.6m                                                                \n",
      "20\t342k\t0.07\t\t0.32\t\t0.336\t\t0.933\t\t0.99m - 20.0m / 34.6m                                                                \n",
      "21\t359k\t0.07\t\t0.32\t\t0.319\t\t0.923\t\t0.99m - 21.0m / 34.8m                                                                \n",
      "VAL f1\t0.4245854172150566 - (0.4245854172150566)                                                                       \n",
      "VAL loss\t0.19796972729348197                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.19796972729348197\n",
      "        | \\     )|_\tf1: 0.4245854172150566\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\36\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 99.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         99                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.0977943793842483                |\n",
      "|  noam_learning_rate_warmup   |                        3474                       |\n",
      "|  noam_learning_rate_factor   |                 3.416830977159012                 |\n",
      "|          adam_beta1          |                 0.9507803636314536                |\n",
      "|          adam_beta2          |                 0.7678178208565866                |\n",
      "|           adam_eps           |               1.546716942918786e-05               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.41907649728403884                |\n",
      "|     pointwise_layer_size     |                        144                        |\n",
      "|      last_layer_dropout      |                 0.6839189845171026                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        123                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 35%|██████████████▎                          | 35/100 [11:44:42<19:09:55, 1061.47s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e0f5ec501d046118c3cdad11b6f5c92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.20\t\t0.17\t\t0.155\t\t0.809\t\t1.53m - 1.5m / 0.0m                                                                    \n",
      "2\t34k\t0.18\t\t0.16\t\t0.192\t\t0.854\t\t1.52m - 3.1m / 53.4m                                                                   \n",
      "3\t51k\t0.17\t\t0.16\t\t0.228\t\t0.886\t\t1.53m - 4.6m / 53.3m                                                                   \n",
      "4\t69k\t0.16\t\t0.20\t\t0.220\t\t0.883\t\t1.59m - 6.2m / 53.5m                                                                   \n",
      "5\t86k\t0.15\t\t0.18\t\t0.221\t\t0.882\t\t1.57m - 7.8m / 55.7m                                                                   \n",
      "6\t103k\t0.15\t\t0.19\t\t0.209\t\t0.881\t\t1.53m - 9.4m / 54.8m                                                                  \n",
      "7\t120k\t0.19\t\t0.21\t\t0.382\t\t0.959\t\t1.53m - 10.9m / 53.6m                                                                 \n",
      "8\t137k\t0.21\t\t0.20\t\t0.237\t\t0.910\t\t1.52m - 12.4m / 53.6m                                                                 \n",
      "9\t154k\t0.19\t\t0.18\t\t0.212\t\t0.874\t\t1.52m - 14.0m / 53.5m                                                                 \n",
      "10\t171k\t0.19\t\t0.25\t\t0.199\t\t0.869\t\t1.53m - 15.5m / 53.5m                                                                \n",
      "11\t188k\t0.19\t\t0.21\t\t0.265\t\t0.916\t\t1.54m - 17.1m / 53.7m                                                                \n",
      "12\t206k\t0.19\t\t0.19\t\t0.382\t\t0.959\t\t1.52m - 18.6m / 54.0m                                                                \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.1567839255064597                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.1567839255064597\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\37\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 39.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         39                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0027468216207986048               |\n",
      "|  noam_learning_rate_warmup   |                        1658                       |\n",
      "|  noam_learning_rate_factor   |                 1.807135430725043                 |\n",
      "|          adam_beta1          |                 0.8584958086957735                |\n",
      "|          adam_beta2          |                 0.7540214013557845                |\n",
      "|           adam_eps           |               6.402364748413604e-08               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         8                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.1604757945413315                |\n",
      "|     pointwise_layer_size     |                        102                        |\n",
      "|      last_layer_dropout      |                 0.4796725472468573                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         93                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 36%|██████████████▊                          | 36/100 [12:04:42<19:36:34, 1103.04s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5caf6a3b72d24b86a99d9d8e26e06928",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.40\t\t0.37\t\t0.227\t\t0.904\t\t2.61m - 2.6m / 0.0m                                                                    \n",
      "2\t34k\t0.42\t\t0.39\t\t0.197\t\t0.869\t\t2.61m - 5.2m / 91.3m                                                                   \n",
      "3\t51k\t0.41\t\t0.40\t\t0.212\t\t0.874\t\t2.59m - 7.8m / 91.3m                                                                   \n",
      "4\t68k\t0.40\t\t0.37\t\t0.382\t\t0.959\t\t2.59m - 10.5m / 90.8m                                                                  \n",
      "5\t85k\t0.38\t\t0.36\t\t0.382\t\t0.959\t\t2.60m - 13.1m / 90.8m                                                                  \n",
      "6\t102k\t0.37\t\t0.35\t\t0.382\t\t0.959\t\t2.60m - 15.7m / 91.0m                                                                 \n",
      "7\t119k\t0.36\t\t0.33\t\t0.265\t\t0.916\t\t2.60m - 18.3m / 91.0m                                                                 \n",
      "8\t136k\t0.36\t\t0.33\t\t0.382\t\t0.959\t\t2.60m - 20.9m / 91.0m                                                                 \n",
      "9\t153k\t0.36\t\t0.34\t\t0.382\t\t0.959\t\t2.60m - 23.5m / 91.1m                                                                 \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.3272534637100522                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3272534637100522\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\38\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 55.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         55                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00039860275910981756              |\n",
      "|  noam_learning_rate_warmup   |                        1455                       |\n",
      "|  noam_learning_rate_factor   |                 2.3746878174525725                |\n",
      "|          adam_beta1          |                 0.7324111382167502                |\n",
      "|          adam_beta2          |                 0.9604599965439324                |\n",
      "|           adam_eps           |                0.02921855343429855                |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.469951584823754                 |\n",
      "|     pointwise_layer_size     |                         89                        |\n",
      "|      last_layer_dropout      |                0.16119603981721378                |\n",
      "|   output_conv_num_filters    |                        172                        |\n",
      "|   output_conv_kernel_size    |                         8                         |\n",
      "|      output_conv_stride      |                         4                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        245                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 37%|███████████████▏                         | 37/100 [12:29:48<21:25:15, 1224.05s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00746d4e48c942358ac7cc5fb20df6c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.29\t\t0.38\t\t0.073\t\t0.583\t\t4.26m - 4.3m / 0.0m                                                                    \n",
      "2\t34k\t0.28\t\t0.39\t\t0.044\t\t0.544\t\t4.33m - 8.6m / 149.1m                                                                  \n",
      "3\t51k\t0.27\t\t0.38\t\t0.111\t\t0.703\t\t4.31m - 12.9m / 151.6m                                                                 \n",
      "4\t68k\t0.27\t\t0.36\t\t0.178\t\t0.836\t\t4.29m - 17.3m / 151.0m                                                                 \n",
      "5\t85k\t0.26\t\t0.39\t\t0.267\t\t0.911\t\t4.29m - 21.6m / 150.2m                                                                 \n",
      "6\t102k\t0.25\t\t0.34\t\t0.216\t\t0.866\t\t4.28m - 25.9m / 150.4m                                                                \n",
      "7\t119k\t0.25\t\t0.37\t\t0.213\t\t0.869\t\t4.28m - 30.2m / 150.0m                                                                \n",
      "8\t136k\t0.24\t\t0.31\t\t0.302\t\t0.925\t\t4.28m - 34.5m / 150.0m                                                                \n",
      "9\t153k\t0.23\t\t0.30\t\t0.231\t\t0.882\t\t4.29m - 38.8m / 150.0m                                                                \n",
      "10\t170k\t0.23\t\t0.31\t\t0.269\t\t0.907\t\t4.29m - 43.1m / 150.2m                                                               \n",
      "11\t188k\t0.23\t\t0.30\t\t0.260\t\t0.902\t\t4.29m - 47.4m / 150.4m                                                               \n",
      "12\t205k\t0.22\t\t0.31\t\t0.250\t\t0.898\t\t4.29m - 51.7m / 150.4m                                                               \n",
      "13\t222k\t0.22\t\t0.31\t\t0.212\t\t0.867\t\t4.29m - 56.0m / 150.4m                                                               \n",
      "VAL f1\t0.301502366742128 - (0.301502366742128)                                                                         \n",
      "VAL loss\t0.30340002963417456                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.30340002963417456\n",
      "        | \\     )|_\tf1: 0.301502366742128\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\39\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 23.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         23                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0004514742875377303               |\n",
      "|  noam_learning_rate_warmup   |                        4113                       |\n",
      "|  noam_learning_rate_factor   |                0.42378334796275585                |\n",
      "|          adam_beta1          |                 0.9234206354221376                |\n",
      "|          adam_beta2          |                 0.7871930791357916                |\n",
      "|           adam_eps           |               2.1406072430164104e-08              |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.6357660789626451                |\n",
      "|     pointwise_layer_size     |                         57                        |\n",
      "|      last_layer_dropout      |                0.08758433549284117                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        162                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 38%|███████████████▌                         | 38/100 [13:29:13<33:10:19, 1926.12s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f7814de19ca42908ac53e7bcf4fe6be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.61\t\t0.49\t\t0.199\t\t0.874\t\t2.80m - 2.8m / 0.0m                                                                    \n",
      "2\t34k\t0.43\t\t1.35\t\t0.044\t\t0.494\t\t2.87m - 5.7m / 98.1m                                                                   \n",
      "3\t51k\t0.39\t\t0.83\t\t0.106\t\t0.686\t\t2.88m - 8.6m / 100.6m                                                                  \n",
      "4\t68k\t0.39\t\t0.50\t\t0.199\t\t0.863\t\t2.78m - 11.4m / 100.9m                                                                 \n",
      "5\t85k\t0.40\t\t0.86\t\t0.091\t\t0.689\t\t2.77m - 14.2m / 97.8m                                                                  \n",
      "6\t102k\t0.41\t\t0.61\t\t0.205\t\t0.853\t\t2.78m - 17.0m / 97.5m                                                                 \n",
      "7\t119k\t0.41\t\t0.54\t\t0.212\t\t0.863\t\t2.78m - 19.8m / 97.5m                                                                 \n",
      "8\t136k\t0.41\t\t0.60\t\t0.178\t\t0.818\t\t2.77m - 22.6m / 97.8m                                                                 \n",
      "9\t153k\t0.41\t\t0.61\t\t0.202\t\t0.860\t\t2.77m - 25.4m / 97.5m                                                                 \n",
      "10\t170k\t0.41\t\t0.58\t\t0.226\t\t0.881\t\t2.81m - 28.2m / 97.6m                                                                \n",
      "11\t187k\t0.41\t\t0.59\t\t0.231\t\t0.882\t\t2.80m - 31.1m / 98.4m                                                                \n",
      "12\t205k\t0.41\t\t0.66\t\t0.221\t\t0.868\t\t2.79m - 33.9m / 98.3m                                                                \n",
      "13\t222k\t0.41\t\t0.67\t\t0.272\t\t0.902\t\t2.79m - 36.7m / 98.1m                                                                \n",
      "14\t239k\t0.40\t\t0.66\t\t0.231\t\t0.873\t\t2.84m - 39.6m / 98.1m                                                                \n",
      "15\t256k\t0.40\t\t0.66\t\t0.374\t\t0.949\t\t2.77m - 42.4m / 99.3m                                                                \n",
      "16\t273k\t0.40\t\t0.69\t\t0.356\t\t0.939\t\t2.77m - 45.2m / 97.8m                                                                \n",
      "17\t290k\t0.40\t\t0.71\t\t0.399\t\t0.954\t\t2.76m - 47.9m / 97.7m                                                                \n",
      "18\t307k\t0.40\t\t0.74\t\t0.198\t\t0.859\t\t2.77m - 50.7m / 97.7m                                                                \n",
      "19\t324k\t0.39\t\t0.74\t\t0.350\t\t0.942\t\t2.77m - 53.5m / 97.8m                                                                \n",
      "20\t341k\t0.40\t\t0.72\t\t0.296\t\t0.920\t\t2.77m - 56.3m / 97.9m                                                                \n",
      "21\t358k\t0.40\t\t0.83\t\t0.373\t\t0.950\t\t2.77m - 59.1m / 97.8m                                                                \n",
      "22\t375k\t0.39\t\t0.77\t\t0.339\t\t0.939\t\t2.77m - 61.9m / 97.9m                                                                \n",
      "VAL f1\t0.3991689750692521 - (0.3991689750692521)                                                                       \n",
      "VAL loss\t0.49480151082006635                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.49480151082006635\n",
      "        | \\     )|_\tf1: 0.3991689750692521\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\40\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 94.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         94                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.05930613310266348                |\n",
      "|  noam_learning_rate_warmup   |                        6685                       |\n",
      "|  noam_learning_rate_factor   |                 2.367153161378073                 |\n",
      "|          adam_beta1          |                 0.8058511464957197                |\n",
      "|          adam_beta2          |                 0.708688116002646                 |\n",
      "|           adam_eps           |               7.443377209036941e-08               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.2162982031739473                |\n",
      "|     pointwise_layer_size     |                        145                        |\n",
      "|      last_layer_dropout      |                0.34560033468653323                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        172                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 39%|███████████████▉                         | 39/100 [14:33:00<42:18:14, 2496.64s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bff6ded1f6c14f8db8779f84b6373f78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.23\t\t0.16\t\t0.309\t\t0.935\t\t1.21m - 1.2m / 0.0m                                                                    \n",
      "2\t34k\t0.16\t\t0.14\t\t0.196\t\t0.846\t\t1.28m - 2.5m / 42.4m                                                                   \n",
      "3\t51k\t0.13\t\t0.12\t\t0.258\t\t0.887\t\t1.22m - 3.8m / 44.8m                                                                   \n",
      "4\t68k\t0.12\t\t0.11\t\t0.266\t\t0.893\t\t1.23m - 5.0m / 42.7m                                                                   \n",
      "5\t86k\t0.10\t\t0.12\t\t0.240\t\t0.877\t\t1.21m - 6.2m / 43.2m                                                                   \n",
      "6\t103k\t0.09\t\t0.12\t\t0.235\t\t0.872\t\t1.21m - 7.4m / 42.4m                                                                  \n",
      "VAL f1\t0.30908248378127895 - (0.30908248378127895)                                                                     \n",
      "VAL loss\t0.11227432948247147                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.11227432948247147\n",
      "        | \\     )|_\tf1: 0.30908248378127895\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\41\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 59.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         59                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.002798516578131797               |\n",
      "|  noam_learning_rate_warmup   |                        8481                       |\n",
      "|  noam_learning_rate_factor   |                 3.2578970437128643                |\n",
      "|          adam_beta1          |                 0.8025190101896233                |\n",
      "|          adam_beta2          |                 0.9964632575403911                |\n",
      "|           adam_eps           |               2.976225968029686e-07               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.49264981299103994                |\n",
      "|     pointwise_layer_size     |                        217                        |\n",
      "|      last_layer_dropout      |                0.14374397823965596                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         93                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 40%|████████████████▍                        | 40/100 [14:41:42<31:43:59, 1903.98s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03894edc9cf14ac9a0c436367c3e0b13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.30\t\t0.23\t\t0.218\t\t0.877\t\t1.68m - 1.7m / 0.0m                                                                    \n",
      "2\t34k\t0.22\t\t0.28\t\t0.110\t\t0.756\t\t1.69m - 3.4m / 58.9m                                                                   \n",
      "3\t51k\t0.19\t\t0.22\t\t0.155\t\t0.802\t\t1.68m - 5.1m / 59.0m                                                                   \n",
      "4\t68k\t0.18\t\t0.23\t\t0.147\t\t0.814\t\t1.68m - 6.8m / 58.9m                                                                   \n",
      "5\t85k\t0.18\t\t0.25\t\t0.140\t\t0.753\t\t1.68m - 8.5m / 58.9m                                                                   \n",
      "6\t102k\t0.17\t\t0.21\t\t0.179\t\t0.841\t\t1.70m - 10.2m / 58.9m                                                                 \n",
      "VAL f1\t0.2181765640803352 - (0.2181765640803352)                                                                       \n",
      "VAL loss\t0.2105425222733985                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2105425222733985\n",
      "        | \\     )|_\tf1: 0.2181765640803352\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\42\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 93.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         93                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.020269323510304995               |\n",
      "|  noam_learning_rate_warmup   |                        6312                       |\n",
      "|  noam_learning_rate_factor   |                 2.952653731520719                 |\n",
      "|          adam_beta1          |                 0.9087277564615046                |\n",
      "|          adam_beta2          |                 0.9717212662730793                |\n",
      "|           adam_eps           |               2.742337799246707e-10               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.7724890814039457                |\n",
      "|     pointwise_layer_size     |                         80                        |\n",
      "|      last_layer_dropout      |                 0.1635883614162279                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        109                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 41%|████████████████▊                        | 41/100 [14:53:10<25:13:40, 1539.34s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4c654be2c874b16ab599827cd82ceba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.23\t\t0.18\t\t0.205\t\t0.875\t\t0.84m - 0.8m / 0.0m                                                                    \n",
      "2\t34k\t0.18\t\t0.18\t\t0.208\t\t0.878\t\t0.84m - 1.7m / 29.5m                                                                   \n",
      "3\t51k\t0.17\t\t0.20\t\t0.117\t\t0.715\t\t0.84m - 2.6m / 29.5m                                                                   \n",
      "4\t68k\t0.17\t\t0.22\t\t0.094\t\t0.612\t\t0.85m - 3.4m / 29.6m                                                                   \n",
      "5\t86k\t0.17\t\t0.23\t\t0.105\t\t0.662\t\t0.85m - 4.3m / 29.7m                                                                   \n",
      "6\t103k\t0.16\t\t0.43\t\t0.046\t\t0.371\t\t0.85m - 5.2m / 29.8m                                                                  \n",
      "7\t120k\t0.16\t\t0.30\t\t0.078\t\t0.576\t\t0.85m - 6.0m / 29.8m                                                                  \n",
      "VAL f1\t0.20841356836024294 - (0.20841356836024294)                                                                     \n",
      "VAL loss\t0.17914677089817785                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.17914677089817785\n",
      "        | \\     )|_\tf1: 0.20841356836024294\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\43\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 83.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         83                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.021521417444878427               |\n",
      "|  noam_learning_rate_warmup   |                        5209                       |\n",
      "|  noam_learning_rate_factor   |                 0.5824938841752136                |\n",
      "|          adam_beta1          |                 0.9301325750718276                |\n",
      "|          adam_beta2          |                 0.8597551534915019                |\n",
      "|           adam_eps           |               0.0007972576099804452               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.2584168315737023                |\n",
      "|     pointwise_layer_size     |                        106                        |\n",
      "|      last_layer_dropout      |                0.03647720970863579                |\n",
      "|   output_conv_num_filters    |                         59                        |\n",
      "|   output_conv_kernel_size    |                         3                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         14                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 42%|█████████████████▏                       | 42/100 [15:00:05<19:21:56, 1202.00s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a6628b71ce54fd5b19e3337eb754670",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.25\t\t0.18\t\t0.381\t\t0.959\t\t0.73m - 0.7m / 0.0m                                                                    \n",
      "2\t34k\t0.19\t\t0.18\t\t0.263\t\t0.909\t\t0.75m - 1.5m / 25.7m                                                                   \n",
      "3\t51k\t0.18\t\t0.17\t\t0.193\t\t0.850\t\t0.75m - 2.3m / 26.2m                                                                   \n",
      "4\t68k\t0.18\t\t0.18\t\t0.205\t\t0.859\t\t0.75m - 3.0m / 26.3m                                                                   \n",
      "5\t85k\t0.17\t\t0.17\t\t0.197\t\t0.851\t\t0.75m - 3.8m / 26.2m                                                                   \n",
      "6\t103k\t0.16\t\t0.16\t\t0.242\t\t0.889\t\t0.75m - 4.5m / 26.2m                                                                  \n",
      "VAL f1\t0.3813291139240506 - (0.3813291139240506)                                                                       \n",
      "VAL loss\t0.16315554837146437                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.16315554837146437\n",
      "        | \\     )|_\tf1: 0.3813291139240506\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190424\\44\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 18.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         18                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.012077903991343716               |\n",
      "|  noam_learning_rate_warmup   |                        1714                       |\n",
      "|  noam_learning_rate_factor   |                  3.23572557452745                 |\n",
      "|          adam_beta1          |                 0.7449554575917096                |\n",
      "|          adam_beta2          |                 0.8494852803858257                |\n",
      "|           adam_eps           |               1.0167541136408124e-10              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.46831597154032684                |\n",
      "|     pointwise_layer_size     |                        238                        |\n",
      "|      last_layer_dropout      |                 0.6186975635153299                |\n",
      "|   output_conv_num_filters    |                        205                        |\n",
      "|   output_conv_kernel_size    |                         4                         |\n",
      "|      output_conv_stride      |                         3                         |\n",
      "|     output_conv_padding      |                         4                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        142                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 43%|██████████████████                        | 43/100 [15:05:21<14:49:30, 936.32s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06d86a1f75bc4f43b3410a7ff9da5676",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.78\t\t0.92\t\t0.258\t\t0.915\t\t6.37m - 6.4m / 0.0m                                                                    \n",
      "2\t34k\t0.81\t\t0.64\t\t0.382\t\t0.959\t\t6.37m - 12.8m / 223.0m                                                                 \n",
      "3\t51k\t0.75\t\t0.67\t\t0.382\t\t0.959\t\t6.32m - 19.1m / 223.0m                                                                 \n",
      "4\t68k\t0.74\t\t0.59\t\t0.382\t\t0.959\t\t6.33m - 25.5m / 221.4m                                                                 \n",
      "5\t85k\t0.73\t\t0.63\t\t0.382\t\t0.959\t\t6.34m - 31.8m / 221.7m                                                                 \n",
      "6\t102k\t0.72\t\t0.61\t\t0.382\t\t0.959\t\t6.32m - 38.2m / 222.0m                                                                \n",
      "7\t119k\t0.72\t\t0.57\t\t0.382\t\t0.959\t\t6.33m - 44.5m / 221.5m                                                                \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.5724387819539267                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.5724387819539267\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\0 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 48.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         48                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.06971267644896768                |\n",
      "|  noam_learning_rate_warmup   |                        8877                       |\n",
      "|  noam_learning_rate_factor   |                 2.816928411697331                 |\n",
      "|          adam_beta1          |                 0.717701753434817                 |\n",
      "|          adam_beta2          |                 0.8204669974643235                |\n",
      "|           adam_eps           |                1.49313236773663e-08               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.2818155888271549                |\n",
      "|     pointwise_layer_size     |                        177                        |\n",
      "|      last_layer_dropout      |                0.03110372247088602                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        236                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 44%|██████████████████                       | 44/100 [15:52:58<23:31:37, 1512.45s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb94c319eafa47898cb4ff5f7fc3ba93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.35\t\t0.27\t\t0.227\t\t0.893\t\t2.75m - 2.7m / 0.0m                                                                    \n",
      "2\t34k\t0.28\t\t0.26\t\t0.218\t\t0.869\t\t2.75m - 5.5m / 96.2m                                                                   \n",
      "3\t51k\t0.26\t\t0.26\t\t0.231\t\t0.879\t\t2.75m - 8.3m / 96.3m                                                                   \n",
      "4\t68k\t0.25\t\t0.23\t\t0.257\t\t0.898\t\t2.74m - 11.1m / 96.3m                                                                  \n",
      "5\t85k\t0.24\t\t0.26\t\t0.160\t\t0.812\t\t2.75m - 13.8m / 96.1m                                                                  \n",
      "6\t103k\t0.24\t\t0.28\t\t0.165\t\t0.811\t\t2.82m - 16.7m / 96.3m                                                                 \n",
      "7\t120k\t0.23\t\t0.28\t\t0.159\t\t0.807\t\t2.75m - 19.4m / 98.5m                                                                 \n",
      "8\t137k\t0.23\t\t0.24\t\t0.210\t\t0.866\t\t2.75m - 22.2m / 96.4m                                                                 \n",
      "9\t154k\t0.23\t\t0.27\t\t0.231\t\t0.875\t\t2.76m - 25.0m / 96.6m                                                                 \n",
      "VAL f1\t0.25668096337842294 - (0.25668096337842294)                                                                     \n",
      "VAL loss\t0.23355906323869102                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.23355906323869102\n",
      "        | \\     )|_\tf1: 0.25668096337842294\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\1 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 77.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         77                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0010175893703528909               |\n",
      "|  noam_learning_rate_warmup   |                        1221                       |\n",
      "|  noam_learning_rate_factor   |                0.17627498107110642                |\n",
      "|          adam_beta1          |                 0.8308762600062335                |\n",
      "|          adam_beta2          |                 0.9830961332218007                |\n",
      "|           adam_eps           |                0.04152269659274712                |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.7378476844643983                |\n",
      "|     pointwise_layer_size     |                         79                        |\n",
      "|      last_layer_dropout      |                0.28631367029675386                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         20                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 45%|██████████████████▍                      | 45/100 [16:20:02<23:37:11, 1546.02s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "357a281c4fed4956b93def78e0716983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.32\t\t0.23\t\t0.177\t\t0.858\t\t1.03m - 1.0m / 0.0m                                                                    \n",
      "2\t34k\t0.22\t\t0.23\t\t0.376\t\t0.958\t\t1.06m - 2.1m / 36.2m                                                                   \n",
      "3\t51k\t0.21\t\t0.23\t\t0.199\t\t0.885\t\t1.10m - 3.2m / 37.2m                                                                   \n",
      "4\t68k\t0.21\t\t0.22\t\t0.333\t\t0.946\t\t1.03m - 4.3m / 38.4m                                                                   \n",
      "5\t85k\t0.21\t\t0.23\t\t0.373\t\t0.956\t\t1.03m - 5.3m / 36.1m                                                                   \n",
      "6\t103k\t0.21\t\t0.21\t\t0.307\t\t0.939\t\t1.02m - 6.4m / 36.1m                                                                  \n",
      "7\t120k\t0.21\t\t0.22\t\t0.382\t\t0.959\t\t1.03m - 7.4m / 35.9m                                                                  \n",
      "8\t137k\t0.21\t\t0.22\t\t0.365\t\t0.955\t\t1.03m - 8.4m / 36.1m                                                                  \n",
      "9\t154k\t0.21\t\t0.22\t\t0.382\t\t0.959\t\t1.03m - 9.5m / 36.3m                                                                  \n",
      "10\t171k\t0.21\t\t0.22\t\t0.329\t\t0.944\t\t1.03m - 10.5m / 36.2m                                                                \n",
      "11\t188k\t0.21\t\t0.21\t\t0.362\t\t0.954\t\t1.03m - 11.6m / 36.3m                                                                \n",
      "12\t205k\t0.21\t\t0.21\t\t0.382\t\t0.959\t\t1.03m - 12.6m / 36.2m                                                                \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.21191268313556064                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.21191268313556064\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\2 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 21.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         21                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 3.4030044693697747                |\n",
      "|  noam_learning_rate_warmup   |                        5720                       |\n",
      "|  noam_learning_rate_factor   |                 2.258344051202165                 |\n",
      "|          adam_beta1          |                 0.7125651876007227                |\n",
      "|          adam_beta2          |                 0.858317781124109                 |\n",
      "|           adam_eps           |               0.0006291676411176279               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.4161565541301053                |\n",
      "|     pointwise_layer_size     |                         54                        |\n",
      "|      last_layer_dropout      |                 0.6091687693250838                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        132                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 46%|██████████████████▊                      | 46/100 [16:33:31<19:52:14, 1324.71s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "333cc60ca0d7460eba75f9e479926629",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.62\t\t0.48\t\t0.301\t\t0.928\t\t4.24m - 4.2m / 0.0m                                                                    \n",
      "2\t34k\t0.54\t\t0.51\t\t0.304\t\t0.925\t\t4.27m - 8.6m / 148.6m                                                                  \n",
      "3\t51k\t0.53\t\t0.57\t\t0.262\t\t0.906\t\t4.30m - 12.9m / 149.7m                                                                 \n",
      "4\t68k\t0.53\t\t0.54\t\t0.304\t\t0.927\t\t4.23m - 17.2m / 150.5m                                                                 \n",
      "5\t85k\t0.52\t\t0.57\t\t0.263\t\t0.908\t\t4.25m - 21.4m / 148.4m                                                                 \n",
      "6\t102k\t0.51\t\t0.57\t\t0.316\t\t0.935\t\t4.28m - 25.7m / 148.9m                                                                \n",
      "7\t119k\t0.52\t\t0.58\t\t0.256\t\t0.910\t\t4.25m - 30.0m / 150.0m                                                                \n",
      "8\t136k\t0.54\t\t0.60\t\t0.265\t\t0.917\t\t4.32m - 34.3m / 148.9m                                                                \n",
      "9\t153k\t0.55\t\t0.72\t\t0.265\t\t0.917\t\t4.32m - 38.7m / 151.1m                                                                \n",
      "10\t171k\t0.56\t\t0.59\t\t0.382\t\t0.959\t\t4.27m - 43.0m / 151.1m                                                               \n",
      "11\t188k\t0.55\t\t0.60\t\t0.258\t\t0.915\t\t4.26m - 47.3m / 149.8m                                                               \n",
      "12\t205k\t0.55\t\t0.58\t\t0.265\t\t0.917\t\t4.34m - 51.6m / 149.5m                                                               \n",
      "13\t222k\t0.55\t\t0.57\t\t0.238\t\t0.911\t\t4.27m - 55.9m / 151.5m                                                               \n",
      "14\t239k\t0.55\t\t0.62\t\t0.127\t\t0.771\t\t4.26m - 60.2m / 149.8m                                                               \n",
      "15\t256k\t0.55\t\t0.57\t\t0.238\t\t0.911\t\t4.36m - 64.5m / 149.5m                                                               \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.48222444367941664                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.48222444367941664\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\3 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 74.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         74                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.0728823391287149                |\n",
      "|  noam_learning_rate_warmup   |                        1555                       |\n",
      "|  noam_learning_rate_factor   |                 0.4550687786310824                |\n",
      "|          adam_beta1          |                 0.8686955047665208                |\n",
      "|          adam_beta2          |                 0.872491721392666                 |\n",
      "|           adam_eps           |                0.004078516585672708               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.763367107881498                 |\n",
      "|     pointwise_layer_size     |                        179                        |\n",
      "|      last_layer_dropout      |                0.26167991705808336                |\n",
      "|   output_conv_num_filters    |                         68                        |\n",
      "|   output_conv_kernel_size    |                         7                         |\n",
      "|      output_conv_stride      |                         4                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        150                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 47%|███████████████████▎                     | 47/100 [17:40:28<31:23:42, 2132.51s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1dee276dd38c4b21bba386c89d065a34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.23\t\t0.30\t\t0.064\t\t0.419\t\t2.81m - 2.8m / 0.0m                                                                    \n",
      "2\t34k\t0.21\t\t0.55\t\t0.043\t\t0.092\t\t2.81m - 5.7m / 98.4m                                                                   \n",
      "3\t51k\t0.22\t\t0.53\t\t0.047\t\t0.142\t\t2.81m - 8.5m / 98.4m                                                                   \n",
      "4\t68k\t0.22\t\t0.47\t\t0.047\t\t0.190\t\t2.80m - 11.3m / 98.3m                                                                  \n",
      "5\t85k\t0.23\t\t0.67\t\t0.040\t\t0.088\t\t2.80m - 14.1m / 98.1m                                                                  \n",
      "6\t103k\t0.24\t\t0.74\t\t0.007\t\t0.057\t\t2.80m - 16.9m / 98.0m                                                                 \n",
      "VAL f1\t0.06381586544012932 - (0.06381586544012932)                                                                     \n",
      "VAL loss\t0.2973193164957996                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2973193164957996\n",
      "        | \\     )|_\tf1: 0.06381586544012932\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\4 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 74.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         74                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.04006125916282892                |\n",
      "|  noam_learning_rate_warmup   |                        2259                       |\n",
      "|  noam_learning_rate_factor   |                 0.3395726916365182                |\n",
      "|          adam_beta1          |                 0.7670787489835764                |\n",
      "|          adam_beta2          |                 0.9105653931319283                |\n",
      "|           adam_eps           |               4.395248324489915e-06               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.10512936111559563                |\n",
      "|     pointwise_layer_size     |                         69                        |\n",
      "|      last_layer_dropout      |                0.34061346826817646                |\n",
      "|   output_conv_num_filters    |                        161                        |\n",
      "|   output_conv_kernel_size    |                         6                         |\n",
      "|      output_conv_stride      |                         10                        |\n",
      "|     output_conv_padding      |                         4                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         47                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 48%|███████████████████▋                     | 48/100 [17:59:56<26:37:25, 1843.19s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e8a2de406d44a37a61625b284be2e56",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.22\t\t0.20\t\t0.206\t\t0.863\t\t1.26m - 1.3m / 0.0m                                                                    \n",
      "2\t34k\t0.19\t\t0.21\t\t0.160\t\t0.807\t\t1.31m - 2.6m / 44.0m                                                                   \n",
      "3\t51k\t0.18\t\t0.19\t\t0.190\t\t0.845\t\t1.31m - 3.9m / 45.8m                                                                   \n",
      "4\t68k\t0.18\t\t0.18\t\t0.229\t\t0.889\t\t1.31m - 5.2m / 45.8m                                                                   \n",
      "5\t85k\t0.17\t\t0.18\t\t0.225\t\t0.869\t\t1.30m - 6.6m / 45.8m                                                                   \n",
      "6\t103k\t0.16\t\t0.16\t\t0.256\t\t0.895\t\t1.34m - 7.9m / 45.6m                                                                  \n",
      "7\t120k\t0.15\t\t0.17\t\t0.197\t\t0.858\t\t1.30m - 9.3m / 46.9m                                                                  \n",
      "8\t137k\t0.15\t\t0.18\t\t0.240\t\t0.882\t\t1.31m - 10.6m / 45.8m                                                                 \n",
      "9\t154k\t0.14\t\t0.17\t\t0.227\t\t0.873\t\t1.31m - 11.9m / 45.9m                                                                 \n",
      "10\t171k\t0.14\t\t0.17\t\t0.259\t\t0.891\t\t1.29m - 13.2m / 46.1m                                                                \n",
      "11\t188k\t0.13\t\t0.17\t\t0.247\t\t0.886\t\t1.30m - 14.5m / 45.6m                                                                \n",
      "12\t205k\t0.12\t\t0.17\t\t0.218\t\t0.865\t\t1.30m - 15.8m / 45.7m                                                                \n",
      "13\t222k\t0.12\t\t0.15\t\t0.278\t\t0.903\t\t1.30m - 17.1m / 45.7m                                                                \n",
      "14\t239k\t0.10\t\t0.15\t\t0.271\t\t0.898\t\t1.29m - 18.5m / 45.7m                                                                \n",
      "15\t256k\t0.10\t\t0.16\t\t0.248\t\t0.885\t\t1.30m - 19.8m / 45.7m                                                                \n",
      "16\t274k\t0.09\t\t0.15\t\t0.287\t\t0.905\t\t1.30m - 21.1m / 45.9m                                                                \n",
      "17\t291k\t0.08\t\t0.16\t\t0.329\t\t0.926\t\t1.30m - 22.4m / 45.9m                                                                \n",
      "18\t308k\t0.08\t\t0.16\t\t0.295\t\t0.914\t\t1.30m - 23.8m / 45.9m                                                                \n",
      "19\t325k\t0.07\t\t0.17\t\t0.283\t\t0.907\t\t1.30m - 25.1m / 45.9m                                                                \n",
      "20\t342k\t0.07\t\t0.16\t\t0.303\t\t0.917\t\t1.30m - 26.4m / 45.9m                                                                \n",
      "21\t359k\t0.06\t\t0.17\t\t0.255\t\t0.898\t\t1.30m - 27.7m / 45.9m                                                                \n",
      "22\t376k\t0.06\t\t0.18\t\t0.300\t\t0.917\t\t1.30m - 29.0m / 45.9m                                                                \n",
      "VAL f1\t0.3287120755475186 - (0.3287120755475186)                                                                       \n",
      "VAL loss\t0.14788667345599318                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.14788667345599318\n",
      "        | \\     )|_\tf1: 0.3287120755475186\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\5 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 16.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         16                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.03431621743152557                |\n",
      "|  noam_learning_rate_warmup   |                        4696                       |\n",
      "|  noam_learning_rate_factor   |                 3.6017642151654674                |\n",
      "|          adam_beta1          |                 0.7932700966690148                |\n",
      "|          adam_beta2          |                 0.9092528491336872                |\n",
      "|           adam_eps           |                0.007991877421766404               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.3270945232535306                |\n",
      "|     pointwise_layer_size     |                        152                        |\n",
      "|      last_layer_dropout      |                0.21544505841260958                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         31                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 49%|████████████████████                     | 49/100 [18:30:00<25:56:40, 1831.38s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b394d97c508463ea7364a690a190289",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.68\t\t0.49\t\t0.315\t\t0.928\t\t2.47m - 2.5m / 0.0m                                                                    \n",
      "2\t34k\t0.56\t\t0.53\t\t0.316\t\t0.928\t\t2.49m - 5.0m / 86.6m                                                                   \n",
      "3\t51k\t0.54\t\t0.58\t\t0.399\t\t0.953\t\t2.48m - 7.5m / 87.1m                                                                   \n",
      "4\t68k\t0.51\t\t0.53\t\t0.376\t\t0.946\t\t2.53m - 10.1m / 86.8m                                                                  \n",
      "5\t85k\t0.49\t\t0.48\t\t0.341\t\t0.932\t\t2.49m - 12.6m / 88.4m                                                                  \n",
      "6\t102k\t0.46\t\t0.52\t\t0.363\t\t0.939\t\t2.39m - 15.0m / 87.2m                                                                 \n",
      "7\t119k\t0.44\t\t0.51\t\t0.282\t\t0.908\t\t2.51m - 17.5m / 84.3m                                                                 \n",
      "8\t136k\t0.42\t\t0.46\t\t0.364\t\t0.938\t\t2.51m - 20.0m / 87.7m                                                                 \n",
      "VAL f1\t0.3989203778677463 - (0.3989203778677463)                                                                       \n",
      "VAL loss\t0.4629940493393314                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.4629940493393314\n",
      "        | \\     )|_\tf1: 0.3989203778677463\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\6 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 95.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         95                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.005397172042208394               |\n",
      "|  noam_learning_rate_warmup   |                        4203                       |\n",
      "|  noam_learning_rate_factor   |                 0.8507843349127986                |\n",
      "|          adam_beta1          |                 0.9357068368592447                |\n",
      "|          adam_beta2          |                 0.8466856645196972                |\n",
      "|           adam_eps           |                0.08493283696270904                |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.320236116611329                 |\n",
      "|     pointwise_layer_size     |                        172                        |\n",
      "|      last_layer_dropout      |                 0.4228379919601364                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        234                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 50%|████████████████████▌                    | 50/100 [18:52:00<23:18:12, 1677.86s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c29bcf4b462b4e97a3798df08b0d166a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.23\t\t0.16\t\t0.200\t\t0.873\t\t2.34m - 2.3m / 0.0m                                                                    \n",
      "2\t34k\t0.17\t\t0.16\t\t0.221\t\t0.884\t\t2.36m - 4.7m / 82.1m                                                                   \n",
      "3\t51k\t0.17\t\t0.16\t\t0.226\t\t0.893\t\t2.36m - 7.1m / 82.5m                                                                   \n",
      "4\t68k\t0.17\t\t0.16\t\t0.241\t\t0.903\t\t2.36m - 9.5m / 82.6m                                                                   \n",
      "5\t86k\t0.17\t\t0.17\t\t0.149\t\t0.796\t\t2.36m - 11.9m / 82.8m                                                                  \n",
      "6\t103k\t0.16\t\t0.16\t\t0.151\t\t0.785\t\t2.36m - 14.3m / 82.7m                                                                 \n",
      "7\t120k\t0.15\t\t0.17\t\t0.117\t\t0.726\t\t2.39m - 16.7m / 82.7m                                                                 \n",
      "8\t137k\t0.15\t\t0.17\t\t0.125\t\t0.732\t\t2.36m - 19.1m / 83.6m                                                                 \n",
      "9\t154k\t0.14\t\t0.16\t\t0.147\t\t0.783\t\t2.37m - 21.4m / 82.9m                                                                 \n",
      "VAL f1\t0.24123197436353924 - (0.24123197436353924)                                                                     \n",
      "VAL loss\t0.15538786062213222                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.15538786062213222\n",
      "        | \\     )|_\tf1: 0.24123197436353924\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\7 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 89.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         89                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.007408696230938569               |\n",
      "|  noam_learning_rate_warmup   |                        4509                       |\n",
      "|  noam_learning_rate_factor   |                 1.452087692115903                 |\n",
      "|          adam_beta1          |                 0.8728839574432408                |\n",
      "|          adam_beta2          |                 0.718441423581353                 |\n",
      "|           adam_eps           |               0.0018465925961318804               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.18960884121114835                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                0.22765241566724265                |\n",
      "|   output_conv_num_filters    |                        391                        |\n",
      "|   output_conv_kernel_size    |                         9                         |\n",
      "|      output_conv_stride      |                         9                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         67                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 51%|████████████████████▉                    | 51/100 [19:15:31<21:45:04, 1598.05s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5538c5a09534e0a8965a14145e2734a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.19\t\t0.17\t\t0.179\t\t0.830\t\t1.83m - 1.8m / 0.0m                                                                    \n",
      "2\t34k\t0.17\t\t0.16\t\t0.197\t\t0.846\t\t1.86m - 3.7m / 64.1m                                                                   \n",
      "3\t51k\t0.16\t\t0.15\t\t0.223\t\t0.873\t\t1.82m - 5.6m / 65.1m                                                                   \n",
      "4\t68k\t0.15\t\t0.16\t\t0.232\t\t0.884\t\t1.86m - 7.5m / 63.8m                                                                   \n",
      "5\t85k\t0.14\t\t0.16\t\t0.212\t\t0.862\t\t1.82m - 9.3m / 65.0m                                                                   \n",
      "6\t103k\t0.12\t\t0.16\t\t0.205\t\t0.854\t\t1.82m - 11.1m / 63.8m                                                                 \n",
      "7\t120k\t0.09\t\t0.16\t\t0.253\t\t0.890\t\t1.82m - 13.0m / 63.8m                                                                 \n",
      "8\t137k\t0.08\t\t0.17\t\t0.233\t\t0.882\t\t1.85m - 14.9m / 63.8m                                                                 \n",
      "9\t154k\t0.06\t\t0.20\t\t0.268\t\t0.908\t\t1.85m - 16.7m / 64.9m                                                                 \n",
      "10\t171k\t0.05\t\t0.22\t\t0.297\t\t0.920\t\t1.82m - 18.6m / 64.9m                                                                \n",
      "11\t188k\t0.04\t\t0.23\t\t0.241\t\t0.898\t\t1.85m - 20.5m / 64.0m                                                                \n",
      "12\t205k\t0.03\t\t0.27\t\t0.315\t\t0.931\t\t1.85m - 22.3m / 65.0m                                                                \n",
      "13\t222k\t0.03\t\t0.34\t\t0.379\t\t0.947\t\t1.82m - 24.2m / 65.0m                                                                \n",
      "14\t239k\t0.03\t\t0.32\t\t0.347\t\t0.941\t\t1.85m - 26.1m / 64.2m                                                                \n",
      "15\t256k\t0.03\t\t0.32\t\t0.316\t\t0.930\t\t1.84m - 28.0m / 65.1m                                                                \n",
      "16\t273k\t0.03\t\t0.33\t\t0.344\t\t0.943\t\t1.85m - 29.9m / 64.9m                                                                \n",
      "17\t290k\t0.03\t\t0.25\t\t0.326\t\t0.936\t\t1.85m - 31.7m / 65.1m                                                                \n",
      "18\t308k\t0.03\t\t0.35\t\t0.314\t\t0.933\t\t1.85m - 33.6m / 65.1m                                                                \n",
      "VAL f1\t0.3787027579162411 - (0.3787027579162411)                                                                       \n",
      "VAL loss\t0.15341028426023906                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.15341028426023906\n",
      "        | \\     )|_\tf1: 0.3787027579162411\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\8 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 70.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         70                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.10302399707540373                |\n",
      "|  noam_learning_rate_warmup   |                        4049                       |\n",
      "|  noam_learning_rate_factor   |                 1.8835084077303943                |\n",
      "|          adam_beta1          |                 0.8175593309769653                |\n",
      "|          adam_beta2          |                 0.9677538622818262                |\n",
      "|           adam_eps           |               5.970222379612095e-08               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.36751058854003427                |\n",
      "|     pointwise_layer_size     |                        216                        |\n",
      "|      last_layer_dropout      |                0.09215688208055087                |\n",
      "|   output_conv_num_filters    |                        304                        |\n",
      "|   output_conv_kernel_size    |                         8                         |\n",
      "|      output_conv_stride      |                         6                         |\n",
      "|     output_conv_padding      |                         4                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        113                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 52%|█████████████████████▎                   | 52/100 [19:50:21<23:16:25, 1745.53s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af66d21791994bd3b65d527abbfe3992",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.22\t\t0.27\t\t0.058\t\t0.575\t\t2.26m - 2.3m / 0.0m                                                                    \n",
      "2\t34k\t0.19\t\t0.19\t\t0.154\t\t0.773\t\t2.29m - 4.6m / 79.2m                                                                   \n",
      "3\t51k\t0.17\t\t0.19\t\t0.134\t\t0.758\t\t2.26m - 6.9m / 80.2m                                                                   \n",
      "4\t68k\t0.16\t\t0.20\t\t0.134\t\t0.778\t\t2.26m - 9.2m / 79.2m                                                                   \n",
      "5\t85k\t0.16\t\t0.18\t\t0.162\t\t0.796\t\t2.27m - 11.4m / 79.4m                                                                  \n",
      "6\t102k\t0.15\t\t0.19\t\t0.155\t\t0.787\t\t2.29m - 13.8m / 79.5m                                                                 \n",
      "7\t120k\t0.15\t\t0.19\t\t0.175\t\t0.823\t\t2.29m - 16.1m / 80.3m                                                                 \n",
      "8\t137k\t0.15\t\t0.20\t\t0.156\t\t0.786\t\t2.28m - 18.4m / 80.3m                                                                 \n",
      "9\t154k\t0.14\t\t0.18\t\t0.212\t\t0.850\t\t2.25m - 20.7m / 80.1m                                                                 \n",
      "10\t171k\t0.14\t\t0.18\t\t0.182\t\t0.821\t\t2.29m - 23.0m / 79.3m                                                                \n",
      "11\t188k\t0.14\t\t0.21\t\t0.164\t\t0.797\t\t2.29m - 25.3m / 80.2m                                                                \n",
      "12\t205k\t0.14\t\t0.16\t\t0.178\t\t0.840\t\t2.28m - 27.6m / 80.2m                                                                \n",
      "13\t222k\t0.13\t\t0.17\t\t0.202\t\t0.847\t\t2.28m - 29.9m / 80.0m                                                                \n",
      "14\t239k\t0.13\t\t0.19\t\t0.295\t\t0.912\t\t2.29m - 32.2m / 80.1m                                                                \n",
      "15\t256k\t0.13\t\t0.21\t\t0.155\t\t0.785\t\t2.25m - 34.5m / 80.4m                                                                \n",
      "16\t273k\t0.13\t\t0.19\t\t0.231\t\t0.882\t\t2.26m - 36.7m / 79.5m                                                                \n",
      "17\t290k\t0.13\t\t0.23\t\t0.192\t\t0.845\t\t2.25m - 39.0m / 79.6m                                                                \n",
      "18\t307k\t0.13\t\t0.27\t\t0.091\t\t0.682\t\t2.26m - 41.3m / 79.5m                                                                \n",
      "19\t325k\t0.13\t\t0.23\t\t0.221\t\t0.864\t\t2.26m - 43.5m / 79.6m                                                                \n",
      "VAL f1\t0.294841628959276 - (0.294841628959276)                                                                         \n",
      "VAL loss\t0.1648006448291597                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.1648006448291597\n",
      "        | \\     )|_\tf1: 0.294841628959276\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\9 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 78.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         78                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0013913492460103862               |\n",
      "|  noam_learning_rate_warmup   |                        1097                       |\n",
      "|  noam_learning_rate_factor   |                 1.5136053243335614                |\n",
      "|          adam_beta1          |                 0.9099366974847387                |\n",
      "|          adam_beta2          |                 0.8731409787192719                |\n",
      "|           adam_eps           |                 0.569662710788784                 |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.45193638453465035                |\n",
      "|     pointwise_layer_size     |                        210                        |\n",
      "|      last_layer_dropout      |                0.01343656470996333                |\n",
      "|   output_conv_num_filters    |                        294                        |\n",
      "|   output_conv_kernel_size    |                         1                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         63                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 53%|█████████████████████▋                   | 53/100 [20:35:47<26:37:37, 2039.51s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66f3f634072649bbb11a3009cad6980e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.25\t\t0.18\t\t0.369\t\t0.955\t\t0.80m - 0.8m / 0.0m                                                                    \n",
      "2\t34k\t0.20\t\t0.18\t\t0.306\t\t0.932\t\t0.83m - 1.7m / 28.0m                                                                   \n",
      "3\t51k\t0.19\t\t0.16\t\t0.264\t\t0.903\t\t0.82m - 2.5m / 28.9m                                                                   \n",
      "4\t68k\t0.17\t\t0.15\t\t0.230\t\t0.871\t\t0.80m - 3.3m / 28.9m                                                                   \n",
      "5\t85k\t0.16\t\t0.15\t\t0.200\t\t0.846\t\t0.80m - 4.1m / 28.0m                                                                   \n",
      "6\t102k\t0.14\t\t0.17\t\t0.144\t\t0.774\t\t0.80m - 4.9m / 28.1m                                                                  \n",
      "VAL f1\t0.3688573120189461 - (0.3688573120189461)                                                                       \n",
      "VAL loss\t0.14649001213899704                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.14649001213899704\n",
      "        | \\     )|_\tf1: 0.3688573120189461\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\10\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 70.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         70                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.1153981399570392                |\n",
      "|  noam_learning_rate_warmup   |                        2851                       |\n",
      "|  noam_learning_rate_factor   |                 1.3379509983366782                |\n",
      "|          adam_beta1          |                 0.9185927401667703                |\n",
      "|          adam_beta2          |                 0.9275545236707106                |\n",
      "|           adam_eps           |               3.697854319169925e-06               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.4472975833375462                |\n",
      "|     pointwise_layer_size     |                        213                        |\n",
      "|      last_layer_dropout      |                0.45917466000704893                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        163                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 54%|██████████████████████▏                  | 54/100 [20:41:30<19:33:26, 1530.59s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f43db29377b4f94a89444ea6e5061a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.26\t\t0.21\t\t0.193\t\t0.872\t\t2.43m - 2.4m / 0.0m                                                                    \n",
      "2\t34k\t0.22\t\t0.20\t\t0.185\t\t0.852\t\t2.44m - 4.9m / 84.9m                                                                   \n",
      "3\t51k\t0.21\t\t0.19\t\t0.226\t\t0.886\t\t2.43m - 7.3m / 85.4m                                                                   \n",
      "4\t68k\t0.21\t\t0.20\t\t0.181\t\t0.836\t\t2.42m - 9.8m / 85.1m                                                                   \n",
      "5\t85k\t0.21\t\t0.21\t\t0.151\t\t0.781\t\t2.43m - 12.2m / 85.0m                                                                  \n",
      "6\t102k\t0.20\t\t0.20\t\t0.172\t\t0.828\t\t2.42m - 14.7m / 85.0m                                                                 \n",
      "7\t120k\t0.20\t\t0.21\t\t0.156\t\t0.796\t\t2.43m - 17.1m / 84.9m                                                                 \n",
      "8\t137k\t0.19\t\t0.20\t\t0.188\t\t0.845\t\t2.44m - 19.6m / 85.3m                                                                 \n",
      "VAL f1\t0.22618496217384593 - (0.22618496217384593)                                                                     \n",
      "VAL loss\t0.19150814442407518                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.19150814442407518\n",
      "        | \\     )|_\tf1: 0.22618496217384593\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\11\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 50.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         50                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.005819919945787489               |\n",
      "|  noam_learning_rate_warmup   |                        5652                       |\n",
      "|  noam_learning_rate_factor   |                 1.6578427702247394                |\n",
      "|          adam_beta1          |                 0.8982225453369741                |\n",
      "|          adam_beta2          |                 0.7901635647573111                |\n",
      "|           adam_eps           |                0.11024937011560748                |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.39857698768555255                |\n",
      "|     pointwise_layer_size     |                         81                        |\n",
      "|      last_layer_dropout      |                0.26022895433280413                |\n",
      "|   output_conv_num_filters    |                         41                        |\n",
      "|   output_conv_kernel_size    |                         6                         |\n",
      "|      output_conv_stride      |                         3                         |\n",
      "|     output_conv_padding      |                         1                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        200                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 55%|██████████████████████▌                  | 55/100 [21:03:00<18:13:54, 1458.55s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d13233c34de24c008375f22c9d723a4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.33\t\t0.28\t\t0.200\t\t0.865\t\t1.83m - 1.8m / 0.0m                                                                    \n",
      "2\t34k\t0.26\t\t0.31\t\t0.135\t\t0.750\t\t1.87m - 3.7m / 64.2m                                                                   \n",
      "3\t51k\t0.22\t\t0.27\t\t0.134\t\t0.771\t\t1.86m - 5.6m / 65.3m                                                                   \n",
      "4\t68k\t0.20\t\t0.33\t\t0.090\t\t0.671\t\t1.84m - 7.5m / 65.1m                                                                   \n",
      "5\t85k\t0.20\t\t0.24\t\t0.171\t\t0.814\t\t1.84m - 9.3m / 64.5m                                                                   \n",
      "6\t102k\t0.19\t\t0.22\t\t0.175\t\t0.835\t\t1.84m - 11.2m / 64.5m                                                                 \n",
      "VAL f1\t0.20024519820187986 - (0.20024519820187986)                                                                     \n",
      "VAL loss\t0.2223320584180878                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2223320584180878\n",
      "        | \\     )|_\tf1: 0.20024519820187986\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\12\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 68.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         68                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.003685952407807889               |\n",
      "|  noam_learning_rate_warmup   |                        7446                       |\n",
      "|  noam_learning_rate_factor   |                 0.387844695737579                 |\n",
      "|          adam_beta1          |                 0.8936920485052363                |\n",
      "|          adam_beta2          |                 0.9775280006885776                |\n",
      "|           adam_eps           |               1.335093603391161e-07               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.13782714649159225                |\n",
      "|     pointwise_layer_size     |                        228                        |\n",
      "|      last_layer_dropout      |                0.30351680977021067                |\n",
      "|   output_conv_num_filters    |                        324                        |\n",
      "|   output_conv_kernel_size    |                         9                         |\n",
      "|      output_conv_stride      |                         7                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        122                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 56%|██████████████████████▉                  | 56/100 [21:15:45<15:17:04, 1250.56s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31936be4d6be4d84b2b946da76f1fa8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.25\t\t0.22\t\t0.193\t\t0.853\t\t2.39m - 2.4m / 0.0m                                                                    \n",
      "2\t34k\t0.21\t\t0.21\t\t0.165\t\t0.821\t\t2.42m - 4.8m / 83.8m                                                                   \n",
      "3\t51k\t0.20\t\t0.21\t\t0.154\t\t0.808\t\t2.42m - 7.3m / 84.8m                                                                   \n",
      "4\t68k\t0.19\t\t0.19\t\t0.217\t\t0.869\t\t2.42m - 9.7m / 84.6m                                                                   \n",
      "5\t85k\t0.17\t\t0.19\t\t0.249\t\t0.894\t\t2.38m - 12.1m / 84.6m                                                                  \n",
      "6\t102k\t0.17\t\t0.21\t\t0.155\t\t0.797\t\t2.42m - 14.6m / 83.6m                                                                 \n",
      "7\t119k\t0.15\t\t0.19\t\t0.216\t\t0.865\t\t2.41m - 17.0m / 84.6m                                                                 \n",
      "8\t137k\t0.14\t\t0.20\t\t0.197\t\t0.858\t\t2.41m - 19.4m / 84.5m                                                                 \n",
      "9\t154k\t0.13\t\t0.19\t\t0.220\t\t0.868\t\t2.41m - 21.8m / 84.5m                                                                 \n",
      "10\t171k\t0.12\t\t0.19\t\t0.231\t\t0.882\t\t2.42m - 24.3m / 84.6m                                                                \n",
      "VAL f1\t0.24887604367373153 - (0.24887604367373153)                                                                     \n",
      "VAL loss\t0.1881419300354409                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.1881419300354409\n",
      "        | \\     )|_\tf1: 0.24887604367373153\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\13\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09202453177167498                |\n",
      "|  noam_learning_rate_warmup   |                        1678                       |\n",
      "|  noam_learning_rate_factor   |                 2.7127811312863224                |\n",
      "|          adam_beta1          |                 0.7504762165299802                |\n",
      "|          adam_beta2          |                 0.8640987609347887                |\n",
      "|           adam_eps           |               0.0012711000772727965               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.025030000809302157               |\n",
      "|     pointwise_layer_size     |                        202                        |\n",
      "|      last_layer_dropout      |                 0.1722635936998813                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        194                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 57%|███████████████████████▎                 | 57/100 [21:41:40<16:01:34, 1341.73s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8567bfeec29a4f6f928197168b242667",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.53\t\t0.46\t\t0.222\t\t0.878\t\t2.45m - 2.4m / 0.0m                                                                    \n",
      "2\t34k\t0.50\t\t0.44\t\t0.212\t\t0.876\t\t2.45m - 4.9m / 85.7m                                                                   \n",
      "3\t51k\t0.46\t\t0.43\t\t0.225\t\t0.879\t\t2.45m - 7.4m / 85.8m                                                                   \n",
      "4\t68k\t0.42\t\t0.37\t\t0.305\t\t0.923\t\t2.43m - 9.8m / 85.9m                                                                   \n",
      "5\t85k\t0.39\t\t0.37\t\t0.260\t\t0.903\t\t2.44m - 12.3m / 85.3m                                                                  \n",
      "6\t102k\t0.36\t\t0.36\t\t0.285\t\t0.921\t\t2.44m - 14.8m / 85.4m                                                                 \n",
      "7\t119k\t0.35\t\t0.35\t\t0.255\t\t0.904\t\t2.45m - 17.2m / 85.5m                                                                 \n",
      "8\t136k\t0.33\t\t0.37\t\t0.255\t\t0.901\t\t2.46m - 19.7m / 85.7m                                                                 \n",
      "9\t154k\t0.31\t\t0.40\t\t0.172\t\t0.849\t\t2.45m - 22.2m / 86.0m                                                                 \n",
      "VAL f1\t0.3051804071759726 - (0.3051804071759726)                                                                       \n",
      "VAL loss\t0.34518182358932126                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.34518182358932126\n",
      "        | \\     )|_\tf1: 0.3051804071759726\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\14\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 96.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         96                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.000855386871729812               |\n",
      "|  noam_learning_rate_warmup   |                        1659                       |\n",
      "|  noam_learning_rate_factor   |                 2.8428634973263582                |\n",
      "|          adam_beta1          |                 0.8284343634080049                |\n",
      "|          adam_beta2          |                 0.7553831226263168                |\n",
      "|           adam_eps           |                0.10387834619704314                |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.4722053109670925                |\n",
      "|     pointwise_layer_size     |                        239                        |\n",
      "|      last_layer_dropout      |                0.11708623013319289                |\n",
      "|   output_conv_num_filters    |                         54                        |\n",
      "|   output_conv_kernel_size    |                         9                         |\n",
      "|      output_conv_stride      |                         3                         |\n",
      "|     output_conv_padding      |                         1                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        165                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 58%|███████████████████████▊                 | 58/100 [22:05:24<15:56:32, 1366.49s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87f4d0499d1e41018b38c38c63e76d78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.18\t\t0.18\t\t0.117\t\t0.731\t\t2.01m - 2.0m / 0.0m                                                                    \n",
      "2\t34k\t0.16\t\t0.16\t\t0.110\t\t0.722\t\t2.01m - 4.1m / 70.5m                                                                   \n",
      "3\t51k\t0.15\t\t0.13\t\t0.205\t\t0.851\t\t2.00m - 6.1m / 70.4m                                                                   \n",
      "4\t68k\t0.14\t\t0.18\t\t0.093\t\t0.649\t\t2.00m - 8.1m / 70.0m                                                                   \n",
      "5\t85k\t0.14\t\t0.16\t\t0.134\t\t0.764\t\t2.00m - 10.1m / 70.1m                                                                  \n",
      "6\t103k\t0.13\t\t0.16\t\t0.190\t\t0.835\t\t1.99m - 12.1m / 70.0m                                                                 \n",
      "7\t120k\t0.13\t\t0.15\t\t0.150\t\t0.791\t\t2.00m - 14.1m / 69.9m                                                                 \n",
      "8\t137k\t0.13\t\t0.14\t\t0.184\t\t0.827\t\t2.00m - 16.1m / 70.1m                                                                 \n",
      "VAL f1\t0.20530777736822706 - (0.20530777736822706)                                                                     \n",
      "VAL loss\t0.12897202101620764                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.12897202101620764\n",
      "        | \\     )|_\tf1: 0.20530777736822706\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\15\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 29.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         29                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.2073224637166024                |\n",
      "|  noam_learning_rate_warmup   |                        5610                       |\n",
      "|  noam_learning_rate_factor   |                 1.4430644520804263                |\n",
      "|          adam_beta1          |                 0.8264925747662553                |\n",
      "|          adam_beta2          |                 0.9467738381625924                |\n",
      "|           adam_eps           |               0.00024153093961976603              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.7052857659304999                |\n",
      "|     pointwise_layer_size     |                        139                        |\n",
      "|      last_layer_dropout      |                0.43931927159370787                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        234                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 59%|████████████████████████▏                | 59/100 [22:23:34<14:37:06, 1283.56s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1096631ed2754d84aa16db70bfe112e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.51\t\t0.48\t\t0.374\t\t0.957\t\t2.96m - 3.0m / 0.0m                                                                    \n",
      "2\t34k\t0.42\t\t0.74\t\t0.087\t\t0.673\t\t2.97m - 6.0m / 103.6m                                                                  \n",
      "3\t51k\t0.40\t\t0.64\t\t0.124\t\t0.737\t\t2.97m - 8.9m / 104.1m                                                                  \n",
      "4\t68k\t0.40\t\t0.57\t\t0.150\t\t0.829\t\t2.97m - 11.9m / 104.0m                                                                 \n",
      "5\t85k\t0.38\t\t0.53\t\t0.178\t\t0.848\t\t2.97m - 14.9m / 104.0m                                                                 \n",
      "6\t102k\t0.37\t\t0.53\t\t0.196\t\t0.860\t\t2.98m - 17.9m / 104.1m                                                                \n",
      "VAL f1\t0.3740831295843521 - (0.3740831295843521)                                                                       \n",
      "VAL loss\t0.4761229908078431                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.4761229908078431\n",
      "        | \\     )|_\tf1: 0.3740831295843521\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\16\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 77.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         77                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.15785409460842156                |\n",
      "|  noam_learning_rate_warmup   |                        1564                       |\n",
      "|  noam_learning_rate_factor   |                0.21932270373190493                |\n",
      "|          adam_beta1          |                 0.8658252061663918                |\n",
      "|          adam_beta2          |                 0.9349054511534876                |\n",
      "|           adam_eps           |               0.0003880291917620494               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.5535965313533324                |\n",
      "|     pointwise_layer_size     |                         34                        |\n",
      "|      last_layer_dropout      |                0.08909471556978997                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        186                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 60%|████████████████████████▌                | 60/100 [22:43:24<13:56:54, 1255.37s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5de73f8c28ea43c5bc4dd0bce22a3165",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.26\t\t0.19\t\t0.245\t\t0.905\t\t1.36m - 1.4m / 0.0m                                                                    \n",
      "2\t34k\t0.20\t\t0.20\t\t0.244\t\t0.902\t\t1.35m - 2.7m / 47.5m                                                                   \n",
      "3\t51k\t0.20\t\t0.20\t\t0.192\t\t0.850\t\t1.35m - 4.1m / 47.3m                                                                   \n",
      "4\t68k\t0.19\t\t0.20\t\t0.182\t\t0.844\t\t1.36m - 5.5m / 47.4m                                                                   \n",
      "5\t85k\t0.18\t\t0.21\t\t0.124\t\t0.753\t\t1.35m - 6.8m / 47.6m                                                                   \n",
      "6\t103k\t0.17\t\t0.21\t\t0.140\t\t0.763\t\t1.35m - 8.2m / 47.4m                                                                  \n",
      "VAL f1\t0.24545944973925554 - (0.24545944973925554)                                                                     \n",
      "VAL loss\t0.1948348704067889                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.1948348704067889\n",
      "        | \\     )|_\tf1: 0.24545944973925554\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\17\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 82.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         82                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.4632556411962862                |\n",
      "|  noam_learning_rate_warmup   |                        6984                       |\n",
      "|  noam_learning_rate_factor   |                 2.315273389350847                 |\n",
      "|          adam_beta1          |                 0.7566830588877065                |\n",
      "|          adam_beta2          |                 0.8024205857595648                |\n",
      "|           adam_eps           |               3.410785199175669e-05               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.29114890012875655                |\n",
      "|     pointwise_layer_size     |                        100                        |\n",
      "|      last_layer_dropout      |                 0.6349605390290353                |\n",
      "|   output_conv_num_filters    |                        100                        |\n",
      "|   output_conv_kernel_size    |                         3                         |\n",
      "|      output_conv_stride      |                         10                        |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        208                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 61%|█████████████████████████                | 61/100 [22:52:50<11:21:38, 1048.68s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de39ff9f6116469da86230d2e2a7ffed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.22\t\t0.22\t\t0.230\t\t0.893\t\t1.32m - 1.3m / 0.0m                                                                    \n",
      "2\t34k\t0.19\t\t0.23\t\t0.125\t\t0.738\t\t1.33m - 2.7m / 46.4m                                                                   \n",
      "3\t51k\t0.18\t\t0.21\t\t0.146\t\t0.798\t\t1.33m - 4.0m / 46.5m                                                                   \n",
      "4\t68k\t0.17\t\t0.20\t\t0.209\t\t0.865\t\t1.33m - 5.4m / 46.5m                                                                   \n",
      "5\t85k\t0.16\t\t0.20\t\t0.181\t\t0.835\t\t1.33m - 6.7m / 46.7m                                                                   \n",
      "6\t102k\t0.16\t\t0.18\t\t0.211\t\t0.861\t\t1.33m - 8.0m / 46.6m                                                                  \n",
      "VAL f1\t0.2295675053445157 - (0.2295675053445157)                                                                       \n",
      "VAL loss\t0.18424444291649794                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.18424444291649794\n",
      "        | \\     )|_\tf1: 0.2295675053445157\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\18\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 86.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         86                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.059369898884224746               |\n",
      "|  noam_learning_rate_warmup   |                        3741                       |\n",
      "|  noam_learning_rate_factor   |                 3.098506713067729                 |\n",
      "|          adam_beta1          |                 0.7379857320696578                |\n",
      "|          adam_beta2          |                 0.8793111507474797                |\n",
      "|           adam_eps           |               3.233778946481354e-08               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.3280218997346322                |\n",
      "|     pointwise_layer_size     |                        136                        |\n",
      "|      last_layer_dropout      |                 0.707398999797392                 |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         70                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 62%|██████████████████████████▋                | 62/100 [23:02:05<9:30:22, 900.58s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec7e9090b5a5412699d76a02b4a1b228",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.24\t\t0.17\t\t0.244\t\t0.900\t\t0.46m - 0.5m / 0.0m                                                                    \n",
      "2\t34k\t0.18\t\t0.17\t\t0.200\t\t0.854\t\t0.47m - 1.0m / 16.2m                                                                   \n",
      "3\t51k\t0.17\t\t0.17\t\t0.250\t\t0.890\t\t0.46m - 1.4m / 16.4m                                                                   \n",
      "4\t68k\t0.15\t\t0.16\t\t0.229\t\t0.871\t\t0.47m - 1.9m / 16.3m                                                                   \n",
      "5\t86k\t0.13\t\t0.17\t\t0.173\t\t0.820\t\t0.47m - 2.4m / 16.6m                                                                   \n",
      "6\t103k\t0.12\t\t0.17\t\t0.233\t\t0.879\t\t0.47m - 2.9m / 16.6m                                                                  \n",
      "7\t120k\t0.11\t\t0.19\t\t0.283\t\t0.904\t\t0.47m - 3.4m / 16.6m                                                                  \n",
      "8\t137k\t0.10\t\t0.20\t\t0.235\t\t0.881\t\t0.47m - 3.9m / 16.7m                                                                  \n",
      "9\t154k\t0.09\t\t0.22\t\t0.252\t\t0.886\t\t0.47m - 4.4m / 16.5m                                                                  \n",
      "10\t171k\t0.09\t\t0.22\t\t0.344\t\t0.931\t\t0.47m - 4.9m / 16.5m                                                                 \n",
      "11\t188k\t0.08\t\t0.19\t\t0.273\t\t0.896\t\t0.47m - 5.4m / 16.6m                                                                 \n",
      "12\t205k\t0.08\t\t0.23\t\t0.269\t\t0.897\t\t0.47m - 5.8m / 16.7m                                                                 \n",
      "13\t222k\t0.07\t\t0.24\t\t0.331\t\t0.925\t\t0.47m - 6.3m / 16.7m                                                                 \n",
      "14\t240k\t0.07\t\t0.27\t\t0.339\t\t0.928\t\t0.47m - 6.8m / 16.7m                                                                 \n",
      "15\t257k\t0.07\t\t0.30\t\t0.373\t\t0.938\t\t0.47m - 7.3m / 16.6m                                                                 \n",
      "16\t274k\t0.06\t\t0.28\t\t0.379\t\t0.941\t\t0.46m - 7.8m / 16.7m                                                                 \n",
      "17\t291k\t0.06\t\t0.34\t\t0.391\t\t0.944\t\t0.47m - 8.3m / 16.6m                                                                 \n",
      "18\t308k\t0.06\t\t0.32\t\t0.424\t\t0.951\t\t0.46m - 8.8m / 16.8m                                                                 \n",
      "19\t325k\t0.06\t\t0.40\t\t0.429\t\t0.953\t\t0.47m - 9.3m / 16.7m                                                                 \n",
      "20\t342k\t0.06\t\t0.45\t\t0.440\t\t0.956\t\t0.47m - 9.8m / 16.9m                                                                 \n",
      "21\t359k\t0.05\t\t0.43\t\t0.452\t\t0.958\t\t0.47m - 10.3m / 16.8m                                                                \n",
      "22\t377k\t0.05\t\t0.38\t\t0.400\t\t0.946\t\t0.47m - 10.8m / 16.9m                                                                \n",
      "23\t394k\t0.05\t\t0.46\t\t0.422\t\t0.952\t\t0.46m - 11.3m / 16.9m                                                                \n",
      "24\t411k\t0.05\t\t0.39\t\t0.428\t\t0.953\t\t0.47m - 11.7m / 16.8m                                                                \n",
      "25\t428k\t0.05\t\t0.42\t\t0.366\t\t0.938\t\t0.51m - 12.3m / 16.9m                                                                \n",
      "26\t445k\t0.04\t\t0.41\t\t0.444\t\t0.956\t\t0.49m - 12.8m / 17.4m                                                                \n",
      "VAL f1\t0.4523281596452328 - (0.4523281596452328)                                                                       \n",
      "VAL loss\t0.16055710902509762                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.16055710902509762\n",
      "        | \\     )|_\tf1: 0.4523281596452328\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\19\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 41.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         41                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.1160655954234278                |\n",
      "|  noam_learning_rate_warmup   |                        4378                       |\n",
      "|  noam_learning_rate_factor   |                 0.7787637098744171                |\n",
      "|          adam_beta1          |                 0.7892561386212651                |\n",
      "|          adam_beta2          |                 0.8485367089071736                |\n",
      "|           adam_eps           |               5.4553408738256224e-05              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.3241299267928794                |\n",
      "|     pointwise_layer_size     |                        133                        |\n",
      "|      last_layer_dropout      |                0.27072365265615245                |\n",
      "|   output_conv_num_filters    |                        294                        |\n",
      "|   output_conv_kernel_size    |                         2                         |\n",
      "|      output_conv_stride      |                         4                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         25                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------+---------------------------------------------------+\n",
      " 63%|███████████████████████████                | 63/100 [23:15:38<8:59:06, 874.23s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5738e15c3f84414883d0614323408166",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.37\t\t0.34\t\t0.202\t\t0.862\t\t1.87m - 1.9m / 0.0m                                                                    \n",
      "2\t34k\t0.32\t\t0.34\t\t0.211\t\t0.870\t\t1.89m - 3.8m / 65.4m                                                                   \n",
      "3\t51k\t0.32\t\t0.34\t\t0.139\t\t0.804\t\t1.83m - 5.6m / 66.1m                                                                   \n",
      "4\t68k\t0.32\t\t0.33\t\t0.190\t\t0.861\t\t1.82m - 7.5m / 64.1m                                                                   \n",
      "5\t85k\t0.32\t\t0.37\t\t0.244\t\t0.892\t\t1.83m - 9.3m / 63.9m                                                                   \n",
      "6\t102k\t0.32\t\t0.34\t\t0.216\t\t0.866\t\t1.87m - 11.2m / 64.1m                                                                 \n",
      "7\t119k\t0.31\t\t0.39\t\t0.198\t\t0.846\t\t1.87m - 13.1m / 65.4m                                                                 \n",
      "8\t136k\t0.33\t\t0.40\t\t0.223\t\t0.874\t\t1.88m - 15.0m / 65.5m                                                                 \n",
      "9\t154k\t0.32\t\t0.38\t\t0.189\t\t0.840\t\t1.87m - 16.9m / 65.7m                                                                 \n",
      "10\t171k\t0.33\t\t0.38\t\t0.235\t\t0.884\t\t1.88m - 18.8m / 65.6m                                                                \n",
      "VAL f1\t0.24359178474765164 - (0.24359178474765164)                                                                     \n",
      "VAL loss\t0.334795971381955                                                                                             \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.334795971381955\n",
      "        | \\     )|_\tf1: 0.24359178474765164\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\20\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 62.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         62                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0015018449298539033               |\n",
      "|  noam_learning_rate_warmup   |                        6669                       |\n",
      "|  noam_learning_rate_factor   |                 3.4566614189625295                |\n",
      "|          adam_beta1          |                 0.9704847694810385                |\n",
      "|          adam_beta2          |                 0.702627264485757                 |\n",
      "|           adam_eps           |                0.003287257605385457               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.12547857527211415                |\n",
      "|     pointwise_layer_size     |                        201                        |\n",
      "|      last_layer_dropout      |                 0.2836649938264188                |\n",
      "|   output_conv_num_filters    |                        223                        |\n",
      "|   output_conv_kernel_size    |                         6                         |\n",
      "|      output_conv_stride      |                         7                         |\n",
      "|     output_conv_padding      |                         1                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         79                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 64%|███████████████████████████▌               | 64/100 [23:35:40<9:43:33, 972.61s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60c433a1400f45dcb11071268ddea003",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.29\t\t0.27\t\t0.263\t\t0.916\t\t1.98m - 2.0m / 0.0m                                                                    \n",
      "2\t34k\t0.27\t\t0.23\t\t0.290\t\t0.927\t\t2.01m - 4.0m / 69.3m                                                                   \n",
      "3\t51k\t0.24\t\t0.24\t\t0.210\t\t0.867\t\t1.96m - 6.0m / 70.3m                                                                   \n",
      "4\t68k\t0.24\t\t0.22\t\t0.228\t\t0.893\t\t1.95m - 8.0m / 68.7m                                                                   \n",
      "5\t85k\t0.24\t\t0.22\t\t0.368\t\t0.956\t\t1.94m - 9.9m / 68.6m                                                                   \n",
      "6\t102k\t0.24\t\t0.25\t\t0.377\t\t0.958\t\t1.96m - 11.9m / 68.3m                                                                 \n",
      "7\t119k\t0.25\t\t0.25\t\t0.265\t\t0.916\t\t1.94m - 13.9m / 68.9m                                                                 \n",
      "8\t136k\t0.25\t\t0.26\t\t0.265\t\t0.916\t\t1.96m - 15.9m / 68.3m                                                                 \n",
      "9\t153k\t0.25\t\t0.25\t\t0.382\t\t0.959\t\t1.95m - 17.8m / 68.7m                                                                 \n",
      "10\t170k\t0.25\t\t0.27\t\t0.265\t\t0.916\t\t1.97m - 19.8m / 68.5m                                                                \n",
      "11\t188k\t0.25\t\t0.26\t\t0.265\t\t0.916\t\t1.97m - 21.8m / 69.1m                                                                \n",
      "12\t205k\t0.25\t\t0.27\t\t0.265\t\t0.916\t\t1.96m - 23.8m / 69.0m                                                                \n",
      "13\t222k\t0.24\t\t0.26\t\t0.382\t\t0.959\t\t1.97m - 25.8m / 69.0m                                                                \n",
      "14\t239k\t0.24\t\t0.25\t\t0.382\t\t0.959\t\t1.97m - 27.8m / 69.0m                                                                \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.22157316031446944                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.22157316031446944\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\21\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 66.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         66                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.004777705088339669               |\n",
      "|  noam_learning_rate_warmup   |                        2673                       |\n",
      "|  noam_learning_rate_factor   |                 2.1795315838044695                |\n",
      "|          adam_beta1          |                 0.8520167049584189                |\n",
      "|          adam_beta2          |                 0.8700224900412079                |\n",
      "|           adam_eps           |               2.810535672492704e-10               |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.7741262095644037                |\n",
      "|     pointwise_layer_size     |                         74                        |\n",
      "|      last_layer_dropout      |                0.43527199038623954                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        192                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 65%|██████████████████████████▋              | 65/100 [24:04:47<11:42:48, 1204.83s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "664b224b5cf2443782cdd56536a5a07c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.27\t\t0.28\t\t0.238\t\t0.899\t\t0.80m - 0.8m / 0.0m                                                                    \n",
      "2\t34k\t0.23\t\t0.30\t\t0.162\t\t0.812\t\t0.80m - 1.6m / 27.9m                                                                   \n",
      "3\t51k\t0.22\t\t0.38\t\t0.106\t\t0.675\t\t0.80m - 2.4m / 27.9m                                                                   \n",
      "4\t68k\t0.21\t\t0.38\t\t0.084\t\t0.703\t\t0.80m - 3.2m / 28.1m                                                                   \n",
      "5\t85k\t0.20\t\t0.36\t\t0.118\t\t0.744\t\t0.80m - 4.1m / 28.1m                                                                   \n",
      "6\t103k\t0.20\t\t0.35\t\t0.180\t\t0.826\t\t0.80m - 4.9m / 28.0m                                                                  \n",
      "VAL f1\t0.23825098984334653 - (0.23825098984334653)                                                                     \n",
      "VAL loss\t0.2782446909131426                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2782446909131426\n",
      "        | \\     )|_\tf1: 0.23825098984334653\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\22\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 32.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         32                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.0708791081851387                |\n",
      "|  noam_learning_rate_warmup   |                        6269                       |\n",
      "|  noam_learning_rate_factor   |                 1.7615338656947697                |\n",
      "|          adam_beta1          |                 0.7008114521460427                |\n",
      "|          adam_beta2          |                 0.7397994705411328                |\n",
      "|           adam_eps           |               3.227472143404188e-05               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.3755704113864824                |\n",
      "|     pointwise_layer_size     |                        208                        |\n",
      "|      last_layer_dropout      |                 0.5378012867339844                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        240                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 66%|████████████████████████████▍              | 66/100 [24:10:31<8:56:25, 946.62s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d80cc0ddcc794961b77adf7a23acda28",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.48\t\t0.42\t\t0.142\t\t0.794\t\t3.30m - 3.3m / 0.0m                                                                    \n",
      "2\t34k\t0.39\t\t0.39\t\t0.159\t\t0.814\t\t3.32m - 6.7m / 115.6m                                                                  \n",
      "3\t51k\t0.38\t\t0.37\t\t0.230\t\t0.884\t\t3.30m - 10.0m / 116.3m                                                                 \n",
      "4\t68k\t0.38\t\t0.37\t\t0.195\t\t0.856\t\t3.33m - 13.3m / 115.7m                                                                 \n",
      "5\t85k\t0.38\t\t0.37\t\t0.243\t\t0.898\t\t3.31m - 16.7m / 116.5m                                                                 \n",
      "6\t102k\t0.39\t\t0.42\t\t0.188\t\t0.846\t\t3.32m - 20.0m / 115.9m                                                                \n",
      "7\t119k\t0.42\t\t0.42\t\t0.176\t\t0.829\t\t3.32m - 23.4m / 116.3m                                                                \n",
      "8\t136k\t0.42\t\t0.46\t\t0.265\t\t0.916\t\t3.31m - 26.7m / 116.4m                                                                \n",
      "9\t154k\t0.43\t\t0.45\t\t0.244\t\t0.912\t\t3.32m - 30.0m / 116.1m                                                                \n",
      "10\t171k\t0.43\t\t0.53\t\t0.382\t\t0.959\t\t3.31m - 33.3m / 116.3m                                                               \n",
      "11\t188k\t0.43\t\t0.49\t\t0.246\t\t0.912\t\t3.31m - 36.7m / 116.1m                                                               \n",
      "12\t205k\t0.43\t\t0.52\t\t0.237\t\t0.910\t\t3.30m - 40.0m / 116.2m                                                               \n",
      "13\t222k\t0.42\t\t0.46\t\t0.382\t\t0.959\t\t3.40m - 43.4m / 116.0m                                                               \n",
      "14\t239k\t0.42\t\t0.42\t\t0.382\t\t0.959\t\t3.32m - 46.7m / 118.2m                                                               \n",
      "15\t256k\t0.42\t\t0.48\t\t0.265\t\t0.917\t\t3.31m - 50.1m / 116.5m                                                               \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.3658519932856927                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3658519932856927\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\23\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 27.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         27                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.017516660304261217               |\n",
      "|  noam_learning_rate_warmup   |                        5409                       |\n",
      "|  noam_learning_rate_factor   |                 1.3870063334584157                |\n",
      "|          adam_beta1          |                 0.9290360992054372                |\n",
      "|          adam_beta2          |                 0.8976242694479695                |\n",
      "|           adam_eps           |               0.0011330229017342563               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         8                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.560025044281195                 |\n",
      "|     pointwise_layer_size     |                        240                        |\n",
      "|      last_layer_dropout      |                0.17740180933699207                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         78                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 67%|███████████████████████████▍             | 67/100 [25:02:40<14:40:42, 1601.28s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9832ba21393a498fb47f2824fea92728",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.52\t\t0.45\t\t0.344\t\t0.949\t\t3.69m - 3.7m / 0.0m                                                                    \n",
      "2\t34k\t0.46\t\t0.48\t\t0.199\t\t0.856\t\t3.77m - 7.6m / 129.1m                                                                  \n",
      "3\t51k\t0.45\t\t0.47\t\t0.331\t\t0.943\t\t3.70m - 11.3m / 132.0m                                                                 \n",
      "4\t68k\t0.45\t\t0.44\t\t0.285\t\t0.925\t\t3.72m - 15.0m / 129.7m                                                                 \n",
      "5\t85k\t0.46\t\t0.49\t\t0.382\t\t0.959\t\t3.74m - 18.8m / 130.4m                                                                 \n",
      "6\t102k\t0.47\t\t0.53\t\t0.265\t\t0.916\t\t3.79m - 22.6m / 130.9m                                                                \n",
      "7\t119k\t0.47\t\t0.50\t\t0.238\t\t0.911\t\t3.69m - 26.3m / 132.6m                                                                \n",
      "8\t137k\t0.47\t\t0.48\t\t0.238\t\t0.910\t\t3.69m - 30.0m / 129.6m                                                                \n",
      "9\t154k\t0.47\t\t0.45\t\t0.212\t\t0.874\t\t3.69m - 33.7m / 129.6m                                                                \n",
      "10\t171k\t0.46\t\t0.52\t\t0.141\t\t0.815\t\t3.70m - 37.4m / 129.7m                                                               \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.43916572534550014                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.43916572534550014\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\24\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 90.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         90                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.025344754740711097               |\n",
      "|  noam_learning_rate_warmup   |                        8834                       |\n",
      "|  noam_learning_rate_factor   |                 3.2261540660340513                |\n",
      "|          adam_beta1          |                 0.8775627305922284                |\n",
      "|          adam_beta2          |                 0.7031930631116351                |\n",
      "|           adam_eps           |                0.004298048585055463               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.603297596236655                 |\n",
      "|     pointwise_layer_size     |                         35                        |\n",
      "|      last_layer_dropout      |                 0.6472473373463112                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        183                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 68%|███████████████████████████▉             | 68/100 [25:42:06<16:16:20, 1830.65s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a4d6f81c13d4dfca6116bad94c7988a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.25\t\t0.17\t\t0.281\t\t0.928\t\t0.84m - 0.8m / 0.0m                                                                    \n",
      "2\t34k\t0.18\t\t0.17\t\t0.228\t\t0.896\t\t0.84m - 1.7m / 29.3m                                                                   \n",
      "3\t51k\t0.18\t\t0.18\t\t0.181\t\t0.847\t\t0.84m - 2.6m / 29.4m                                                                   \n",
      "4\t68k\t0.17\t\t0.19\t\t0.174\t\t0.835\t\t0.84m - 3.4m / 29.4m                                                                   \n",
      "5\t86k\t0.16\t\t0.21\t\t0.099\t\t0.718\t\t0.84m - 4.3m / 29.4m                                                                   \n",
      "6\t103k\t0.16\t\t0.24\t\t0.070\t\t0.641\t\t0.84m - 5.1m / 29.5m                                                                  \n",
      "VAL f1\t0.28066248880931066 - (0.28066248880931066)                                                                     \n",
      "VAL loss\t0.16834350116011026                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.16834350116011026\n",
      "        | \\     )|_\tf1: 0.28066248880931066\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\25\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 53.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         53                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0007882146442638553               |\n",
      "|  noam_learning_rate_warmup   |                        1768                       |\n",
      "|  noam_learning_rate_factor   |                 1.5055614956886911                |\n",
      "|          adam_beta1          |                 0.8426824971994313                |\n",
      "|          adam_beta2          |                 0.7463272710630966                |\n",
      "|           adam_eps           |               0.00013383167922919933              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.3213009560100158                |\n",
      "|     pointwise_layer_size     |                         35                        |\n",
      "|      last_layer_dropout      |                 0.7408028390478001                |\n",
      "|   output_conv_num_filters    |                        347                        |\n",
      "|   output_conv_kernel_size    |                         2                         |\n",
      "|      output_conv_stride      |                         7                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         49                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 69%|████████████████████████████▎            | 69/100 [25:48:05<11:57:51, 1389.39s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ddaf5483509143b4b5ddab956da48766",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.29\t\t0.27\t\t0.179\t\t0.841\t\t1.19m - 1.2m / 0.0m                                                                    \n",
      "2\t34k\t0.28\t\t0.28\t\t0.283\t\t0.910\t\t1.19m - 2.4m / 41.6m                                                                   \n",
      "3\t51k\t0.29\t\t0.33\t\t0.161\t\t0.792\t\t1.19m - 3.6m / 41.7m                                                                   \n",
      "4\t68k\t0.29\t\t0.35\t\t0.119\t\t0.682\t\t1.19m - 4.8m / 41.6m                                                                   \n",
      "5\t85k\t0.31\t\t0.32\t\t0.124\t\t0.698\t\t1.22m - 6.1m / 41.8m                                                                   \n",
      "6\t102k\t0.32\t\t0.31\t\t0.127\t\t0.712\t\t1.21m - 7.3m / 42.7m                                                                  \n",
      "7\t119k\t0.27\t\t0.29\t\t0.149\t\t0.758\t\t1.20m - 8.5m / 42.3m                                                                  \n",
      "VAL f1\t0.2834797891036907 - (0.2834797891036907)                                                                       \n",
      "VAL loss\t0.26667210117998946                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.26667210117998946\n",
      "        | \\     )|_\tf1: 0.2834797891036907\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\26\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 99.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         99                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.18608357745487716                |\n",
      "|  noam_learning_rate_warmup   |                        7183                       |\n",
      "|  noam_learning_rate_factor   |                 2.643973767383568                 |\n",
      "|          adam_beta1          |                 0.7175519273657989                |\n",
      "|          adam_beta2          |                 0.7679199294861847                |\n",
      "|           adam_eps           |               6.054905059117608e-09               |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.045091754864760164               |\n",
      "|     pointwise_layer_size     |                        158                        |\n",
      "|      last_layer_dropout      |                0.01764130406433431                |\n",
      "|   output_conv_num_filters    |                        137                        |\n",
      "|   output_conv_kernel_size    |                         3                         |\n",
      "|      output_conv_stride      |                         4                         |\n",
      "|     output_conv_padding      |                         0                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         54                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 70%|█████████████████████████████▍            | 70/100 [25:57:42<9:32:50, 1145.69s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d54fe755de6648faa97fc31a3eea7ed7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.18\t\t0.13\t\t0.233\t\t0.876\t\t1.02m - 1.0m / 0.0m                                                                    \n",
      "2\t34k\t0.13\t\t0.11\t\t0.291\t\t0.906\t\t1.01m - 2.1m / 35.6m                                                                   \n",
      "3\t51k\t0.12\t\t0.11\t\t0.288\t\t0.910\t\t1.01m - 3.1m / 35.4m                                                                   \n",
      "4\t69k\t0.11\t\t0.12\t\t0.204\t\t0.860\t\t1.00m - 4.1m / 35.4m                                                                   \n",
      "5\t86k\t0.10\t\t0.11\t\t0.358\t\t0.933\t\t1.00m - 5.1m / 35.2m                                                                   \n",
      "6\t103k\t0.10\t\t0.11\t\t0.208\t\t0.856\t\t1.00m - 6.1m / 35.2m                                                                  \n",
      "7\t120k\t0.10\t\t0.12\t\t0.280\t\t0.904\t\t1.00m - 7.2m / 35.2m                                                                  \n",
      "8\t137k\t0.10\t\t0.13\t\t0.316\t\t0.918\t\t1.00m - 8.2m / 35.1m                                                                  \n",
      "9\t154k\t0.10\t\t0.13\t\t0.218\t\t0.864\t\t1.03m - 9.2m / 35.2m                                                                  \n",
      "10\t171k\t0.10\t\t0.13\t\t0.259\t\t0.889\t\t1.03m - 10.3m / 36.1m                                                                \n",
      "VAL f1\t0.357553803537183 - (0.357553803537183)                                                                         \n",
      "VAL loss\t0.10884815182119334                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.10884815182119334\n",
      "        | \\     )|_\tf1: 0.357553803537183\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\27\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 85.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         85                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.039282831486777886               |\n",
      "|  noam_learning_rate_warmup   |                        2156                       |\n",
      "|  noam_learning_rate_factor   |                 2.117735482800423                 |\n",
      "|          adam_beta1          |                 0.8026372617998836                |\n",
      "|          adam_beta2          |                 0.927964183695142                 |\n",
      "|           adam_eps           |               2.3479962577130235e-06              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.19120719214616422                |\n",
      "|     pointwise_layer_size     |                        244                        |\n",
      "|      last_layer_dropout      |                 0.3379147558713902                |\n",
      "|   output_conv_num_filters    |                         88                        |\n",
      "|   output_conv_kernel_size    |                         4                         |\n",
      "|      output_conv_stride      |                         7                         |\n",
      "|     output_conv_padding      |                         5                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         53                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 71%|█████████████████████████████▊            | 71/100 [26:09:02<8:06:12, 1005.96s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f64facded95c448385202115184db8f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.19\t\t0.17\t\t0.195\t\t0.850\t\t0.75m - 0.8m / 0.0m                                                                    \n",
      "2\t34k\t0.17\t\t0.17\t\t0.212\t\t0.861\t\t0.76m - 1.6m / 26.4m                                                                   \n",
      "3\t51k\t0.15\t\t0.14\t\t0.289\t\t0.914\t\t0.74m - 2.3m / 26.6m                                                                   \n",
      "4\t68k\t0.14\t\t0.14\t\t0.267\t\t0.899\t\t0.75m - 3.1m / 26.0m                                                                   \n",
      "5\t85k\t0.13\t\t0.15\t\t0.243\t\t0.884\t\t0.75m - 3.9m / 26.3m                                                                   \n",
      "6\t103k\t0.12\t\t0.15\t\t0.186\t\t0.850\t\t0.75m - 4.6m / 26.3m                                                                  \n",
      "7\t120k\t0.11\t\t0.17\t\t0.168\t\t0.842\t\t0.75m - 5.4m / 26.4m                                                                  \n",
      "8\t137k\t0.11\t\t0.18\t\t0.117\t\t0.750\t\t0.75m - 6.2m / 26.3m                                                                  \n",
      "VAL f1\t0.2887453874538745 - (0.2887453874538745)                                                                       \n",
      "VAL loss\t0.1405706977844238                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.1405706977844238\n",
      "        | \\     )|_\tf1: 0.2887453874538745\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\28\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 83.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         83                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0019144285096226045               |\n",
      "|  noam_learning_rate_warmup   |                        6584                       |\n",
      "|  noam_learning_rate_factor   |                 0.8375264018153117                |\n",
      "|          adam_beta1          |                 0.7427352496267637                |\n",
      "|          adam_beta2          |                 0.8827926491950435                |\n",
      "|           adam_eps           |               0.0001628713643292417               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.08338941027887428                |\n",
      "|     pointwise_layer_size     |                         89                        |\n",
      "|      last_layer_dropout      |               0.0038161076942198058               |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         40                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 72%|██████████████████████████████▉            | 72/100 [26:15:58<6:26:51, 828.99s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4020a1cbcfa94e04963b446c808fb953",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.34\t\t0.20\t\t0.201\t\t0.877\t\t0.43m - 0.4m / 0.0m                                                                    \n",
      "2\t34k\t0.20\t\t0.17\t\t0.289\t\t0.921\t\t0.43m - 0.9m / 15.1m                                                                   \n",
      "3\t51k\t0.18\t\t0.16\t\t0.277\t\t0.910\t\t0.43m - 1.3m / 15.2m                                                                   \n",
      "4\t68k\t0.17\t\t0.15\t\t0.291\t\t0.913\t\t0.43m - 1.8m / 15.1m                                                                   \n",
      "5\t85k\t0.15\t\t0.14\t\t0.320\t\t0.924\t\t0.43m - 2.2m / 15.1m                                                                   \n",
      "6\t103k\t0.14\t\t0.14\t\t0.299\t\t0.913\t\t0.43m - 2.7m / 15.1m                                                                  \n",
      "7\t120k\t0.13\t\t0.13\t\t0.321\t\t0.922\t\t0.43m - 3.1m / 15.1m                                                                  \n",
      "8\t137k\t0.12\t\t0.13\t\t0.305\t\t0.916\t\t0.43m - 3.6m / 15.1m                                                                  \n",
      "9\t154k\t0.11\t\t0.12\t\t0.323\t\t0.922\t\t0.43m - 4.0m / 15.1m                                                                  \n",
      "10\t171k\t0.10\t\t0.13\t\t0.370\t\t0.937\t\t0.43m - 4.5m / 15.3m                                                                 \n",
      "11\t188k\t0.09\t\t0.12\t\t0.327\t\t0.922\t\t0.43m - 4.9m / 15.2m                                                                 \n",
      "12\t205k\t0.09\t\t0.12\t\t0.359\t\t0.933\t\t0.43m - 5.4m / 15.1m                                                                 \n",
      "13\t222k\t0.08\t\t0.13\t\t0.364\t\t0.934\t\t0.43m - 5.8m / 15.2m                                                                 \n",
      "14\t239k\t0.07\t\t0.13\t\t0.395\t\t0.943\t\t0.43m - 6.2m / 15.2m                                                                 \n",
      "15\t256k\t0.07\t\t0.14\t\t0.356\t\t0.932\t\t0.43m - 6.7m / 15.2m                                                                 \n",
      "16\t274k\t0.06\t\t0.13\t\t0.356\t\t0.933\t\t0.43m - 7.1m / 15.2m                                                                 \n",
      "17\t291k\t0.06\t\t0.14\t\t0.386\t\t0.941\t\t0.43m - 7.6m / 15.3m                                                                 \n",
      "18\t308k\t0.05\t\t0.15\t\t0.369\t\t0.936\t\t0.43m - 8.0m / 15.3m                                                                 \n",
      "19\t325k\t0.05\t\t0.15\t\t0.303\t\t0.917\t\t0.43m - 8.5m / 15.3m                                                                 \n",
      "VAL f1\t0.3949814126394052 - (0.3949814126394052)                                                                       \n",
      "VAL loss\t0.11720909026731928                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.11720909026731928\n",
      "        | \\     )|_\tf1: 0.3949814126394052\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\29\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 92.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         92                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.28584533790226996                |\n",
      "|  noam_learning_rate_warmup   |                        4598                       |\n",
      "|  noam_learning_rate_factor   |                 0.5902434563662821                |\n",
      "|          adam_beta1          |                 0.8442989364373867                |\n",
      "|          adam_beta2          |                 0.8714179835631862                |\n",
      "|           adam_eps           |                3.21837843276626e-08               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         8                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.5181977155237955                |\n",
      "|     pointwise_layer_size     |                         33                        |\n",
      "|      last_layer_dropout      |                0.21981388132183366                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        113                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 73%|███████████████████████████████▍           | 73/100 [26:25:15<5:36:14, 747.19s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c9ec1e6774448be82ec26325146be64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.23\t\t0.17\t\t0.159\t\t0.819\t\t1.65m - 1.7m / 0.0m                                                                    \n",
      "2\t34k\t0.18\t\t0.18\t\t0.139\t\t0.776\t\t1.66m - 3.3m / 57.8m                                                                   \n",
      "3\t51k\t0.17\t\t0.17\t\t0.146\t\t0.785\t\t1.66m - 5.0m / 58.0m                                                                   \n",
      "4\t68k\t0.17\t\t0.16\t\t0.150\t\t0.794\t\t1.65m - 6.7m / 58.0m                                                                   \n",
      "5\t86k\t0.16\t\t0.16\t\t0.162\t\t0.828\t\t1.65m - 8.3m / 57.9m                                                                   \n",
      "6\t103k\t0.16\t\t0.17\t\t0.158\t\t0.817\t\t1.65m - 10.0m / 57.9m                                                                 \n",
      "7\t120k\t0.15\t\t0.18\t\t0.108\t\t0.703\t\t1.67m - 11.7m / 58.0m                                                                 \n",
      "8\t137k\t0.15\t\t0.19\t\t0.115\t\t0.712\t\t1.66m - 13.4m / 58.4m                                                                 \n",
      "9\t154k\t0.15\t\t0.17\t\t0.155\t\t0.794\t\t1.66m - 15.1m / 58.3m                                                                 \n",
      "10\t171k\t0.14\t\t0.17\t\t0.127\t\t0.732\t\t1.66m - 16.7m / 58.3m                                                                \n",
      "VAL f1\t0.16191219077334523 - (0.16191219077334523)                                                                     \n",
      "VAL loss\t0.16106538488644057                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.16106538488644057\n",
      "        | \\     )|_\tf1: 0.16191219077334523\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\30\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 52.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         52                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               4.567485947218595e-05               |\n",
      "|  noam_learning_rate_warmup   |                        7298                       |\n",
      "|  noam_learning_rate_factor   |                0.16707230363180423                |\n",
      "|          adam_beta1          |                 0.9450123607176795                |\n",
      "|          adam_beta2          |                 0.8718605987924073                |\n",
      "|           adam_eps           |                 0.2582727648876223                |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.21252179647418246                |\n",
      "|     pointwise_layer_size     |                        142                        |\n",
      "|      last_layer_dropout      |                 0.4657464244508612                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        157                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 74%|███████████████████████████████▊           | 74/100 [26:43:24<6:08:14, 849.80s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0589c9b08f0436cb948b8a1074cfd42",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.55\t\t0.35\t\t0.117\t\t0.734\t\t1.70m - 1.7m / 0.0m                                                                    \n",
      "2\t34k\t0.32\t\t0.26\t\t0.249\t\t0.907\t\t1.69m - 3.4m / 59.5m                                                                   \n",
      "3\t51k\t0.28\t\t0.26\t\t0.252\t\t0.909\t\t1.69m - 5.1m / 59.3m                                                                   \n",
      "4\t68k\t0.28\t\t0.25\t\t0.315\t\t0.939\t\t1.69m - 6.9m / 59.3m                                                                   \n",
      "5\t85k\t0.27\t\t0.25\t\t0.281\t\t0.924\t\t1.71m - 8.6m / 59.1m                                                                   \n",
      "6\t102k\t0.27\t\t0.25\t\t0.274\t\t0.917\t\t1.73m - 10.3m / 60.0m                                                                 \n",
      "7\t119k\t0.26\t\t0.24\t\t0.283\t\t0.922\t\t1.68m - 12.0m / 60.6m                                                                 \n",
      "8\t136k\t0.26\t\t0.24\t\t0.292\t\t0.925\t\t1.69m - 13.7m / 59.2m                                                                 \n",
      "9\t154k\t0.26\t\t0.24\t\t0.271\t\t0.910\t\t1.68m - 15.4m / 59.3m                                                                 \n",
      "VAL f1\t0.31479666829985303 - (0.31479666829985303)                                                                     \n",
      "VAL loss\t0.23567424966738776                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.23567424966738776\n",
      "        | \\     )|_\tf1: 0.31479666829985303\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\31\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 19.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         19                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.010910262630578708               |\n",
      "|  noam_learning_rate_warmup   |                        1910                       |\n",
      "|  noam_learning_rate_factor   |                 3.3438351656574925                |\n",
      "|          adam_beta1          |                 0.836512215850245                 |\n",
      "|          adam_beta2          |                 0.722976217118222                 |\n",
      "|           adam_eps           |               4.0310400277798337e-07              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.6072444648219592                |\n",
      "|     pointwise_layer_size     |                        151                        |\n",
      "|      last_layer_dropout      |                0.18689720186567832                |\n",
      "|   output_conv_num_filters    |                         98                        |\n",
      "|   output_conv_kernel_size    |                         4                         |\n",
      "|      output_conv_stride      |                         7                         |\n",
      "|     output_conv_padding      |                         1                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         63                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 75%|████████████████████████████████▎          | 75/100 [27:00:16<6:14:22, 898.48s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14df847baf7147bca891e603a9e518cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.82\t\t0.79\t\t0.265\t\t0.916\t\t3.80m - 3.8m / 0.0m                                                                    \n",
      "2\t34k\t0.84\t\t0.74\t\t0.382\t\t0.959\t\t3.82m - 7.6m / 132.9m                                                                  \n",
      "3\t51k\t0.71\t\t0.66\t\t0.382\t\t0.959\t\t3.79m - 11.5m / 133.8m                                                                 \n",
      "4\t68k\t0.67\t\t0.58\t\t0.382\t\t0.959\t\t3.83m - 15.3m / 132.8m                                                                 \n",
      "5\t85k\t0.66\t\t0.59\t\t0.382\t\t0.959\t\t3.84m - 19.2m / 134.0m                                                                 \n",
      "6\t102k\t0.64\t\t0.56\t\t0.382\t\t0.959\t\t3.88m - 23.1m / 134.3m                                                                \n",
      "7\t119k\t0.65\t\t0.58\t\t0.382\t\t0.959\t\t3.82m - 26.9m / 135.6m                                                                \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.5608124909577549                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.5608124909577549\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\32\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 17.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         17                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.05274230127260164                |\n",
      "|  noam_learning_rate_warmup   |                        4029                       |\n",
      "|  noam_learning_rate_factor   |                 0.9501480569273263                |\n",
      "|          adam_beta1          |                 0.729319628896962                 |\n",
      "|          adam_beta2          |                 0.8244928187973734                |\n",
      "|           adam_eps           |               1.1951669918154056e-09              |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.7491943798512692                |\n",
      "|     pointwise_layer_size     |                        169                        |\n",
      "|      last_layer_dropout      |                 0.6373774861635262                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         37                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                       False                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 76%|███████████████████████████████▉          | 76/100 [27:29:19<7:40:47, 1151.98s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d336f5c01eab46bd917a204f3df34836",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.80\t\t0.77\t\t0.158\t\t0.826\t\t2.39m - 2.4m / 0.0m                                                                    \n",
      "2\t34k\t0.63\t\t0.84\t\t0.280\t\t0.913\t\t2.44m - 4.9m / 83.5m                                                                   \n",
      "3\t51k\t0.63\t\t0.88\t\t0.281\t\t0.917\t\t2.41m - 7.3m / 85.5m                                                                   \n",
      "4\t68k\t0.64\t\t1.17\t\t0.288\t\t0.942\t\t2.49m - 9.8m / 84.6m                                                                   \n",
      "5\t85k\t0.64\t\t1.12\t\t0.322\t\t0.933\t\t2.40m - 12.2m / 87.1m                                                                  \n",
      "6\t102k\t0.65\t\t1.22\t\t0.339\t\t0.948\t\t2.42m - 14.7m / 84.1m                                                                 \n",
      "7\t119k\t0.63\t\t1.66\t\t0.346\t\t0.952\t\t2.38m - 17.1m / 84.9m                                                                 \n",
      "8\t136k\t0.62\t\t2.05\t\t0.388\t\t0.959\t\t2.40m - 19.5m / 83.9m                                                                 \n",
      "9\t153k\t0.62\t\t2.51\t\t0.372\t\t0.958\t\t2.36m - 21.9m / 84.2m                                                                 \n",
      "10\t171k\t0.60\t\t2.80\t\t0.382\t\t0.959\t\t2.36m - 24.3m / 83.2m                                                                \n",
      "11\t188k\t0.61\t\t3.22\t\t0.382\t\t0.959\t\t2.36m - 26.6m / 83.3m                                                                \n",
      "12\t205k\t0.61\t\t3.30\t\t0.382\t\t0.959\t\t2.38m - 29.0m / 83.4m                                                                \n",
      "13\t222k\t0.62\t\t3.62\t\t0.382\t\t0.959\t\t2.35m - 31.4m / 83.7m                                                                \n",
      "VAL f1\t0.3875432525951557 - (0.3875432525951557)                                                                       \n",
      "VAL loss\t0.7725655968813973                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.7725655968813973\n",
      "        | \\     )|_\tf1: 0.3875432525951557\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\33\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 57.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         57                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.008252181630693677               |\n",
      "|  noam_learning_rate_warmup   |                        5594                       |\n",
      "|  noam_learning_rate_factor   |                0.22211264920509918                |\n",
      "|          adam_beta1          |                 0.9940350367811108                |\n",
      "|          adam_beta2          |                 0.7974522005693725                |\n",
      "|           adam_eps           |                0.004768784668815792               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         5                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.7884823540121073                |\n",
      "|     pointwise_layer_size     |                        105                        |\n",
      "|      last_layer_dropout      |                 0.3116771781355313                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        221                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 77%|████████████████████████████████▎         | 77/100 [28:02:41<8:59:15, 1406.77s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a0e9e06ac1f45e6880a2bfcb08410b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.45\t\t0.33\t\t0.023\t\t0.852\t\t2.65m - 2.6m / 0.0m                                                                    \n",
      "2\t34k\t0.29\t\t0.29\t\t0.382\t\t0.959\t\t2.63m - 5.3m / 92.7m                                                                   \n",
      "3\t51k\t0.26\t\t0.30\t\t0.265\t\t0.916\t\t2.64m - 8.0m / 92.2m                                                                   \n",
      "4\t68k\t0.26\t\t0.29\t\t0.265\t\t0.916\t\t2.75m - 10.7m / 92.5m                                                                  \n",
      "5\t85k\t0.26\t\t0.29\t\t0.265\t\t0.916\t\t2.68m - 13.4m / 96.0m                                                                  \n",
      "6\t102k\t0.25\t\t0.29\t\t0.265\t\t0.916\t\t2.65m - 16.1m / 93.8m                                                                 \n",
      "7\t119k\t0.26\t\t0.29\t\t0.265\t\t0.916\t\t2.65m - 18.8m / 92.9m                                                                 \n",
      "VAL f1\t0.38205453392517436 - (0.38205453392517436)                                                                     \n",
      "VAL loss\t0.2853334898139998                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2853334898139998\n",
      "        | \\     )|_\tf1: 0.38205453392517436\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEvalHyperoptRandom\\20190425\\34\n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 61.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         61                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0022971692852368506               |\n",
      "|  noam_learning_rate_warmup   |                        1136                       |\n",
      "|  noam_learning_rate_factor   |                 3.363922745971823                 |\n",
      "|          adam_beta1          |                 0.8286958241177107                |\n",
      "|          adam_beta2          |                 0.8369242471254511                |\n",
      "|           adam_eps           |               1.888797968899112e-08               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         7                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.2977643755571659                |\n",
      "|     pointwise_layer_size     |                        145                        |\n",
      "|      last_layer_dropout      |                0.20677566850741275                |\n",
      "|   output_conv_num_filters    |                        220                        |\n",
      "|   output_conv_kernel_size    |                         6                         |\n",
      "|      output_conv_stride      |                         4                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        107                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 78%|████████████████████████████████▊         | 78/100 [28:23:33<8:18:47, 1360.35s/it, best loss: 0.10995660924978375]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3071cdcfaa4441984dbfed4d8a3d70f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t17k\t0.29\t\t0.28\t\t0.246\t\t0.912\t\t2.54m - 2.5m / 0.0m                                                                    \n",
      "2\t34k\t0.30\t\t0.29\t\t0.212\t\t0.874\t\t2.63m - 5.2m / 88.8m                                                                   \n",
      "3\t51k\t0.30\t\t0.28\t\t0.265\t\t0.916\t\t2.60m - 7.8m / 91.8m                                                                   \n",
      "4\t68k\t0.28\t\t0.26\t\t0.265\t\t0.916\t\t2.88m - 10.7m / 91.1m                                                                  \n",
      " 78%|████████████████████████████████▊         | 78/100 [28:34:28<8:18:47, 1360.35s/it, best loss: 0.10995660924978375]"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    logger.info('Current commit: ' + utils.get_current_git_commit())\n",
    "    print('Current commit: ' + utils.get_current_git_commit())\n",
    "except Exception as err:\n",
    "    logger.exception('Could not print current commit')\n",
    "\n",
    "trials = Trials()\n",
    "try:\n",
    "\n",
    "    best = fmin(objective,\n",
    "        space=search_space,\n",
    "        algo=rand.suggest,\n",
    "        max_evals=runs,\n",
    "        trials=trials)\n",
    "\n",
    "    print(best)\n",
    "except Exception as err:\n",
    "    logger.exception('Could not complete optimization')\n",
    "    print('Could not complete optimization. The log file provides more details.')\n",
    "\n",
    "\n",
    "path = os.path.join(os.getcwd(), 'logs', f'hp_run_{main_experiment_name}.pkl')\n",
    "with open(path, 'wb') as f:\n",
    "    pickle.dump(trials, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}