12_HyperOpt-Organic2019.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\felix\\\\Anaconda3\\\\python.exe'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "sys.path\n",
    "\n",
    "sys.executable\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import time\n",
    "import logging\n",
    "from hyperopt.plotting import *\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials, base\n",
    "from data.data_loader import Dataset\n",
    "from misc.preferences import PREFERENCES\n",
    "from misc.run_configuration import from_hyperopt, OutputLayerType, LearningSchedulerType, OptimizerType\n",
    "from misc import utils\n",
    "from misc.hyperopt_space import *\n",
    "\n",
    "from optimizer import get_optimizer\n",
    "from criterion import NllLoss, LossCombiner\n",
    "from models.transformer.encoder import TransformerEncoder\n",
    "from models.jointAspectTagger import JointAspectTagger\n",
    "from trainer.train import Trainer\n",
    "import pprint\n",
    "from data.organic2019 import organic_dataset as dsl\n",
    "from data.organic2019 import ORGANIC_TASK_ALL, ORGANIC_TASK_ENTITIES, ORGANIC_TASK_ATTRIBUTES, ORGANIC_TASK_ENTITIES_COMBINE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Functions\n",
    "\n",
    "These functions will load the dataset and the model. The run configuration will determine the architecture and hyper parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_model(dataset, rc, experiment_name):\n",
    "    loss = LossCombiner(4, dataset.class_weights, NllLoss)\n",
    "    transformer = TransformerEncoder(dataset.source_embedding,\n",
    "                                     hyperparameters=rc)\n",
    "    model = JointAspectTagger(transformer, rc, 4, 20, dataset.target_names)\n",
    "    optimizer = get_optimizer(model, rc)\n",
    "    trainer = Trainer(\n",
    "                        model,\n",
    "                        loss,\n",
    "                        optimizer,\n",
    "                        rc,\n",
    "                        dataset,\n",
    "                        experiment_name,\n",
    "                        enable_tensorboard=False,\n",
    "                        verbose=False)\n",
    "    return trainer\n",
    "\n",
    "def load_dataset(rc, logger, task):\n",
    "    dataset = Dataset(\n",
    "        task,\n",
    "        logger,\n",
    "        rc,\n",
    "        source_index=0,\n",
    "        target_vocab_index=1,\n",
    "        data_path=PREFERENCES.data_root,\n",
    "        train_file=PREFERENCES.data_train,\n",
    "        valid_file=PREFERENCES.data_validation,\n",
    "        test_file=PREFERENCES.data_test,\n",
    "        file_format='.tsv',\n",
    "        init_token=None,\n",
    "        eos_token=None\n",
    "    )\n",
    "    dataset.load_data(dsl, verbose=False)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preamble - Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PREFERENCES.defaults(\n",
    "    data_root='./data/data/organic2019',\n",
    "    data_train='train.csv',    \n",
    "    data_validation='validation.csv',\n",
    "    data_test='test.csv',\n",
    "    early_stopping='highest_5_F1'\n",
    ")\n",
    "main_experiment_name = 'Organic_HyperOpt'\n",
    "use_cuda = True\n",
    "\n",
    "# get general logger just for search\n",
    "experiment_name = utils.create_loggers(experiment_name=main_experiment_name)\n",
    "logger = logging.getLogger(__name__)\n",
    "dataset_logger = logging.getLogger('data_loader')\n",
    "logger.info('Run hyper parameter random grid search for experiment with name ' + main_experiment_name)\n",
    "\n",
    "num_optim_iterations = 100\n",
    "logger.info('num_optim_iterations: ' + str(num_optim_iterations))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Current commit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "utils.get_current_git_commit()\n",
    "logger.info('Current commit: ' + utils.get_current_git_commit())\n",
    "print('Current commit: ' + utils.get_current_git_commit())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search Spaces\n",
    "\n",
    "- BatchSize:\n",
    "    How big should each batch be?\n",
    "- Num Encoder Blocks\n",
    "    How many encoder blocks should be replicated?\n",
    "    AYNIA: 2-8\n",
    "    \n",
    "- Pointwise Layer Size\n",
    "    How big should the layer between attention heads be?\n",
    "    AYNIA: 1024 - 4096\n",
    "    This: 64 - 2048\n",
    "    \n",
    "    64: Prev. Experiments have shown that a smaller size can be beneficial because a smaller layer contains less parameters.\n",
    "    2048: This model has about a third of the AYNIA model size (1000 vs. 300). Going to big, therefore doesn't make much sense.\n",
    "\n",
    "- Clip Comments to \n",
    "    How long should comments be\n",
    "    This: 30 - 500\n",
    "    \n",
    "- Initial Learning Rate\n",
    "    What is the initial learning rate\n",
    "- Optimizer:\n",
    "    - Noam:\n",
    "        (FROM: https://github.com/tensorflow/tensor2tensor/issues/280#issuecomment-359477755)\n",
    "        decreasing the learning rate aka learning rate decay (usually exponential, piecewise-constant or inverse-time) is a standard practice in ML for decades. Increasing the learning rate in the early stages with a warmup (usually linear or exponential growth) is a more recent practice, popular esp. in deep learning on ImageNet, see e.g. He et al. 2016 or Goyal et al. 2017.\n",
    "        The \"noam\" scheme is just a particular way how to put the warmup and decay together (linear warmup for a given number of steps followed by exponential decay).\n",
    "\n",
    "        Learning rate schedules is an active research area. See e.g. papers on cyclical learning rate (corresponding to learning_rate_decay_scheme=cosine available in tensor2tensor) and super-convergence, which provide also more insights into the theory behind the learning rate, batch size, gradient noise etc.\n",
    "    \n",
    "        - learning rate factor\n",
    "        - learning rate warmup (steps)\n",
    "            AYNIA: 4000\n",
    "            THIS: 100 - 8000\n",
    "    - Adam:\n",
    "        - Beta 1\n",
    "            AYNIA: 0.9\n",
    "\n",
    "        - Beta 2\n",
    "            AYNIA: 0.98\n",
    "\n",
    "\n",
    "    - ?\n",
    "- Transformer Dropout Rate\n",
    "    Dropout rate for the transformer layers.\n",
    "    AYNIA: 0.1\n",
    "    THIS: 0.1 - 0.8\n",
    "- Number of Transformer Heads\n",
    "    How many attention heads should be used:\n",
    "    AYNIA: 8\n",
    "    THIS: [1, 2, 3, 4, 5, 6, 10, 12, 15, 20] (Have to be divide 300)\n",
    "    \n",
    "- Last Layer Dropout Rate\n",
    "    Dropout rate right before the last layer\n",
    "    AYNIA: -\n",
    "    This 0.0 - 0.8\n",
    "- Last Layer Types\n",
    "    - Sum\n",
    "    - Convolutions:\n",
    "        - num conv filters\n",
    "        - kernel size\n",
    "        - stride\n",
    "        - padding\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#search_space = hp\n",
    "search_space = {\n",
    "    'batch_size': hp.quniform('batch_size', 10, 64, 1),\n",
    "    'num_encoder_blocks': hp.quniform('num_encoder_blocks', 1, 4, 1),\n",
    "    'pointwise_layer_size': hp.quniform('pointwise_layer_size', 32, 350, 1),\n",
    "    'clip_comments_to': hp.quniform('clip_comments_to', 45, 180, 1),\n",
    "    'dropout_rate': hp.uniform('dropout_rate', 0.0, 0.8),\n",
    "    'output_dropout_rate': hp.uniform('last_layer_dropout', 0.0, 0.8),\n",
    "    'num_heads': hp.choice('num_heads', [1, 2, 3, 4, 5]),\n",
    "    'transformer_use_bias': hp_bool('transformer_use_bias'),\n",
    "    'output_layer': hp.choice('output_layer', [\n",
    "        {\n",
    "            'type': OutputLayerType.Convolutions,\n",
    "            'output_conv_num_filters': hp.quniform('output_conv_num_filters', 10, 400, 1),\n",
    "            'output_conv_kernel_size': hp.quniform('output_conv_kernel_size', 1, 10, 1),\n",
    "            'output_conv_stride': hp.quniform('output_conv_stride', 1, 10, 1),\n",
    "            'output_conv_padding': hp.quniform('output_conv_padding', 0, 5, 1),\n",
    "        },\n",
    "        {\n",
    "            'type': OutputLayerType.LinearSum\n",
    "        }\n",
    "    ]),\n",
    "    'learning_rate_scheduler': hp.choice('learning_rate_scheduler', [\n",
    "        {\n",
    "            'type': LearningSchedulerType.Noam,\n",
    "            'noam_learning_rate_warmup': hp.quniform('noam_learning_rate_warmup', 1000, 9000, 1),\n",
    "            'noam_learning_rate_factor': hp.uniform('noam_learning_rate_factor', 0.01, 4)\n",
    "        }\n",
    "    ]),\n",
    "    'optimizer': hp.choice('optimizer', [\n",
    "        {\n",
    "            'type': OptimizerType.Adam,\n",
    "            'adam_beta1': hp.uniform('adam_beta1', 0.7, 0.999),\n",
    "            'adam_beta2': hp.uniform('adam_beta2', 0.7, 0.999),\n",
    "            'adam_eps': hp.loguniform('adam_eps', np.log(1e-10), np.log(1)),\n",
    "            'learning_rate': hp.lognormal('adam_learning_rate', np.log(0.01), np.log(10)),\n",
    "            'adam_weight_decay': 1*10**hp.quniform('adam_weight_decay', -8, -3, 1)\n",
    "        },\n",
    "        #{\n",
    "        #    'type': OptimizerType.SGD,\n",
    "        #    'sgd_momentum': hp.uniform('sgd_momentum', 0.4, 1),\n",
    "        #    'sgd_weight_decay': hp.loguniform('sgd_weight_decay', np.log(1e-4), np.log(1)),\n",
    "        #    'sgd_nesterov': hp_bool('sgd_nesterov'),\n",
    "        #    'learning_rate': hp.lognormal('sgd_learning_rate', np.log(0.01), np.log(10))\n",
    "    ]),\n",
    "    'task': hp.choice('task', [\n",
    "        ORGANIC_TASK_ENTITIES,\n",
    "        ORGANIC_TASK_ENTITIES_COMBINE\n",
    "    ]),\n",
    "    'use_spell_checker': hp_bool('use_spell_checker'),\n",
    "    'embedding_type': hp.choice('embedding_type', ['fasttext', 'glove'])\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def objective(parameters):\n",
    "    run_time = time.time()\n",
    "    \n",
    "    utils.reset_loggers()\n",
    "    experiment_name = utils.create_loggers(experiment_name=main_experiment_name)\n",
    "    logger = logging.getLogger(__name__)\n",
    "    dataset_logger = logging.getLogger('data_loader')\n",
    "\n",
    "    # generate hp's from parameters\n",
    "    try:\n",
    "        rc = from_hyperopt(parameters, use_cuda, model_size=300, early_stopping=5, num_epochs=35, log_every_xth_iteration=-1, language='en')\n",
    "    except Exception as err:\n",
    "        print('Could not convert params: ' + str(err))\n",
    "        logger.exception(\"Could not load parameters from hyperopt configuration: \" + parameters)\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "    logger.info('New Params:')\n",
    "    logger.info(rc)\n",
    "    print('\\n\\n#########################################################################')\n",
    "    print(rc)\n",
    "\n",
    "    logger.debug('Load dataset')\n",
    "    try:\n",
    "        dataset = load_dataset(rc, dataset_logger, rc.task)\n",
    "    except Exception as err:\n",
    "        print('Could not load dataset: ' + str(err))\n",
    "        logger.exception(\"Could not load dataset\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "    logger.debug('dataset loaded')\n",
    "    logger.debug('Load model')\n",
    "\n",
    "    try:\n",
    "        trainer = load_model(dataset, rc, experiment_name)\n",
    "    except Exception as err:\n",
    "        print('Could not load model: ' + str(err))\n",
    "        logger.exception(\"Could not load model\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "\n",
    "    logger.debug('model loaded')\n",
    "\n",
    "    logger.debug('Begin training')\n",
    "    model = None\n",
    "    try:\n",
    "        result = trainer.train(use_cuda=rc.use_cuda, perform_evaluation=False)\n",
    "        model = result['model']\n",
    "    except Exception as err:\n",
    "        print('Exception while training: ' + str(err))\n",
    "        logger.exception(\"Could not complete iteration\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    if math.isnan(trainer.get_best_loss()):\n",
    "        print('Loss is nan')\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    # perform evaluation and log results\n",
    "    result = None\n",
    "    try:\n",
    "        result = trainer.perform_final_evaluation(use_test_set=True, verbose=False)\n",
    "    except Exception as err:\n",
    "        logger.exception(\"Could not complete iteration evaluation.\")\n",
    "        print('Could not complete iteration evaluation: ' + str(err))\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "    print(f'VAL f1\\t{trainer.get_best_f1()} - ({result[1][1]})')\n",
    "    print(f'VAL loss\\t{trainer.get_best_loss()}')\n",
    "    \n",
    "    print(f\"       .---.\\n \\\n",
    "         /     \\\\\\n\\\n",
    "          \\\\.@-@./\\n\\\n",
    "          /`\\\\_/`\\\\\\n\\\n",
    "         //  _  \\\\\\\\\\tLoss: {trainer.get_best_loss()}\\n\\\n",
    "        | \\\\     )|_\\tf1: {trainer.get_best_f1()}\\n\\\n",
    "       /`\\\\_`>  <_/ \\\\\\n\\\n",
    "       \\\\__/'---'\\\\__/\\n\")\n",
    "    \n",
    "    return {\n",
    "            'loss': result[1][0],\n",
    "            'status': STATUS_OK,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1(),\n",
    "            'sample_iterations': trainer.get_num_samples_seen(),\n",
    "            'iterations': trainer.get_num_iterations(),\n",
    "            'rc': rc,\n",
    "            'results': {\n",
    "                'train': {\n",
    "                    'loss': result[0][0],\n",
    "                    'f1': result[0][1]\n",
    "                },\n",
    "                'validation': {\n",
    "                    'loss': result[1][0],\n",
    "                    'f1': result[1][1]\n",
    "                },\n",
    "                'test': {\n",
    "                    'loss': result[2][0],\n",
    "                    'f1': result[2][1]\n",
    "                }\n",
    "            }\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_objective(params):\n",
    "    rc = from_hyperopt(params, use_cuda, 300, 4, 35, -1, 'de')\n",
    "    #print(rc)\n",
    "\n",
    "    return {\n",
    "        'loss': params['x'] ** 2,\n",
    "        'status': STATUS_OK\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trials = Trials()\n",
    "# domain = base.Domain(test_objective, search_space) \n",
    "\n",
    "best = fmin(objective,\n",
    "    space=search_space,\n",
    "    algo=tpe.suggest,\n",
    "    max_evals=num_optim_iterations,\n",
    "    trials=trials)\n",
    "\n",
    "print(best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "path = 'C:\\\\Users\\\\felix\\\\OneDrive\\\\Studium\\\\Studium\\\\6. Semester\\\\MA\\\\Project\\\\ABSA-Transformer\\\\logs\\\\Organic_HyperOpt\\\\20190327\\\\0\\\\trials.pkl'\n",
    "with open(path, 'wb') as f:\n",
    "    pickle.dump(trials, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trials.trials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "plt.figure(num=None, figsize=(20, 10), dpi=80)\n",
    "main_plot_history(trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "losses = [t['result']['loss'] for t in trials.trials if t['result']['status'] == STATUS_OK]\n",
    "range(len(losses))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(num=None, figsize=(20, 10), dpi=80)\n",
    "fig.suptitle('Loss over time')\n",
    "plt.scatter(range(len(losses)), losses)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "main_plot_histogram(trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.figure(num=None, figsize=(25, 15), dpi=80)\n",
    "main_plot_vars(trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}