alldata-random-groups.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3476165b",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0afcc617",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "80697c05",
   "metadata": {},
   "outputs": [],
   "source": [
    "import data\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6c222f97",
   "metadata": {},
   "outputs": [],
   "source": [
    "readerTypes = [data.CacuData, data.FccPmData, data.SuperconductivityData, data.GarmentProductivityData]\n",
    "targets = ['ViolentCrimesPerPop', 'PM', 'critical_temp', 'actual_productivity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3531c585",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p models_clean_randgroup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a69069dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_group_constraints(groups):\n",
    "    interaction_constraints = []\n",
    "    i=0\n",
    "    for g in groups:\n",
    "        interaction_constraints.append(list(range(i, i+len(g))))\n",
    "        i += len(g)\n",
    "    return interaction_constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3c683b3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------\n",
      "Dataset: Communities and Crime Unnormalized\n",
      "Calculating full model mse...\n",
      "Calculating E^r model mse...\n",
      "Calculating semantic group E^r mse...\n",
      "Calculating random group 2 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "109da930b0ac4a0496b74e12f491a0bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 3 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d9ab51808874d8c85fdd0471d718eb8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 5 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6df572b29f84aec998ce18679d38504",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 6 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f63be93cf2264a008bf85c160bba60c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------\n",
      "Dataset: PM2.5 Data of Five Chinese Cities\n",
      "Calculating full model mse...\n",
      "Calculating E^r model mse...\n",
      "Calculating random group 2 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5c19c3ed61143f59020e3f2831a2fe5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 3 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d0ecd4448c1491faca38c8ae601580f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------\n",
      "Dataset: Superconductivity\n",
      "Calculating full model mse...\n",
      "Calculating E^r model mse...\n",
      "Calculating random group 2 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d405f5b383b42f1b969067d02f08033",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 3 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59a1f0feb4dd4dfdba1cf1e9f7213b20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 5 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbeaec4a01ae462d8bdfc4a7923427d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 6 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c208b79c26b4f8f8ff575e82cfd3898",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------\n",
      "Dataset: Productivity Prediction of Garment Employees\n",
      "Calculating full model mse...\n",
      "Calculating E^r model mse...\n",
      "Calculating random group 2 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00fabf2d778a4b28848858d98a44a077",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 3 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fbb741f3b2e4f3088ad54db21d48d4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mse_collector = {}\n",
    "\n",
    "for ReaderType, target in zip(readerTypes, targets):\n",
    "    reader = ReaderType('data/UCI/archive.ics.uci.edu/ml/machine-learning-databases')\n",
    "    reader.readAndCache()\n",
    "    print(\"---------\")\n",
    "    print(\"Dataset:\", reader.name)\n",
    "    cols = reader.cols\n",
    "    ncols = len(cols)\n",
    "\n",
    "    trainer = utils.Trainer(\n",
    "        cache_dir='models_clean',\n",
    "        default_params={\n",
    "            'objective': 'regression',\n",
    "            'metric': 'mse',\n",
    "            'boosting': 'gbdt',\n",
    "            'num_leaves': 31,\n",
    "            'learning_rate': 0.05,\n",
    "            'verbose': -1,\n",
    "            'device': 'cpu',\n",
    "            'lambda_l2': 0.003,\n",
    "            'early_stopping_round': 10,\n",
    "            'num_boost_round': 100,\n",
    "        },\n",
    "        nocache_params={\n",
    "            'num_threads': 20,\n",
    "        }\n",
    "    )\n",
    "    evaluator = utils.Evaluator(reader, target)\n",
    "    \n",
    "    data_mse = {}\n",
    "    \n",
    "    print(\"Calculating full model mse...\")\n",
    "    trainData = utils.TrainingData(reader, target, columns=cols)\n",
    "    booster = trainer.train_model(trainData)\n",
    "    new_mse = evaluator.get_model_mse(booster)\n",
    "    data_mse['base_mse'] = evaluator.base_mse\n",
    "    data_mse['features_e'] = {'ngroups': 1, 'mse':new_mse}\n",
    "    \n",
    "    print(\"Calculating E^r model mse...\")\n",
    "    booster = trainer.train_model(trainData, {\n",
    "        'interaction_constraints': [[i] for i in range(len(reader.cols))]\n",
    "    })\n",
    "    new_mse = evaluator.get_model_mse(booster)\n",
    "    data_mse['features_er'] = {'ngroups': len(reader.cols), 'mse':new_mse}\n",
    "    mse_collector[reader.name] = data_mse\n",
    "    \n",
    "    #e_r semantic groups\n",
    "    if reader.groups is not None:\n",
    "        print(\"Calculating semantic group E^r mse...\")\n",
    "        trainData = utils.TrainingData(reader, target, columns=[i for g in reader.groups.values() for i in g])\n",
    "        booster = trainer.train_model(trainData, {\n",
    "            'interaction_constraints': get_group_constraints(reader.groups),\n",
    "        })\n",
    "        new_mse = evaluator.get_model_mse(booster)\n",
    "        data_mse['semantic_groups'] = {'ngroups': len(reader.groups), 'mse':new_mse}\n",
    "    \n",
    "    for groupsize in [2,3,5,6]:\n",
    "        np.random.seed(reader.seed)\n",
    "        ngroups = len(cols)//groupsize\n",
    "        if ngroups <= 2:\n",
    "            continue\n",
    "        mses = []\n",
    "        print(\"Calculating random group\", groupsize, \"E^r mse...\")\n",
    "        for i in tqdm(range(100)):\n",
    "            feature_perm = np.random.permutation(cols)\n",
    "            groups = np.array_split(feature_perm, ngroups)[::-1] # array split puts longer groups first, lets switch the order\n",
    "            trainData = utils.TrainingData(reader, target, columns=[i for g in groups for i in g])\n",
    "            booster = trainer.train_model(trainData, {\n",
    "                'interaction_constraints': get_group_constraints(groups),\n",
    "            })\n",
    "            new_mse = evaluator.get_model_mse(booster)\n",
    "            mses.append(new_mse)\n",
    "        data_mse['random_groups_'+str(groupsize)] = {'ngroups': ngroups, 'mse':np.mean(mses)}\n",
    "    mse_collector[reader.name] = data_mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f1fc5d49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Communities and Crime Unnormalized\n",
      "- features_e 1 0.38027166104328924\n",
      "- features_er 124 0.4062480053720343\n",
      "- semantic_groups 13 0.3856312415736422\n",
      "- random_groups_2 62 0.4060129406739165\n",
      "- random_groups_3 41 0.3997381865503116\n",
      "- random_groups_5 24 0.3806585189902847\n",
      "- random_groups_6 20 0.37401852563522353\n",
      "PM2.5 Data of Five Chinese Cities\n",
      "- features_e 1 0.6251303299193164\n",
      "- features_er 14 0.8326940909237712\n",
      "- random_groups_2 7 0.7823754497255953\n",
      "- random_groups_3 4 0.7336194592628724\n",
      "Superconductivity\n",
      "- features_e 1 0.09774168690569907\n",
      "- features_er 81 0.17746699215750641\n",
      "- random_groups_2 40 0.1424491548571722\n",
      "- random_groups_3 27 0.13269479017036584\n",
      "- random_groups_5 16 0.12284317077204072\n",
      "- random_groups_6 13 0.11926090689541963\n",
      "Productivity Prediction of Garment Employees\n",
      "- features_e 1 0.5532743667197897\n",
      "- features_er 14 0.6151372488688107\n",
      "- random_groups_2 7 0.5861638971292005\n",
      "- random_groups_3 4 0.5760601387406408\n"
     ]
    }
   ],
   "source": [
    "for name, data_mse in mse_collector.items():\n",
    "    print(name)\n",
    "    for kind, mse in data_mse.items():\n",
    "        if kind == \"base_mse\":\n",
    "            continue\n",
    "        print(\"-\", kind, mse['ngroups'], mse['mse']/data_mse['base_mse'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e43e442c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\midrule\n",
      "\\multicolumn{3}{l}{Communities and Crime Unnormalized}\\\\\n",
      "\\midrule\n",
      "Base features ($E$)&1&0.3803\\\\\n",
      "Base features ($E^r$)&124&0.4062\\\\\n",
      "Semantic groups&13&0.3856\\\\\n",
      "Random groups (2)&62&0.4060\\\\\n",
      "Random groups (3)&41&0.3997\\\\\n",
      "Random groups (5)&24&0.3807\\\\\n",
      "Random groups (6)&20&0.3740\\\\\n",
      "\\midrule\n",
      "\\multicolumn{3}{l}{PM2.5 Data of Five Chinese Cities}\\\\\n",
      "\\midrule\n",
      "Base features ($E$)&1&0.6251\\\\\n",
      "Base features ($E^r$)&14&0.8327\\\\\n",
      "Random groups (2)&7&0.7824\\\\\n",
      "Random groups (3)&4&0.7336\\\\\n",
      "\\midrule\n",
      "\\multicolumn{3}{l}{Superconductivity}\\\\\n",
      "\\midrule\n",
      "Base features ($E$)&1&0.0977\\\\\n",
      "Base features ($E^r$)&81&0.1775\\\\\n",
      "Random groups (2)&40&0.1424\\\\\n",
      "Random groups (3)&27&0.1327\\\\\n",
      "Random groups (5)&16&0.1228\\\\\n",
      "Random groups (6)&13&0.1193\\\\\n",
      "\\midrule\n",
      "\\multicolumn{3}{l}{Productivity Prediction of Garment Employees}\\\\\n",
      "\\midrule\n",
      "Base features ($E$)&1&0.5533\\\\\n",
      "Base features ($E^r$)&14&0.6151\\\\\n",
      "Random groups (2)&7&0.5862\\\\\n",
      "Random groups (3)&4&0.5761\\\\\n"
     ]
    }
   ],
   "source": [
    "# latex translate dict\n",
    "ltd = {\n",
    "    'features_e': 'Base features ($E$)',\n",
    "    'features_er': 'Base features ($E^r$)',\n",
    "    'semantic_groups': 'Semantic groups',\n",
    "    'random_groups_2': 'Random groups (2)',\n",
    "    'random_groups_3': 'Random groups (3)',\n",
    "    'random_groups_5': 'Random groups (5)',\n",
    "    'random_groups_6': 'Random groups (6)',\n",
    "}\n",
    "for name, data_mse in mse_collector.items():\n",
    "    print('\\midrule')\n",
    "    print(\"\\multicolumn{3}{l}{%s}\"%name, end=\"\\\\\\\\\\n\")\n",
    "    print('\\midrule')\n",
    "    for kind, mse in data_mse.items():\n",
    "        if kind == \"base_mse\":\n",
    "            continue\n",
    "        print(ltd[kind], mse['ngroups'], \"%.4f\"%(mse['mse']/data_mse['base_mse']), sep=\"&\", end=\"\\\\\\\\\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3ba568a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = list(ltd.keys())\n",
    "data = [[(mse_dict[l]['mse']/mse_dict['base_mse'] if l in mse_dict else 0) for l in labels] for mse_dict in mse_collector.values()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "325c2b9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbcAAAE0CAYAAACvs32dAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1rUlEQVR4nO3deVxVZf4H8M8F5QKy5QKCIougQrgFkYhWmmHopGmlkz93UEnNhdQyc0EzK4vQSsxdWjG1bcZMJsvdBCRNxVQwLyqmoIKKgcDz+8Pxjle4yL0XOZzHz/v1Oq8ZnnOufD8D49fnnOecoxFCCBAREUnESukCiIiIahqbGxERSYfNjYiIpMPmRkRE0mFzIyIi6bC5ERGRdNjciIhIOmxuREQknXpKF1Ad5eXlOHv2LBwdHaHRaJQuh4iIFCCEwJUrV+Dh4QErq6rnZqpobmfPnoWnp6fSZRARUR2Qk5OD5s2bV3mMKpqbo6MjgJuBnJycFK6GiIiUUFhYCE9PT31PqIoqmtutU5FOTk5sbkRE97nqXJ7ighIiIpIOmxsREUmHzY2IiKTD5kZERNJRxYISstAc5yr2FdReHUREtYQzNyIiko5ZzW3JkiXw8fGBra0tgoODsWPHjiqP/+yzz9C+fXvY29vD3d0dI0aMQH5+vlkFExER3Y3JzS05ORmTJk3CjBkzkJGRga5duyIyMhI6na7S43fu3ImhQ4ciKioKhw8fxldffYXU1FRER0dbXDwREVFlTG5u8fHxiIqKQnR0NAICApCQkABPT08kJiZWevzevXvh7e2NCRMmwMfHB126dMGYMWOQlpZmcfFERESVMam5lZSUID09HREREQbjERER2L17d6Wf6dy5M06fPo1NmzZBCIG//voL69evR+/evY1+n+LiYhQWFhpsRERE1WVSc8vLy0NZWRnc3NwMxt3c3HDu3LlKP9O5c2d89tlnGDhwIGxsbNC0aVO4uLjggw8+MPp9FixYAGdnZ/3GhyYTEZEpzFpQcudzvYQQRp/1deTIEUyYMAGzZs1Ceno6Nm/ejJMnTyImJsbonz99+nQUFBTot5ycHHPKJCKi+5RJ97k1btwY1tbWFWZp58+frzCbu2XBggUIDw/H1KlTAQDt2rVDgwYN0LVrV7zxxhtwd3ev8BmtVgutVmtKaURERHomNTcbGxsEBwcjJSUF/fr104+npKSgb9++lX6mqKgI9eoZfhtra2sAN2d8RHSbqm64B3jTPVE1mXxaMjY2FitWrMCqVauQmZmJyZMnQ6fT6U8zTp8+HUOHDtUf//TTT2Pjxo1ITExEdnY2du3ahQkTJiA0NBQeHh41l4SIiOi/TH781sCBA5Gfn4+5c+ciNzcXQUFB2LRpE7y8vAAAubm5Bve8DR8+HFeuXMGHH36Il19+GS4uLujevTvefvvtmktBRER0G41QwbnBwsJCODs7o6CggC8rNQefLakePC1JZJQpvYDPliQiIumwuRERkXTY3IiISDpsbkREJB02NyIikg7fxE1E9x5X7FIt48yNiIikw+ZGRETSYXMjIiLpsLkREZF02NyIiEg6bG5ERCQdNjciIpIOmxsREUmHzY2IiKTD5kZERNJhcyMiIumwuRERkXTY3IiISDpsbkREJB02NyIikg6bGxERSYfNjYiIpMPmRkRE0jGruS1ZsgQ+Pj6wtbVFcHAwduzYUeXxxcXFmDFjBry8vKDVatGyZUusWrXKrILvqTnOVW9ERKQK9Uz9QHJyMiZNmoQlS5YgPDwcH3/8MSIjI3HkyBG0aNGi0s8MGDAAf/31F1auXAk/Pz+cP38epaWlFhdP915mm4Aq9wcczaylSoiIqs/k5hYfH4+oqChER0cDABISEvDjjz8iMTERCxYsqHD85s2bsW3bNmRnZ6Nhw4YAAG9vb8uqJiIiqoJJpyVLSkqQnp6OiIgIg/GIiAjs3r270s989913CAkJwTvvvINmzZqhVatWmDJlCq5fv270+xQXF6OwsNBgIyIiqi6TZm55eXkoKyuDm5ubwbibmxvOnTtX6Weys7Oxc+dO2Nra4uuvv0ZeXh7Gjh2LixcvGr3utmDBAsTFxZlSGhERkZ5ZC0o0Go3B10KICmO3lJeXQ6PR4LPPPkNoaCh69eqF+Ph4rFmzxujsbfr06SgoKNBvOTk55pRJRET3KZNmbo0bN4a1tXWFWdr58+crzOZucXd3R7NmzeDs/L/VhgEBARBC4PTp0/D396/wGa1WC61Wa0ppREREeiY1NxsbGwQHByMlJQX9+vXTj6ekpKBv376VfiY8PBxfffUVrl69CgcHBwDAsWPHYGVlhebNm1tQOhHdjitbif7H5NWSsbGxGDJkCEJCQhAWFoZly5ZBp9MhJiYGwM1TimfOnEFSUhIAYNCgQZg3bx5GjBiBuLg45OXlYerUqRg5ciTs7OxqNg0RSYdNm8xhcnMbOHAg8vPzMXfuXOTm5iIoKAibNm2Cl5cXACA3Nxc6nU5/vIODA1JSUvDSSy8hJCQEjRo1woABA/DGG2/UXAoiIqLbmNzcAGDs2LEYO3ZspfvWrFlTYaxNmzZISUkx51sRERGZjM+WJCIi6Zg1c6OKeF2AiKju4MyNiIikw+ZGRETSYXMjIiLpsLkREZF02NyIiEg6bG5ERCQdNjciIpIOmxsREUmHzY2IiKTD5kZERNJhcyMiIumwuRERkXTY3IiISDpsbkREJB02NyIikg6bGxERSYfNjYiIpMPmRkRE0mFzIyIi6bC5ERGRdNjciIhIOmxuREQkHbOa25IlS+Dj4wNbW1sEBwdjx44d1frcrl27UK9ePXTo0MGcb0tERFQtJje35ORkTJo0CTNmzEBGRga6du2KyMhI6HS6Kj9XUFCAoUOH4oknnjC7WCIiouowubnFx8cjKioK0dHRCAgIQEJCAjw9PZGYmFjl58aMGYNBgwYhLCzM7GKJiIiqo54pB5eUlCA9PR2vvvqqwXhERAR2795t9HOrV69GVlYWPv30U7zxxht3/T7FxcUoLi7Wf11YWGhKmfcd71f/XeX+P21rqRC6K/6siGqHSc0tLy8PZWVlcHNzMxh3c3PDuXPnKv3M8ePH8eqrr2LHjh2oV696327BggWIi4szpTQiUhCbNtU1Zi0o0Wg0Bl8LISqMAUBZWRkGDRqEuLg4tGrVqtp//vTp01FQUKDfcnJyzCmTiIjuUybN3Bo3bgxra+sKs7Tz589XmM0BwJUrV5CWloaMjAyMHz8eAFBeXg4hBOrVq4ctW7age/fuFT6n1Wqh1WpNKY2IiEjPpOZmY2OD4OBgpKSkoF+/fvrxlJQU9O3bt8LxTk5O+P333w3GlixZgq1bt2L9+vXw8fExs2y6H1R1quvPt3rXYiVEpDYmNTcAiI2NxZAhQxASEoKwsDAsW7YMOp0OMTExAG6eUjxz5gySkpJgZWWFoKAgg8+7urrC1ta2wnht4HUBIqquu/59wX9g1WkmN7eBAwciPz8fc+fORW5uLoKCgrBp0yZ4eXkBAHJzc+96zxsRESnjfmnaJjc3ABg7dizGjh1b6b41a9ZU+dk5c+Zgzpw55nxbIiKiauGzJYmISDpsbkREJB02NyIikg6bGxERSYfNjYiIpMPmRkRE0mFzIyIi6bC5ERGRdNjciIhIOmxuREQkHTY3IiKSDpsbERFJh82NiIikw+ZGRETSYXMjIiLpsLkREZF02NyIiEg6bG5ERCQdNjciIpIOmxsREUmHzY2IiKTD5kZERNJhcyMiIumwuRERkXTMam5LliyBj48PbG1tERwcjB07dhg9duPGjXjyySfRpEkTODk5ISwsDD/++KPZBRMREd2Nyc0tOTkZkyZNwowZM5CRkYGuXbsiMjISOp2u0uO3b9+OJ598Eps2bUJ6ejq6deuGp59+GhkZGRYXT0REVBmTm1t8fDyioqIQHR2NgIAAJCQkwNPTE4mJiZUen5CQgGnTpuHhhx+Gv78/3nzzTfj7++P777+3uHgiIqLKmNTcSkpKkJ6ejoiICIPxiIgI7N69u1p/Rnl5Oa5cuYKGDRua8q2JiIiqrZ4pB+fl5aGsrAxubm4G425ubjh37ly1/oz33nsP165dw4ABA4weU1xcjOLiYv3XhYWFppRJRET3ObMWlGg0GoOvhRAVxirzxRdfYM6cOUhOToarq6vR4xYsWABnZ2f95unpaU6ZRER0nzKpuTVu3BjW1tYVZmnnz5+vMJu7U3JyMqKiorBu3Tr06NGjymOnT5+OgoIC/ZaTk2NKmUREdJ8zqbnZ2NggODgYKSkpBuMpKSno3Lmz0c998cUXGD58OD7//HP07t37rt9Hq9XCycnJYCMiIqouk665AUBsbCyGDBmCkJAQhIWFYdmyZdDpdIiJiQFwc9Z15swZJCUlAbjZ2IYOHYpFixahU6dO+lmfnZ0dnJ2dazAKERHRTSY3t4EDByI/Px9z585Fbm4ugoKCsGnTJnh5eQEAcnNzDe55+/jjj1FaWopx48Zh3Lhx+vFhw4ZhzZo1licgi7Rd27bK/etqqQ4ioppkcnMDgLFjx2Ls2LGV7ruzYf3yyy/mfAsiIiKz8dmSREQkHTY3IiKSDpsbERFJh82NiIikY9aCEiJSRlWrW7myleh/2NyISFG8HYXuBZ6WJCIi6XDmRkR0D1Q1I/192O+1WMn9ic2NiIiq5W6nkOtS0+ZpSSIikg6bGxERSYfNjYiIpMNrbibgPUZEROrAmRsREUmHzY2IiKTD5kZERNJhcyMiIumwuRERkXTY3IiISDpsbkREJB02NyIikg6bGxERSYfNjYiIpMPmRkRE0mFzIyIi6ZjV3JYsWQIfHx/Y2toiODgYO3bsqPL4bdu2ITg4GLa2tvD19cXSpUvNKpaIiKg6TG5uycnJmDRpEmbMmIGMjAx07doVkZGR0Ol0lR5/8uRJ9OrVC127dkVGRgZee+01TJgwARs2bLC4eCIiosqY3Nzi4+MRFRWF6OhoBAQEICEhAZ6enkhMTKz0+KVLl6JFixZISEhAQEAAoqOjMXLkSLz77rsWF09ERFQZk97nVlJSgvT0dLz66qsG4xEREdi9e3eln9mzZw8iIiIMxnr27ImVK1fixo0bqF+/foXPFBcXo7i4WP91QUEBAKCwsNCUcisoLy6qcn+hRlS5v+x6mdF9V8uM7wMsr70qluSqKhNQd3Pdy+97L/F3sKK6+jt410x3+b5V5VLy99eSXHf7Wd3rXLf+fCGq/v/JrYOq7cyZMwKA2LVrl8H4/PnzRatWrSr9jL+/v5g/f77B2K5duwQAcfbs2Uo/M3v2bAGAGzdu3Lhxq7Dl5OTctV+Z9SZujUZj8LUQosLY3Y6vbPyW6dOnIzY2Vv91eXk5Ll68iEaNGlX5fe6lwsJCeHp6IicnB05OTorUUNNkzAQwl5rImAmQM1ddyCSEwJUrV+Dh4XHXY01qbo0bN4a1tTXOnTtnMH7+/Hm4ublV+pmmTZtWeny9evXQqFGjSj+j1Wqh1WoNxlxcXEwp9Z5xcnKS5pf1FhkzAcylJjJmAuTMpXQmZ2fnah1n0oISGxsbBAcHIyUlxWA8JSUFnTt3rvQzYWFhFY7fsmULQkJCKr3eRkREZCmTV0vGxsZixYoVWLVqFTIzMzF58mTodDrExMQAuHlKcejQofrjY2JicOrUKcTGxiIzMxOrVq3CypUrMWXKlJpLQUREdBuTr7kNHDgQ+fn5mDt3LnJzcxEUFIRNmzbBy8sLAJCbm2twz5uPjw82bdqEyZMn46OPPoKHhwcWL16MZ599tuZS1AKtVovZs2dXOF2qZjJmAphLTWTMBMiZS22ZNEJUZ00lERGRevDZkkREJB02NyIikg6bGxERSYfNjYiIpMPmdh+5ceMGunXrhmPHjildChHRPcXmdhcnTpzAjz/+iOvXrwNA9R7YWUfVr18fhw4dUuwRZvfS9evXUVT0vwfCnjp1CgkJCdiyZYuCVRGRUtjcjMjPz0ePHj3QqlUr9OrVC7m5uQCA6OhovPzyywpXZ76hQ4di5cqVSpdR4/r27YukpCQAwOXLl/HII4/gvffeQ9++fY2+jomUU3bHk/5//fVXbN++HTdu3FCoopp16dIlpKam4vTp00qXUmMuX76M5cuXY+bMmVixYoX+bS111l0frXyfGjJkiOjZs6fIyckRDg4OIisrSwghxI8//igCAwMVrs5848ePF05OTuKhhx4So0ePFpMnTzbY1KpRo0bi0KFDQgghli9fLtq1ayfKysrEunXrRJs2bRSuznQlJSVi6tSpomXLluLhhx8Wq1atMth/7tw5YWVlpVB15jt79qwIDw8X1tbW4tFHHxUXL14UvXv3FhqNRmg0GtGqVSujbwupq6ZPny6uXbsmhLj5cxs1apSwsrISGo1GWFlZiX79+onr168rXKXpnn32WbFhwwYhhBCHDx8WjRs3Fk2aNBGPPPKIcHNzE02bNhVHjhxRuErjOHMzYsuWLXj77bfRvHlzg3F/f3+cOnVKoaosd+jQITz00ENwcnLCsWPHkJGRod9+++03pcszW1FRERwdHQHc/Nn1798fVlZW6NSpkyp/XvPnz0dSUhJiYmIQERGByZMnY8yYMQbHCBWeIn/llVcghMDXX38Nd3d3/OMf/0BhYSFycnJw6tQpuLm5Yf78+UqXaZK3334bV69eBQAsXLgQ33zzDb766iucPn0a3377Lfbt24eFCxcqXKXptm3bhrZt2wIApkyZgoiICJw+fRp79+5FTk4OevfujUmTJilbZFWU7q51lYODgzh27Jj+v9+aue3bt080bNhQydKoEm3bthWLFi0SOp1OODk5id27dwshhEhLSxNubm4KV2c6Pz8/8f333+u/PnHihPD39xfDhw8X5eXlqp25ubu7iz179gghhMjPzxcajUb85z//0e/funWr8PX1Vao8s2g0GvHXX38JIYTo0KGDWLlypcH+5ORkERAQoERpFrGzsxMnTpwQQtz8ue3fv99g/x9//CGcnZ0VqKx6OHMz4tFHH9VfwwFuvnuuvLwcCxcuRLdu3RSsrGbItFAGAGbNmoUpU6bA29sboaGhCAsLA3BzFtexY0eFqzPdmTNnEBQUpP+6ZcuW+OWXX7Bnzx4MGTKkwjUrtbh06RKaNWsGAGjYsCHs7e31z6UFbua8dX1bTW4t0srJyUFoaKjBvtDQUFWePWjXrh22bt0K4Oary+7McOrUKdjZ2SlRWrWY9bLS+8HChQvx+OOPIy0tDSUlJZg2bRoOHz6MixcvYteuXUqXZ7b8/HwMGDAAP//8MzQaDY4fPw5fX19ER0fDxcUF7733ntIlmuW5555Dly5dkJubi/bt2+vHn3jiCfTr10/ByszTtGlTZGVlwdvbWz/m4eGBrVu3olu3bhg2bJhyxVnA1dUVubm58PT0BACMHz8eDRs21O+/dOkSGjRooFR5Zlu+fDkcHByg1Wpx6dIlg30FBQWqedjw7WbOnImhQ4eifv36mDBhAiZPnoz8/HwEBATgjz/+wOzZszFkyBClyzRO6aljXZabmytmzZolevfuLSIjI8WMGTNUd7H7TrIulLmdTqer1mvo67KoqCgxcuTISvedPn1a+Pn5qfK0ZJ8+fURCQoLR/R9++KHo3r17LVZkOS8vL+Ht7a3f7sz3/vvvi06dOilUnWXWr18vmjdvrl8gc2uztbUVkyZNEqWlpUqXaBTfCmChsWPHYu7cuWjcuLHSpVRL06ZN8eOPP6J9+/ZwdHTEgQMH4Ovri5MnT6Jt27b6C+NqU1pairi4OCxevFifwcHBAS+99BJmz56tuhfjnjp1CkePHkXPnj0r3Z+bm4stW7aodgZnTGpqKuzs7AxOyard3r17odVqVXl6HLh528b+/fuRnZ2N8vJyuLu7Izg4WL+Aq85SuruqnaOjo372owayLpQZM2aMcHV1FUuXLhUHDhwQBw4cEEuXLhVNmzYVY8aMUbq8e65Xr16qP6tQGRlzyZhJiLqXi9fcLCRUNvG9tVBm3rx5AORZKPPFF1/gyy+/RGRkpH6sXbt2aNGiBf75z39i6dKlClZ3723fvl2/OEgmMuaSMRNQ93Kxud1nZF0oY2tra7D44hZvb2/Y2NjUfkFEpCjeCnCfCQwMxMGDBxEaGoonn3wS165dQ//+/ZGRkYGWLVsqXZ7Zxo0bh3nz5qG4uFg/VlxcjPnz52P8+PEKVkZESuDM7T7UtGlTxMXFVXmM2hbKZGRk4KeffkLz5s31twIcOHAAJSUleOKJJ9C/f3/9sRs3blSqTCKqJWxuVKlPP/0UU6ZMUU1zc3FxwbPPPmswduteKiK6/7C5WWjw4MFwcnJSuowap7aFMqtXr1a6BCKqQ3jNzYjNmzdj586d+q8/+ugjdOjQAYMGDTJ4AkFiYqJqZjckr9dee83gSR+ykDGXjJmAOphL6XsR6qqgoCDx73//WwghxMGDB4VWqxXTp08XjzzyiBg+fLjC1d17t98Dpwbe3t7Cx8fH6KZWa9asEf/617/0X0+dOlU4OzuLsLAw8eeffypYmWVkzCVjJiHUm4vNzYgGDRqIkydPCiGEmD17tnj22WeFEEKkp6er8inzplJbc0tISDDYFi5cKAYNGiQaNmwoFixYoHR5ZmvVqpX46aefhBBC7N69W9jZ2YmPP/5YPP3006Jfv34KV2c+GXPJmEkI9ebiNTcjbGxsUFRUBAD4z3/+g6FDhwK4+STzwsJCJUujSkycOLHS8Y8++ghpaWm1XE3NycnJgZ+fHwDgm2++wXPPPYfRo0cjPDwcjz/+uLLFWUDGXDJmAtSbi9fcjOjSpQtiY2Mxb9487Nu3D7179wYAHDt2rMILTGUky0KZyMhIbNiwQekyzObg4ID8/HwAN1/f06NHDwA3b1qvS0+DMJWMuWTMBKg3F5ubER9++CHq1auH9evXIzExUf8Oqh9++AFPPfWUwtWZ735bKLN+/fq6dZHbRE8++SSio6MRHR2NY8eO6f+Rdfjw4UqfyKIWMuaSMROg3lxsbka0aNEC//rXv3DgwAFERUXpx99//30sXrxYwcosM3XqVP1p1d9//x0vv/wyevXqhezsbMTGxipcnfk6duyIhx56SL917NgR7u7ueO211/Daa68pXZ7ZPvroI4SFheHChQvYsGEDGjVqBABIT0/HCy+8oHB15pMxl4yZAPXm4itvqpCVlYXVq1cjKysLixYtgqurKzZv3gxPT088+OCDSpdnFgcHBxw6dAje3t6YM2cODh06hPXr12P//v3o1asXzp07p3SJZrnziStWVlZo0qQJHn/8cbRp00ahqohIKVxQYsS2bdsQGRmJ8PBwbN++HfPnz4erqysOHjyIFStWYP369UqXaBZZF8rMnj1b6RLumUuXLmHlypXIzMyERqNBmzZtMHLkSFWfbgXkzCVjJkCduThzMyIsLAzPP/88YmNjDV7qmZqaimeeeQZnzpxRukSz9OnTByUlJQgPD8e8efNw8uRJNGvWDFu2bMH48eNx7NgxpUs0W1lZGb755hv9/wEDAwPRp08fWFtbK12a2bZt24Y+ffrA2dkZISEhAG6eDrp8+TK+++47PPbYYwpXaB4Zc8mYCVBxLmXvRKi7GjRoILKzs4UQhvd8nTx5Umi1WiVLs8ipU6dE7969Rbt27cSKFSv045MmTRIvvfSSgpVZ5vjx48Lf31/Y29uLjh07ig4dOgh7e3vRunVrceLECaXLM9uDDz4oRo0aJUpLS/VjpaWlYvTo0eLBBx9UsDLLyJhLxkxCqDcXm5sRzZo1E7t27RJCGDa3jRs3Cl9fXyVLo0pERkaKp556SuTn5+vH8vLyxFNPPSV69eqlYGWWsbW1FUePHq0wfvToUWFra6tARTVDxlwyZhJCvbm4WtKIQYMG4ZVXXsG5c+f0b6vetWsXpkyZor9OpVZZWVl4/fXX8cILL+D8+fMAbt4icPjwYYUrM9+2bdvwzjvvGFwDaNSoEd566y1s27ZNwcos89BDDyEzM7PCeGZmJjp06FD7BdUQGXPJmAlQby4uKDFi/vz5GD58OJo1awYhBAIDA1FWVoZBgwbh9ddfV7o8s8m6UEar1eLKlSsVxq9evarqN3FPmDABEydOxIkTJ9CpUycAwN69e/HRRx/hrbfewsGDB/XHtmvXTqkyTSZjLhkzAerNxQUllRBCQKfToUmTJjh37hz279+P8vJydOzYEf7+/kqXZxFZF8oMHToU+/fvx8qVKxEaGgoA+PXXXzFq1CgEBwdjzZo1yhZoJiurqk+uaDQaCCGg0WhQVlZWS1VZTsZcMmYC1JuLM7dKCCHg7++Pw4cPw9/fH76+vkqXVGN+//13fP755xXGmzRpon/EjhotXrwYw4YNQ1hYGOrXrw8AKC0tRZ8+fbBo0SKFqzPfyZMnlS7hnpAxl4yZAPXmYnOrhJWVFfz9/ZGfn6/6mdqdXFxckJubCx8fH4PxjIwM/SPG1EYIgYKCAnzxxRc4e/YsMjMz9aeSbz3wVa28vLyULuGekDGXjJkA9eZiczPinXfewdSpU5GYmIigoCCly6kxtxbKfPXVV9IslLlzpq32hna7pKSkKver9WcmYy4ZMwHqzcVrbkY88MADKCoqQmlpKWxsbGBnZ2ew/+LFiwpVZpkbN25g+PDh+PLLLyGEQL169fQLZdasWaPaG54ffPBBrFy5Un/BWxYPPPCAwdc3btxAUVERbGxsYG9vr9rfQxlzyZgJUG8uztyMSEhIULqEGieEwNmzZ7F8+XLMmzdPqoUyss60b39Twy3Hjx/Hiy++iKlTpypQUc2QMZeMmQD15uLM7T5SXl4OW1tb/ek7mcg60zYmLS0NgwcPxtGjR5UupUbJmEvGTEDdz8WZmxE6na7K/S1atKilSmqOzAtlZJxpV8Xa2hpnz55VuowaJ2MuGTMBdT8Xm5sR3t7e0Gg0RvfXpfs5TCHr6bthw4YpXcI98d133xl8LYRAbm4uPvzwQ4SHhytUleVkzCVjJkC9uXha0ogDBw4YfH3jxg1kZGQgPj4e8+fPR//+/RWqzDKynr4z9roejUYDrVar2qeU3HkDrUajQZMmTdC9e3e89957cHd3V6gyy8iYS8ZMgHpzsbmZ6N///jcWLlyIX375RelSzLJ27doq96t1BmRlZVXlTLt58+YYPnw4Zs+efdcnLhCR+vG0pIlatWqF1NRUpcswm1qb192sWbMGM2bMwPDhwxEaGgohBFJTU7F27Vq8/vrruHDhAt59911otVq89tprSpdrllv/Dq2qiauRjLlkzASoLFctvHlAlQoKCgy2y5cvi8zMTDFw4EDRvn17pcsz26lTp6rc1Kp79+4iOTm5wnhycrLo3r27EEKIpKQk0bp169ouzWJr164VQUFBQqvVCq1WK9q2bSuSkpKULstiMuaSMZMQ6szFmZsRLi4uFf51IoSAp6cnvvzyS4WqspysC2X27NmDpUuXVhjv2LEj9uzZAwDo0qXLXVfB1jXx8fGYOXMmxo8fj/DwcAghsGvXLsTExCAvLw+TJ09WukSzyJhLxkyAinMp2lrrsF9++cVg2759u8jMzBQ3btxQujSL/PbbbwZbamqqWLZsmWjTpo3YsGGD0uWZzd/fX7zyyisVxl955RXRqlUrIYQQqampwsPDo7ZLs4i3t7dYu3ZthfE1a9YIb29vBSqqGTLmkjGTEOrNxZmbERqNBp07d0a9eob/E5WWlmL79u149NFHFarMMu3bt68wFhISAg8PDyxcuFC1q0DfffddPP/88/jhhx/w8MMPQ6PRIDU1FUePHtW/oy41NRUDBw5UuFLT5ObmonPnzhXGO3fujNzcXAUqqhky5pIxE6DeXFw2ZkS3bt0qXRZfUFCAbt26KVDRvaX2hTJ9+vTBsWPH0KtXL1y8eBF5eXmIjIzE0aNH8Y9//AMA8OKLLyI+Pl7hSk3j5+eHdevWVRhPTk5W9Y34MuaSMROg3lycuRkh/vvyvTvl5+ejQYMGClRUM+68H0z894bMOXPm1Olf1Orw8vLCggULqjxm7NixmDt3Lho3blxLVVkmLi4OAwcOxPbt2xEeHg6NRoOdO3fip59+qvQvHLWQMZeMmQD15uJ9bne4dVru22+/xVNPPQWtVqvfV1ZWhoMHD6J169bYvHmzUiVapLL7wcRtC2XCwsIUqqx2ODk54bffflPVC2j379+P+Ph4g/fUvfzyy+jYsaPSpVlExlwyZgLUmYsztzs4OzsDuPkXvqOjo8ETPGxsbNCpUyeMGjVKqfIs9vPPPxt8bWVlhSZNmsDPz6/C9UUZqenfcjdu3MDo0aMxc+ZMfPrpp0qXU2NkzCVjJkDduThzMyIuLg5TpkxR9SnIymzfvt3oQpndu3erdqFMdTk6OuLAgQOqmbm5uLhg//79qqm3umTMJWMmQL25uKDEiNmzZ0vX2ID7b6GM2vXr1w/ffPON0mXUOBlzyZgJUG8u+c9DWWD9+vVYt24ddDodSkpKDPbt379foaosI+tCGVn5+flh3rx52L17N4KDgyv8jCZMmKBQZZaRMZeMmQD15uJpSSMWL16MGTNmYNiwYVi+fDlGjBiBrKwspKamYty4cZg/f77SJZpE9oUy1aW205I+Pj5G92k0GmRnZ9diNTVHxlwyZgLUm4szNyOWLFmCZcuW4YUXXsDatWsxbdo0+Pr6YtasWap8LYzsC2Wqa/DgwXByclK6jGo7efKk0iXcEzLmkjEToN5cnLkZYW9vj8zMTHh5ecHV1RUpKSlo3749jh8/jk6dOiE/P1/pEs0i60KZ1atXw8HBAc8//7zB+FdffYWioiJp34ZARJXjzM2Ipk2bIj8/H15eXvDy8sLevXvRvn17nDx5UlXLye80e/ZspUu4J956661KH5zs6uqK0aNHq7a5xcbGVjqu0Whga2sLPz8/9O3bFw0bNqzlyiwjYy4ZMwHqzcWZmxHR0dHw9PTE7NmzsXTpUsTGxiI8PBxpaWno378/Vq5cqXSJZpNxoYytrS2OHj0Kb29vg/E///wTAQEBuH79ujKFWahbt27Yv38/ysrK0Lp1awghcPz4cVhbW6NNmzb4448/9E+MCAwMVLrcapMxl4yZAPXm4q0ARixbtgwzZswAAMTExGDNmjUICAhAXFwcEhMTFa7OfIsXL8aIESPg6uqKjIwMhIaGolGjRsjOzkZkZKTS5ZnN1dUVBw8erDB+4MABNGrUSIGKakbfvn3Ro0cPnD17Funp6di/fz/OnDmDJ598Ei+88ALOnDmDRx99tO6+dsQIGXPJmAlQca7aev0A1Q2tW7cWn3/+uRBCCAcHB5GVlSWEEGLmzJli3LhxSpZmkalTpwovLy+xdetWUVpaKkpLS8VPP/0kvLy8xMsvv6x0eWbz8PAQhw8frjB+6NAh/et70tPTRaNGjWq7NIvImEvGTEKoNxdnblXYsWMHBg8ejLCwMJw5cwYA8Mknn2Dnzp0KV2Y+nU6nf32FnZ0drly5AgAYMmQIvvjiCyVLs8gbb7yBRx55BE888QTs7OxgZ2eHiIgIdO/eHW+++abS5ZmtoKAA58+frzB+4cIF/UOwXVxcKpxerutkzCVjJkC9udjcjNiwYQN69uwJOzs7ZGRkoLi4GABw5coVVf9leWuhDAD9QhkAql8oY2Njg+TkZBw9ehSfffYZNm7ciKysLKxatQo2NjZKl2e2vn37YuTIkfj6669x+vRpnDlzBl9//TWioqLwzDPPAAD27duHVq1aKVuoiWTMJWMmQMW5lJ461lUdOnTQv3329tN3GRkZws3NTcnSLBIVFSXmzJkjhBAiMTFR2NnZiR49eggXFxcxcuRIhaujO125ckVER0cLGxsbYWVlJaysrISNjY0YNWqUuHr1qhDi5u9kRkaGsoWaSMZcMmYSQr25uFrSCHt7exw5cgTe3t4GT7XIzs5GYGAg/v77b6VLNEt5eTnKy8v1D05et24ddu7cCT8/P8TExKhqlhMbG4t58+ahQYMGRpcr36K2l5Te6erVq8jOzoYQAi1btoSDg4PB/tOnT8PDwwNWVuo6GSNjLhkzAerLxfvcjHB3d8eJEycqLC3fuXOnah7dVBkrKyuDX74BAwZgwIABClZkvoyMDNy4cUP/32Xm4OCAdu3aGd0fGBiouvfUAXLmkjEToL5cbG5GjBkzBhMnTsSqVaug0Whw9uxZ7NmzB1OmTMGsWbOULs8iO3bswMcff4ysrCysX78ezZo1wyeffAIfHx906dJF6fKq7fZ30935nrr7jawnYGTMJWMmoO7lqhvzxzri4MGDKC8vBwBMmzYNzzzzDLp164arV6/i0UcfRXR0NMaMGYPx48crXKn5ZF0oM3LkSP3Kz9tdu3YNI0eOVKAiIlISm9ttOnbsiLy8PACAr68vYmNjceHCBezbtw979+7FhQsXMG/ePIWrtMwbb7yBpUuXYvny5ahfv75+vHPnzqp9OgkArF27ttKnkFy/fh1JSUkKVERESuJpydu4uLjg5MmTcHV1xZ9//ony8nI0aNAAISEhSpdWY/74449K37bt5OSEy5cv135BFiosLIQQAkIIXLlyBba2tvp9ZWVl2LRpE1xdXRWskIiUwOZ2m2effRaPPfYY3N3dodFoEBISAmtr60qPravvMLob2RbKuLi4QKPRQKPRVHqfjUajQVxcnAKV1a7KXkArAxlzyZgJqHu52Nxus2zZMvTv3x8nTpzAhAkTMGrUKDg6OipdVo2SbaHMzz//DCEEunfvjg0bNhg8mdzGxgZeXl7w8PBQsMLaUdcu5tcUGXPJmAmoe7l4n5sRI0aMwOLFi6VobgcPHkRQUJD+FoAZM2bg/fff19+rp9VqMWXKFFVfTzx16hQ8PT3rzD02tS0nJwceHh5GzzSolYy5ZMwE1L1cbG73AWtra+Tm5sLV1RW+vr5ITU2Fra0tMjMzUV5ejsDAwAo3ZKrR5cuXsW/fPpw/f16/6vWWoUOHKlSVZf7++2988MEH+PnnnyvNpdZFQDLmkjEToN5cPC15H7gfFsp8//33+L//+z9cu3YNjo6OBuf/NRqNapvbyJEjkZKSgueeew6hoaF17rqGuWTMJWMmQL25OHO7D4wePRpJSUlwd3eHTqdD8+bNpVso06pVK/Tq1Qtvvvkm7O3tlS6nxjg7O2PTpk0IDw9XupQaJWMuGTMB6s3Fmdt94H5YKHPmzBlMmDBBqsYGAM2aNZPuZwXImUvGTICKc9XiQ5qpDhg+fLgoLCxUuowa169fP5GcnKx0GTVu06ZN4qmnnhJ//vmn0qXUKBlzyZhJCPXm4sztPrN69WqlS7gnevfujalTp+LIkSNo27atwdNXAKBPnz4KVWaZkJAQ/P333/D19YW9vX2FXBcvXlSoMsvImEvGTIB6c/GaG0mhqlsANBoNysrKarGamtOjRw/odDpERUXBzc2twsX8YcOGKVSZZWTMJWMmQL252NyI6jB7e3vs2bMH7du3V7qUGiVjLhkzAerNdX/e8UqkEm3atKn0gdBqJ2MuGTMB6s3FmRtJ49q1a9i2bRt0Oh1KSkoM9k2YMEGhqiyzZcsWxMXFYf78+ZVeS3RyclKoMsvImEvGTIB6c7G5kRQyMjLQq1cvFBUV4dq1a2jYsCHy8vJgb28PV1dX1d6/d+ta4p3XOYQQqr6WKGMuGTMB6s3F1ZIkhcmTJ+Ppp59GYmIiXFxcsHfvXtSvXx+DBw/GxIkTlS7PbLK+YVzGXDJmAtSbizM3koKLiwt+/fVXtG7dGi4uLtizZw8CAgLw66+/YtiwYTh69KjSJRJRLeLMjaRQv359/WkTNzc36HQ6BAQEwNnZGTqdTuHqLHP58mWsXLkSmZmZ0Gg0CAwMxMiRI+Hs7Kx0aRaRMZeMmQB15uLMjaQQERGB4cOHY9CgQYiJiUFGRgYmTJiATz75BJcuXcKvv/6qdIlmSUtLQ8+ePWFnZ4fQ0FAIIZCWlobr169jy5YteOihh5Qu0Swy5pIxE6DeXGxuJIW0tDRcuXIF3bp1w4ULFzBs2DDs3LkTfn5+WL16teru0bmla9eu8PPzw/Lly1Gv3s0TLaWlpYiOjkZ2dja2b9+ucIXmkTGXjJkA9eZicyOqw+zs7JCRkYE2bdoYjB85cgQhISEoKipSqDLLyJhLxkyAenPxJm6iOszJyanSa4Y5OTnqfFL7f8mYS8ZMgHpzsbmRFPLz8zFu3DgEBgaicePGaNiwocGmVgMHDkRUVBSSk5ORk5OD06dP48svv0R0dDReeOEFpcszm4y5ZMwEqDcXV0uSFAYPHoysrCyjD3dVq3fffVf/JvHS0lIAN1eGvvjii3jrrbcUrs58MuaSMROg3ly85kZScHR0xM6dO1W7cORuioqKkJWVBSEE/Pz8pHkpq4y5ZMwEqC8XmxtJ4eGHH8YHH3yATp06KV0KEdUBbG4khdTUVLz66quYNWsWgoKCVPNw18r079+/2sdu3LjxHlZSs2TMJWMmQI5cvOZGUnBxcUFBQQG6d+9uMF7XH+5amduf+iCEwNdffw1nZ2eEhIQAANLT03H58mWT/gKqC2TMJWMmQI5cnLmRFEJDQ1GvXj1MnDix0gUljz32mEKVWeaVV17BxYsXsXTpUlhbWwMAysrKMHbsWDg5OWHhwoUKV2geGXPJmAlQby42N5KCvb09MjIy0Lp1a6VLqVFNmjTBzp07K+T6448/0LlzZ+Tn5ytUmWVkzCVjJkC9uXifG0khJCQEOTk5SpdR40pLS5GZmVlhPDMzE+Xl5QpUVDNkzCVjJkC9uXjNjaTw0ksvYeLEiZg6dWqlbwtu166dQpVZZsSIERg5ciROnDihXwm6d+9evPXWWxgxYoTC1ZlPxlwyZgJUnEsQSUCj0VTYrKys9P+pVmVlZeLtt98WHh4e+lweHh7i7bffFqWlpUqXZzYZc8mYSQj15uI1N5LCqVOnqtzv5eVVS5XcO4WFhQDUdVtDdciYS8ZMgLpysbkREZF0uKCEpPHJJ58gPDwcHh4e+plcQkICvv32W4UrM99ff/2FIUOGwMPDA/Xq1YO1tbXBplYy5pIxE6DeXFxQQlJITEzErFmzMGnSJMyfP19/07aLiwsSEhLQt29fhSs0z/Dhw6HT6TBz5ky4u7tL80BoGXPJmAlQby6eliQpBAYG4s0338QzzzwDR0dHHDhwAL6+vjh06BAef/xx5OXlKV2iWRwdHbFjxw506NBB6VJqlIy5ZMwEqDcXT0uSFE6ePImOHTtWGNdqtbh27ZoCFdUMT09PyPjvTxlzyZgJUG8uNjeSgo+PD3777bcK4z/88AMCAwNrv6AakpCQgFdffRV//vmn0qXUKBlzyZgJUG8unpYkKaxevRozZ87Ee++9h6ioKKxYsQJZWVlYsGABVqxYgX/+859Kl2iWBx54AEVFRSgtLYW9vX2Fm9MvXryoUGWWkTGXjJkA9ebighKSwogRI1BaWopp06ahqKgIgwYNQvPmzbFo0SLVNjbg5r+aZSRjLhkzAerNxZkbSeH69esQQsDe3h55eXnIzs7Grl27EBgYiJ49eypdHhHVMjY3kkJERAT69++PmJgYXL58GW3atEH9+vWRl5eH+Ph4vPjii0qXaLHr16/jxo0bBmNqeFLE3ciYS8ZMgLpycUEJSWH//v3o2rUrAGD9+vVwc3PDqVOnkJSUhMWLFytcnfmuXbuG8ePHw9XVFQ4ODnjggQcMNrWSMZeMmQD15mJzIykUFRXB0dERALBlyxb0798fVlZW6NSp012fO1mXTZs2DVu3bsWSJUug1WqxYsUKxMXFwcPDA0lJSUqXZzYZc8mYCVBxrlp/VDPRPdC2bVuxaNEiodPphJOTk9i9e7cQQoi0tDTh5uamcHXm8/T0FD///LMQQghHR0dx/PhxIYQQSUlJIjIyUsHKLCNjLhkzCaHeXJy5kRRmzZqFKVOmwNvbG4888gjCwsIA3JzFVXZzt1pcvHgRPj4+AG5e27i17LpLly7Yvn27kqVZRMZcMmYC1JuLzY2k8Nxzz0Gn0yEtLQ2bN2/Wjz/xxBN4//33FazMMr6+vvqbZwMDA7Fu3ToAwPfffw8XFxflCrOQjLlkzASoOJfSU0ciMi4+Pl4sWrRICCHE1q1bhZ2dnbCxsRFWVlYiISFB4erMJ2MuGTMJod5cvBWAqI66ceMGIiIi8PHHH6NVq1YAoJ+dtmzZEu3bt1e4QvPImEvGTIC6c/EJJUR1VP369XHo0CGDV4y0aNECLVq0ULAqy8mYS8ZMgLpz8ZobUR02dOhQrFy5UukyapyMuWTMBKg3F2duRHVYSUkJVqxYgZSUFISEhKBBgwYG++Pj4xWqzDIy5pIxE6DeXLzmRlSHdevWzeg+jUaDrVu31mI1NUfGXDJmAtSbi82NiIikw2tuREQkHTY3IiKSDpsbERFJh82NiIikw+ZGRETSYXMjIiLpsLkREZF02NyIiEg6/w+3XEAg0j9yAwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 500x200 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5,2))\n",
    "#labels = [\"a\",\"b\",\"c\"]\n",
    "#data = [[1,2,3],[3,2,1],[5,6,7],[1,1,4]]\n",
    "slack = 3\n",
    "width = 1/(len(labels)*(len(data)+slack))\n",
    "pos_0 = np.linspace(0,1,len(data[0])+1)[:-1]\n",
    "for i,d in enumerate(data):\n",
    "    plt.bar(pos_0+i*width, d, width=width)\n",
    "plt.xticks(pos_0+((len(data)-1)*width/2), labels, rotation = 90);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a050a9c",
   "metadata": {},
   "source": [
    "# Contributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "07d50c52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------\n",
      "Dataset: Communities and Crime Unnormalized\n",
      "Calculating random group 1 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4664945f0e2646349fbabac447ebb427",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 2 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4079aec696da4204be6b01a425b22ae4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 3 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "903c96e076f141df90de8dd8e093184b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 5 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7828a1e790cb46d7a144699b796a2a59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 6 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4aee2e815ea948fb95b83208229837ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------\n",
      "Dataset: PM2.5 Data of Five Chinese Cities\n",
      "Calculating random group 1 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b11dc35ce72473fb1933bd2380242e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 2 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae9cebd3423f417f8200259ddb3d66d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 3 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ccaf76718234ff49ea9f2274d43c8cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------\n",
      "Dataset: Superconductivity\n",
      "Calculating random group 1 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea60325e277e4d238495182d441c82ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 2 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b0e39e468a34959b283a774c5bc828d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 3 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e44cdd6f65b545ce89de30c2166b1beb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 5 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "640975b6dc7c44c580c11a35731a078a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 6 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38adfd31fb704950862f99014dafe814",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------\n",
      "Dataset: Productivity Prediction of Garment Employees\n",
      "Calculating random group 1 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa511c9a29ac48ec8181af50765e28e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 2 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8737ac3267a840bab869858322fa4acd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating random group 3 E^r mse...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7271c717607744b6b2b65c42b0ff89e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mse_collector = {}\n",
    "\n",
    "for ReaderType, target in zip(readerTypes, targets):\n",
    "    reader = ReaderType('data/UCI/archive.ics.uci.edu/ml/machine-learning-databases')\n",
    "    reader.readAndCache()\n",
    "    print(\"---------\")\n",
    "    print(\"Dataset:\", reader.name)\n",
    "    cols = reader.cols\n",
    "    ncols = len(cols)\n",
    "\n",
    "    trainer = utils.Trainer(\n",
    "        cache_dir='models_clean',\n",
    "        default_params={\n",
    "            'objective': 'regression',\n",
    "            'metric': 'mse',\n",
    "            'boosting': 'gbdt',\n",
    "            'num_leaves': 31,\n",
    "            'learning_rate': 0.05,\n",
    "            'verbose': -1,\n",
    "            'device': 'cpu',\n",
    "            'lambda_l2': 0.003,\n",
    "            'early_stopping_round': 10,\n",
    "            'num_boost_round': 100,\n",
    "        },\n",
    "        nocache_params={\n",
    "            'num_threads': 20,\n",
    "        }\n",
    "    )\n",
    "    evaluator = utils.Evaluator(reader, target)\n",
    "    \n",
    "    data_mse = {}\n",
    "\n",
    "    for groupsize in [1,2,3,5,6]:\n",
    "        np.random.seed(reader.seed)\n",
    "        ngroups = len(cols)//groupsize\n",
    "        if ngroups <= 2:\n",
    "            continue\n",
    "        mses_e = []\n",
    "        mses_er = []\n",
    "        print(\"Calculating random group\", groupsize, \"E^r mse...\")\n",
    "        for i in tqdm(range(100)):\n",
    "            contribs_e = [evaluator.base_mse]\n",
    "            contribs_er = [evaluator.base_mse]\n",
    "            feature_perm = np.random.permutation(cols)\n",
    "            groups = np.array_split(feature_perm, ngroups)[::-1] # array split puts longer groups first, lets switch the order\n",
    "            groups_sofar = []\n",
    "            for g in groups:\n",
    "                groups_sofar.append(g)\n",
    "                trainData = utils.TrainingData(reader, target, columns=[i for g in groups_sofar for i in g])\n",
    "                \n",
    "                booster = trainer.train_model(trainData)\n",
    "                new_mse = evaluator.get_model_mse(booster)\n",
    "                contribs_e.append(new_mse - np.sum(contribs_e))\n",
    "                \n",
    "                booster = trainer.train_model(trainData, {\n",
    "                    'interaction_constraints': get_group_constraints(groups_sofar),\n",
    "                })\n",
    "                new_mse = evaluator.get_model_mse(booster)\n",
    "                contribs_er.append(new_mse - np.sum(contribs_er))\n",
    "                \n",
    "                if len(contribs_e) > 5:\n",
    "                    break\n",
    "            mses_e.append(contribs_e)\n",
    "            mses_er.append(contribs_er)\n",
    "        data_mse['random_groups_'+str(groupsize)] = {\n",
    "            'ngroups': ngroups,\n",
    "            'mses_e':np.array(mses_e).mean(axis=0),\n",
    "            'mses_er':np.array(mses_er).mean(axis=0),\n",
    "            'base_mse':evaluator.base_mse,\n",
    "        }\n",
    "    mse_collector[reader.name] = data_mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "0ca8cb80",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Communities and Crime Unnormalized\n",
      "random_groups_1\n",
      "[ 1.         -0.13581962 -0.08875594 -0.06440981 -0.06515439 -0.04183157]\n",
      "[ 1.         -0.13581962 -0.07642613 -0.04481189 -0.04739945 -0.0311251 ]\n",
      "random_groups_2\n",
      "[ 1.         -0.22457556 -0.12956155 -0.07881013 -0.04517915 -0.02978859]\n",
      "[ 1.         -0.22457556 -0.09316917 -0.06551522 -0.03657607 -0.02333432]\n",
      "random_groups_3\n",
      "[ 1.         -0.28898537 -0.14396553 -0.06835585 -0.03172062 -0.02056061]\n",
      "[ 1.         -0.28898537 -0.11219175 -0.05635408 -0.02855884 -0.02293215]\n",
      "random_groups_5\n",
      "[ 1.         -0.39597951 -0.1119512  -0.04566263 -0.02381191 -0.01540892]\n",
      "[ 1.         -0.39597951 -0.08902798 -0.03812489 -0.01701975 -0.01808468]\n",
      "random_groups_6\n",
      "[ 1.         -0.43268542 -0.10032866 -0.03642931 -0.01943242 -0.01332475]\n",
      "[ 1.         -0.43268542 -0.07983357 -0.0371828  -0.01468369 -0.01199517]\n",
      "PM2.5 Data of Five Chinese Cities\n",
      "random_groups_1\n",
      "[ 1.00000000e+00 -7.18138161e-04 -2.94985703e-02 -3.70319403e-02\n",
      " -5.12673413e-02 -5.06445057e-02]\n",
      "[ 1.00000000e+00 -7.18138161e-04 -1.56089196e-02 -1.26814316e-02\n",
      " -1.70604560e-02 -1.86081602e-02]\n",
      "random_groups_2\n",
      "[ 1.         -0.03021671 -0.08829927 -0.08660095 -0.0680027  -0.04601854]\n",
      "[ 1.         -0.03021671 -0.04494142 -0.04026754 -0.03286226 -0.02795517]\n",
      "random_groups_3\n",
      "[ 1.         -0.06724835 -0.13786941 -0.11402131 -0.05572701]\n",
      "[ 1.         -0.06724835 -0.07913392 -0.07379036 -0.04620792]\n",
      "Superconductivity\n",
      "random_groups_1\n",
      "[ 1.         -0.5045679  -0.20332096 -0.07466269 -0.03264497 -0.01861451]\n",
      "[ 1.         -0.5045679  -0.1277895  -0.05814092 -0.02677624 -0.01931214]\n",
      "random_groups_2\n",
      "[ 1.         -0.70789127 -0.10730837 -0.0304007  -0.01467616 -0.00868043]\n",
      "[ 1.         -0.70789127 -0.06802198 -0.02481484 -0.01077722 -0.00716009]\n",
      "random_groups_3\n",
      "[ 1.         -0.78255627 -0.0630431  -0.01909125 -0.00986001 -0.0062097 ]\n",
      "[ 1.         -0.78255627 -0.03914256 -0.01266324 -0.00773704 -0.00526427]\n",
      "random_groups_5\n",
      "[ 1.         -0.83380903 -0.03514531 -0.01180783 -0.00654961 -0.0039611 ]\n",
      "[ 1.         -0.83380903 -0.02036745 -0.00735598 -0.00499194 -0.00243341]\n",
      "random_groups_6\n",
      "[ 1.         -0.84560213 -0.02895204 -0.01031223 -0.00588251 -0.00340335]\n",
      "[ 1.         -0.84560213 -0.01665104 -0.00670754 -0.00358785 -0.00209669]\n",
      "Productivity Prediction of Garment Employees\n",
      "random_groups_1\n",
      "[ 1.         -0.05226961 -0.06977801 -0.06416187 -0.07032881 -0.03735998]\n",
      "[ 1.         -0.05226961 -0.05027451 -0.04403773 -0.04674913 -0.02997602]\n",
      "random_groups_2\n",
      "[ 1.         -0.12204762 -0.13449149 -0.08116702 -0.05942362 -0.03652705]\n",
      "[ 1.         -0.12204762 -0.10825361 -0.06012342 -0.05135466 -0.02794101]\n",
      "random_groups_3\n",
      "[ 1.         -0.18620949 -0.15150359 -0.09593701 -0.01312057]\n",
      "[ 1.         -0.18620949 -0.11712233 -0.07957884 -0.0410292 ]\n"
     ]
    }
   ],
   "source": [
    "for name, group_setups in mse_collector.items():\n",
    "    print(name)\n",
    "    for group, mse_info in group_setups.items():\n",
    "        print(group)\n",
    "        print(mse_info['mses_e']/mse_info['base_mse'])\n",
    "        print(mse_info['mses_er']/mse_info['base_mse'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "1042ccc1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grouping&$\\phi_1$&$\\phi_2$&$\\phi_3$&$\\phi_4$&$\\phi_5$\\\\\n",
      "\\midrule\n",
      "\\multicolumn{6}{l}{Communities and Crime Unnormalized}\\\\\n",
      "\\midrule\n",
      "Individual features ($E$) & -0.1358 & -0.0888 & -0.0644 & -0.0652 & -0.0418\\\\\n",
      "Individual features ($E^r$) & -0.1358 & -0.0764 & -0.0448 & -0.0474 & -0.0311\\\\\n",
      "Random groups of size 2 ($E$) & -0.2246 & -0.1296 & -0.0788 & -0.0452 & -0.0298\\\\\n",
      "Random groups of size 2 ($E^r$) & -0.2246 & -0.0932 & -0.0655 & -0.0366 & -0.0233\\\\\n",
      "Random groups of size 3 ($E$) & -0.2890 & -0.1440 & -0.0684 & -0.0317 & -0.0206\\\\\n",
      "Random groups of size 3 ($E^r$) & -0.2890 & -0.1122 & -0.0564 & -0.0286 & -0.0229\\\\\n",
      "Random groups of size 5 ($E$) & -0.3960 & -0.1120 & -0.0457 & -0.0238 & -0.0154\\\\\n",
      "Random groups of size 5 ($E^r$) & -0.3960 & -0.0890 & -0.0381 & -0.0170 & -0.0181\\\\\n",
      "Random groups of size 6 ($E$) & -0.4327 & -0.1003 & -0.0364 & -0.0194 & -0.0133\\\\\n",
      "Random groups of size 6 ($E^r$) & -0.4327 & -0.0798 & -0.0372 & -0.0147 & -0.0120\\\\\n",
      "\\midrule\n",
      "\\multicolumn{6}{l}{PM2.5 Data of Five Chinese Cities}\\\\\n",
      "\\midrule\n",
      "Individual features ($E$) & -0.0007 & -0.0295 & -0.0370 & -0.0513 & -0.0506\\\\\n",
      "Individual features ($E^r$) & -0.0007 & -0.0156 & -0.0127 & -0.0171 & -0.0186\\\\\n",
      "Random groups of size 2 ($E$) & -0.0302 & -0.0883 & -0.0866 & -0.0680 & -0.0460\\\\\n",
      "Random groups of size 2 ($E^r$) & -0.0302 & -0.0449 & -0.0403 & -0.0329 & -0.0280\\\\\n",
      "Random groups of size 3 ($E$) & -0.0672 & -0.1379 & -0.1140 & -0.0557\\\\\n",
      "Random groups of size 3 ($E^r$) & -0.0672 & -0.0791 & -0.0738 & -0.0462\\\\\n",
      "\\midrule\n",
      "\\multicolumn{6}{l}{Superconductivity}\\\\\n",
      "\\midrule\n",
      "Individual features ($E$) & -0.5046 & -0.2033 & -0.0747 & -0.0326 & -0.0186\\\\\n",
      "Individual features ($E^r$) & -0.5046 & -0.1278 & -0.0581 & -0.0268 & -0.0193\\\\\n",
      "Random groups of size 2 ($E$) & -0.7079 & -0.1073 & -0.0304 & -0.0147 & -0.0087\\\\\n",
      "Random groups of size 2 ($E^r$) & -0.7079 & -0.0680 & -0.0248 & -0.0108 & -0.0072\\\\\n",
      "Random groups of size 3 ($E$) & -0.7826 & -0.0630 & -0.0191 & -0.0099 & -0.0062\\\\\n",
      "Random groups of size 3 ($E^r$) & -0.7826 & -0.0391 & -0.0127 & -0.0077 & -0.0053\\\\\n",
      "Random groups of size 5 ($E$) & -0.8338 & -0.0351 & -0.0118 & -0.0065 & -0.0040\\\\\n",
      "Random groups of size 5 ($E^r$) & -0.8338 & -0.0204 & -0.0074 & -0.0050 & -0.0024\\\\\n",
      "Random groups of size 6 ($E$) & -0.8456 & -0.0290 & -0.0103 & -0.0059 & -0.0034\\\\\n",
      "Random groups of size 6 ($E^r$) & -0.8456 & -0.0167 & -0.0067 & -0.0036 & -0.0021\\\\\n",
      "\\midrule\n",
      "\\multicolumn{6}{l}{Productivity Prediction of Garment Employees}\\\\\n",
      "\\midrule\n",
      "Individual features ($E$) & -0.0523 & -0.0698 & -0.0642 & -0.0703 & -0.0374\\\\\n",
      "Individual features ($E^r$) & -0.0523 & -0.0503 & -0.0440 & -0.0467 & -0.0300\\\\\n",
      "Random groups of size 2 ($E$) & -0.1220 & -0.1345 & -0.0812 & -0.0594 & -0.0365\\\\\n",
      "Random groups of size 2 ($E^r$) & -0.1220 & -0.1083 & -0.0601 & -0.0514 & -0.0279\\\\\n",
      "Random groups of size 3 ($E$) & -0.1862 & -0.1515 & -0.0959 & -0.0131\\\\\n",
      "Random groups of size 3 ($E^r$) & -0.1862 & -0.1171 & -0.0796 & -0.0410\\\\\n"
     ]
    }
   ],
   "source": [
    "# latex translate dict\n",
    "ltd = {\n",
    "    'random_groups_1': 'Individual features',\n",
    "    'random_groups_2': 'Random groups of size 2',\n",
    "    'random_groups_3': 'Random groups of size 3',\n",
    "    'random_groups_5': 'Random groups of size 5',\n",
    "    'random_groups_6': 'Random groups of size 6',\n",
    "}\n",
    "print(\"Grouping\", *[\"$\\phi_%d$\"%i for i in range(1,6)], sep=\"&\", end=\"\\\\\\\\\\n\")\n",
    "for name, group_setups in mse_collector.items():\n",
    "    print('\\midrule')\n",
    "    print(\"\\multicolumn{6}{l}{%s}\"%name, end=\"\\\\\\\\\\n\")\n",
    "    print('\\midrule')\n",
    "    for group, mse_info in group_setups.items():\n",
    "        print(ltd[group]+\" ($E$)\", *[\"%.4f\"%f for f in mse_info['mses_e'][1:]/mse_info['base_mse']], sep=\" & \", end=\"\\\\\\\\\\n\")\n",
    "        print(ltd[group]+\" ($E^r$)\", *[\"%.4f\"%f for f in mse_info['mses_er'][1:]/mse_info['base_mse']], sep=\" & \", end=\"\\\\\\\\\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "45dae2e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'random_groups_1': {'ngroups': 124,\n",
       "  'mses_e': array([306935.62065916, 265247.72746687, 238005.36866657, 218235.70327279,\n",
       "         198237.50100627, 185397.90328788]),\n",
       "  'mses_er': array([306935.62065916, 265247.72746687, 241789.82453004, 228035.45950236,\n",
       "         213486.88135603, 203933.47955831]),\n",
       "  'base_mse': 306935.6206591561},\n",
       " 'random_groups_2': {'ngroups': 62,\n",
       "  'mses_e': array([306935.62065916, 238005.36866657, 198238.31292433, 174048.67654813,\n",
       "         160181.58577769, 151038.40720213]),\n",
       "  'mses_er': array([306935.62065916, 238005.36866657, 209408.43270914, 189299.47943938,\n",
       "         178072.97997092, 170910.84507696]),\n",
       "  'base_mse': 306935.6206591561},\n",
       " 'random_groups_3': {'ngroups': 41,\n",
       "  'mses_e': array([306935.62065916, 218235.70327279, 174047.55452482, 153066.7080982 ,\n",
       "         143330.52051213, 137019.73700659]),\n",
       "  'mses_er': array([306935.62065916, 218235.70327279, 183800.05904047, 166502.9846103 ,\n",
       "         157737.25976544, 150698.56650277]),\n",
       "  'base_mse': 306935.6206591561},\n",
       " 'random_groups_5': {'ngroups': 24,\n",
       "  'mses_e': array([306935.62065916, 185395.38889446, 151033.57922368, 137018.09172626,\n",
       "         129709.36888472, 124979.82284863]),\n",
       "  'mses_er': array([306935.62065916, 185395.38889446, 158069.5301603 , 146367.6435164 ,\n",
       "         141143.67583933, 135592.84364348]),\n",
       "  'base_mse': 306935.6206591561},\n",
       " 'random_groups_6': {'ngroups': 20,\n",
       "  'mses_e': array([306935.62065916, 174129.03835951, 143334.5989014 , 132153.14698693,\n",
       "         126188.64430359, 122098.80403846]),\n",
       "  'mses_er': array([306935.62065916, 174129.03835951, 149625.27046909, 138212.54467234,\n",
       "         133705.59837986, 130023.8519817 ]),\n",
       "  'base_mse': 306935.6206591561}}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for name, group_setups in mse_collector.items():\n",
    "    print(name)\n",
    "    for group, mse_info in group_setups.items():\n",
    "        print(group)\n",
    "        print(mse_info['mses_e']/mse_info['base_mse'])\n",
    "        print(mse_info['mses_er']/mse_info['base_mse'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f5263c1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "notebook_module": {
   "alias": "notebook_module",
   "code": "",
   "compatibility": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}