Stacking-MIR2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import pickle\n",
    "import os\n",
    "\n",
    "from numpy import exp, sqrt, dot\n",
    "from scipy.spatial.distance import cdist\n",
    "from scipy.io import loadmat\n",
    "from random import shuffle\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import scale\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.gaussian_process import GaussianProcessRegressor\n",
    "\n",
    "from scipy.stats import kurtosis\n",
    "from scipy.stats import moment\n",
    "from scipy.stats import skew\n",
    "\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kernel mean embedding stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Kernel(object):\n",
    "    \"\"\" Kernel mean embedding class\n",
    "        Kernel class from Zoltan Szabo\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, par=None):\n",
    "        \"\"\" Initialization.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        par : dictionary, optional\n",
    "              Name of the kernel and its parameters (default is\n",
    "              {\"name\": \"RBF\", \"sigma\": 1}). The name of the kernel comes\n",
    "              from \"RBF\", \"exponential\", \"Cauchy\", \"student\", \"Matern3p2\",\n",
    "              \"Matern5p2\", \"polynomial\", \"ratquadr\" (rational quadratic),\n",
    "              \"invmquadr\" (inverse multiquadr).\n",
    "        \"\"\"\n",
    "        if par is None:\n",
    "            par = {\"name\": \"RBF\", \"sigma\": 1}\n",
    "\n",
    "        name = par[\"name\"]\n",
    "        self.name = name\n",
    "\n",
    "        # other attributes:\n",
    "        if name == \"RBF\" or name == \"exponential\" or name == \"Cauchy\":\n",
    "            self.sigma = par[\"sigma\"]\n",
    "        elif name == \"student\":\n",
    "            self.d = par[\"d\"]\n",
    "        elif name == \"Matern3p2\" or name == \"Matern5p2\":\n",
    "            self.l = par[\"l\"]\n",
    "        elif name == \"polynomial\":\n",
    "            self.c = par[\"c\"]\n",
    "            self.exponent = par[\"exponent\"]\n",
    "        elif name == \"ratquadr\" or name == \"invmquadr\":\n",
    "            self.c = par[\"c\"]\n",
    "        else:\n",
    "            raise Exception(\"kernel=?\")\n",
    "\n",
    "    def gram_matrix(self, y1, y2):\n",
    "        \"\"\"  Compute the Gram matrix = [k(y1[i,:], y2[j,:])]; i, j: running.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        y1 : (number of samples1, dimension)-ndarray\n",
    "             One row of y1 corresponds to one sample.\n",
    "        y2 : (number of samples2, dimension)-ndarray\n",
    "             One row of y2 corresponds to one sample.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        g : ndarray.\n",
    "            Gram matrix of y1 and y2.\n",
    "        \"\"\"\n",
    "\n",
    "        if self.name == \"RBF\":\n",
    "            sigma = self.sigma\n",
    "            g = cdist(y1, y2)\n",
    "            g = exp(-g ** 2 / (2 * sigma ** 2))\n",
    "        elif self.name == \"exponential\":\n",
    "            sigma = self.sigma\n",
    "            g = cdist(y1, y2)\n",
    "            g = exp(-g / (2 * sigma ** 2))\n",
    "        elif self.name == \"Cauchy\":\n",
    "            sigma = self.sigma\n",
    "            g = cdist(y1, y2)\n",
    "            g = 1 / (1 + g ** 2 / sigma ** 2)\n",
    "        elif self.name == \"student\":\n",
    "            d = self.d\n",
    "            g = cdist(y1, y2)\n",
    "            g = 1 / (1 + g ** d)\n",
    "        elif self.name == \"Matern3p2\":\n",
    "            l = self.l\n",
    "            g = cdist(y1, y2) \n",
    "            g = (1 + sqrt(3) * g / l) * exp(-sqrt(3) * g / l)\n",
    "        elif self.name == \"Matern5p2\":\n",
    "            l = self.l\n",
    "            g = cdist(y1, y2)\n",
    "            g = (1 + sqrt(5) * g / l + 5 * g ** 2 / (3 * l ** 2)) * \\\n",
    "                exp(-sqrt(5) * g / l)\n",
    "        elif self.name == \"polynomial\":\n",
    "            c = self.c\n",
    "            exponent = self.exponent\n",
    "            g = (dot(y1, y2.T) + c) ** exponent\n",
    "        elif self.name == \"ratquadr\":\n",
    "            c = self.c\n",
    "            g = cdist(y1, y2) ** 2\n",
    "            g = 1 - g / (g + c)\n",
    "        elif self.name == \"invmquadr\":\n",
    "            c = self.c\n",
    "            g = cdist(y1, y2)\n",
    "            g = 1 / sqrt(g ** 2 + c ** 2)\n",
    "        else:\n",
    "            raise Exception(\"kernel=?\")\n",
    "\n",
    "        return g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Compute the linear kernel product of \n",
    "# the mean embedding of X1 and X2\n",
    "# denoted as K(i, j) above\n",
    "def mean_embedding(X1, X2, kernel):\n",
    "    k = Kernel(kernel)\n",
    "    gram_mat = k.gram_matrix(X1, X2)\n",
    "    # Number of instances in the bag\n",
    "    N = float(gram_mat.shape[0])\n",
    "    mu_X1_X2 = gram_mat.ravel().sum() / N**2\n",
    "    return (mu_X1_X2)\n",
    "\n",
    "# Return a symmetrised matrix\n",
    "def symmetrise(A):\n",
    "    return(A + A.T - np.diag(A.diagonal()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Compute the Gram matrix K given the kernel and \n",
    "# the smoothing parameter theta\n",
    "def compute_gram_train(df, kernel, theta):\n",
    "    col_feature = [col for col in df.columns if col != 'id']\n",
    "    nb_bag = df[\"id\"].nunique()\n",
    "    K_matrix = np.zeros((nb_bag, nb_bag))\n",
    "    \n",
    "    print(\"Computing {0} Gram matrix for theta={1}:\".format(kernel, theta))\n",
    "    for i in range(nb_bag):\n",
    "        #if (i%50 == 0):\n",
    "            #print(\"Bag number: {0}\". format(i))\n",
    "\n",
    "        for j in range(i+1):\n",
    "            bag_i = df['id'].unique()[i]\n",
    "            bag_j = df['id'].unique()[j]\n",
    "    \n",
    "            X1 = df.loc[df[\"id\"] == bag_i, col_feature].values\n",
    "            X2 = df.loc[df[\"id\"] == bag_j, col_feature].values\n",
    "\n",
    "            K_matrix[i,j] = mean_embedding(X1, X2, {'name': kernel, 'c': theta})\n",
    "            \n",
    "    return symmetrise(K_matrix)\n",
    "\n",
    "def compute_gram_test(df_train, df_test, kernel, theta):\n",
    "    col_feature = [col for col in df_train.columns if col != 'id']\n",
    "    nb_bag_train = df_train[\"id\"].nunique()\n",
    "    nb_bag_test = df_test[\"id\"].nunique()\n",
    "    K_matrix = np.zeros((nb_bag_train, nb_bag_test))\n",
    "    \n",
    "    for i in range(len(df_train['id'].unique())):\n",
    "        #if (i%50 == 0):\n",
    "            #print(\"Bag number: {0}\". format(i))\n",
    "        \n",
    "        for j in range(len(df_test['id'].unique())):\n",
    "            bag_i = df_train['id'].unique()[i]\n",
    "            bag_j = df_test['id'].unique()[j]\n",
    "            \n",
    "            X1 = df_train.loc[df_train[\"id\"] == bag_i, col_feature].values\n",
    "            X2 = df_test.loc[df_test[\"id\"] == bag_j, col_feature].values\n",
    "            \n",
    "            K_matrix[i,j] = mean_embedding(X1, X2, {'name': kernel, 'c': theta})\n",
    "        \n",
    "    return K_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Class for kernel ridge regression\n",
    "class RidgeRegression(object):\n",
    "    def __init__(self, l2_reg):\n",
    "        self.l2_reg = l2_reg\n",
    "\n",
    "    def fit(self, G, y):\n",
    "        # Train size\n",
    "        n_train = G.shape[0]\n",
    "        ridge_mat = G + (self.l2_reg * n_train) * np.identity(n_train)\n",
    "        self.ridge_mat = ridge_mat\n",
    "        # Shape of y_train is (1, n_train)\n",
    "        self.y_train = y\n",
    "\n",
    "    def predict(self, G):\n",
    "        y_test_hat = self.y_train.dot(np.linalg.solve(self.ridge_mat, G))\n",
    "        return y_test_hat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# moments stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_quantile(series, percentile):\n",
    "    result = series.quantile([percentile])\n",
    " \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def moments_df(full_data, n_moments, quantile=False):\n",
    "    \n",
    "    full_data.columns = ['meta_feature', 'id']\n",
    "    list_operation = ['np.mean', 'np.var', 'skew', 'kurtosis']\n",
    "    list_quantile = [0.25, 0.5, 0.75]\n",
    "\n",
    "    dic_df = {}\n",
    "\n",
    "    for moments in range(1, n_moments+1):\n",
    "        if moments < 5:\n",
    "            dic_df[moments] = eval(\"np.stack(full_data.groupby('id').apply(lambda group: \" + \n",
    "                                   list_operation[moments-1] + \"(group)).values)[:,0]\")\n",
    "        else:\n",
    "            dic_df[moments] = np.stack(full_data.groupby('id').apply(lambda group: moment(group, moment=moments)))[:,0]\n",
    "            \n",
    "    if quantile:\n",
    "        count = 1\n",
    "        for quant in list_quantile:\n",
    "            dic_df[n_moments+count] = full_data.groupby('id').apply(lambda group: get_quantile(group, quant)).iloc[:,0].values\n",
    "            count += 1    \n",
    "        n_moments += len(list_quantile)\n",
    "\n",
    "    array_data = np.reshape(dic_df[1], (dic_df[1].shape[0], 1))\n",
    "    \n",
    "    if n_moments > 1:\n",
    "        for key in range(2, n_moments+1):\n",
    "            array_data = np.hstack([array_data, np.reshape(dic_df[key], (dic_df[key].shape[0], 1))])   \n",
    "\n",
    "    df = pd.DataFrame(array_data)\n",
    "#     if dataset != \"CORN\" and dataset != \"WHEAT\":\n",
    "#          df = pd.concat([df, full_data.groupby('id').mean()[constant_columns].reset_index()], axis=1)\n",
    "    df['id'] = full_data['id'].unique()\n",
    "    df.columns = ['moment_'+str(index) for index in range(1,n_moments+1)] + ['id']\n",
    "    df = df.set_index('id')\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset = \"CORN\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_data():\n",
    "    \n",
    "    if dataset == \"CORN\" or dataset == \"WHEAT\":\n",
    "        full_data = pickle.load(open(dataset + \".p\", \"rb\"))\n",
    "        columns = ([\"id\"] +  ['label'] + [\"reflectance_\" + str(i) for i in range(92)])\n",
    "        full_data.columns = columns\n",
    "        \n",
    "    else:\n",
    "        mat_dict = loadmat('../MIR/' + dataset + '.mat')\n",
    "        full_data = pd.DataFrame(mat_dict[dataset])\n",
    "\n",
    "        # Rename columns to something more interpretable\n",
    "        columns = ([\"id\"] + [\"reflectance_\" + str(i) for i in range(7)]\n",
    "                   + [\"solar_\" + str(i) for i in range(5)] + ['label'])\n",
    "        full_data.columns = columns\n",
    "\n",
    "        if dataset == \"MISR2\":\n",
    "            full_data = full_data[full_data['id'].isin(list(full_data['id'].value_counts().index[full_data['id'].value_counts() == 100]))]\n",
    "\n",
    "    return full_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Algorithm (instance-MIR 2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def instance_stacking(l2=0.1, kernel_parameter=1):\n",
    "\n",
    "    full_data = load_data()\n",
    "    random_seed_list = list(range(10))\n",
    "    \n",
    "    final_loss_train, final_loss_test = [], []\n",
    "    count = 1\n",
    "    \n",
    "    for index in range(10):\n",
    "        kf = KFold(n_splits=5, shuffle=True, random_state=random_seed_list[index])\n",
    "        kf2 = KFold(n_splits=50, shuffle=True)\n",
    "        cols_exclude = [\"id\", \"label\"]\n",
    "        features = [col for col in list(full_data.columns) if col not in cols_exclude]\n",
    "        list_loss_train, list_loss_test = [], []\n",
    "\n",
    "        for train_index, test_index in kf.split(list(full_data['id'].unique())):\n",
    "            train_index, test_index = np.array(full_data['id'].unique())[list(train_index)], np.array(full_data['id'].unique())[list(test_index)]\n",
    "\n",
    "            fold_number = 0\n",
    "            dic_val_x, dic_val_y = {}, {}\n",
    "            for train_index2, val_index in kf2.split(list(train_index)):\n",
    "\n",
    "                train_index2 = train_index[list(train_index2)]\n",
    "                val_index = train_index[list(val_index)]\n",
    "                \n",
    "                train = full_data[full_data['id'].apply(lambda value: value in train_index2)]\n",
    "                val = full_data[full_data['id'].apply(lambda value: value in val_index)]\n",
    "                test = full_data[full_data['id'].apply(lambda value: value in test_index)]\n",
    "            \n",
    "                scaler = StandardScaler()\n",
    "                scaler.fit(train[features])\n",
    "                train[features], val[features], test[features] = scaler.transform(train[features]), scaler.transform(val[features]), scaler.transform(test[features])\n",
    "\n",
    "                train_x, train_y = train[features], train['label']\n",
    "                val_x, val_y = val[features], val['label']\n",
    "                test_x, test_y = test[features], test['label']\n",
    "                \n",
    "                #mlp = MLPRegressor(hidden_layer_sizes=(128,), learning_rate_init=0.001, max_iter=100, alpha=0.05)\n",
    "                mlp = MLPRegressor(hidden_layer_sizes=(512,), learning_rate_init=0.001, max_iter=75, alpha=1)\n",
    "                mlp.fit(train_x, train_y)\n",
    "                train_pred = mlp.predict(train_x)\n",
    "                val_pred = mlp.predict(val_x)\n",
    "                test_pred = mlp.predict(test_x)\n",
    "            \n",
    "                df_val_pred = pd.DataFrame(np.concatenate([np.reshape(val_pred, (val_pred.shape[0],1)), \n",
    "                         np.reshape(val['id'].values, (val_pred.shape[0],1))], axis=1))\n",
    "    \n",
    "                df_test_pred = pd.DataFrame(np.concatenate([np.reshape(test_pred, (test_pred.shape[0],1)), \n",
    "                         np.reshape(test['id'].values, (test_pred.shape[0],1))], axis=1))\n",
    "            \n",
    "                # truth\n",
    "                true_val_y = val.groupby(['id']).mean()['label'].values\n",
    "                true_test_y = test.groupby(['id']).mean()['label'].values\n",
    "                \n",
    "                dic_val_x[fold_number], dic_val_y[fold_number] = df_val_pred, true_val_y\n",
    "                fold_number += 1\n",
    "            \n",
    "            arr_val_y = np.concatenate([dic_val_y[x] for x in dic_val_y])\n",
    "            df_val_x = pd.DataFrame(np.concatenate([dic_val_x[x] for x in dic_val_x]))\n",
    "            df_val_x.columns, df_test_pred.columns = ['feature', 'id'], ['feature', 'id']\n",
    "            \n",
    "################################################################################################################            \n",
    "            # kernel mean embedding + krr stuff\n",
    "            kernel_embedded_matrix_train = compute_gram_train(df_val_x, 'invmquadr', kernel_parameter)\n",
    "            kernel_embedded_matrix_test = compute_gram_test(df_val_x, df_test_pred, 'invmquadr', kernel_parameter)\n",
    "            krr = RidgeRegression(l2_reg=l2)\n",
    "            krr.fit(kernel_embedded_matrix_train, arr_val_y)\n",
    "            y_val_pred = krr.predict(kernel_embedded_matrix_train)\n",
    "            y_test_pred = krr.predict(kernel_embedded_matrix_test)\n",
    "################################################################################################################\n",
    "            \n",
    "#             df_val_x = moments_df(df_val_x, 3, True)\n",
    "#             df_test_pred = moments_df(df_test_pred, 3, True)\n",
    "#             scaler = StandardScaler()\n",
    "#             scaler.fit(df_val_x)\n",
    "#             df_val_x, df_test_pred = scaler.transform(df_val_x), scaler.transform(df_test_pred)\n",
    "           \n",
    "#             regr = Ridge(alpha = 0.01)\n",
    "#             regr.fit(df_val_x, arr_val_y)\n",
    "#             y_val_pred = regr.predict(df_val_x)\n",
    "#             y_test_pred = regr.predict(df_test_pred)\n",
    "            \n",
    "            training_loss = np.sqrt(mean_squared_error(np.reshape(y_val_pred,(y_val_pred.shape[0],1)), \n",
    "                                                           np.reshape(arr_val_y, (y_val_pred.shape[0],1))))\n",
    "\n",
    "            testing_loss = np.sqrt(mean_squared_error(np.reshape(y_test_pred,(y_test_pred.shape[0],1)), \n",
    "                                                          np.reshape(true_test_y, (y_test_pred.shape[0],1))))\n",
    " \n",
    "            list_loss_train.append(training_loss)\n",
    "            list_loss_test.append(testing_loss)\n",
    "        \n",
    "            print(training_loss, testing_loss)\n",
    "            \n",
    "        final_loss_train.append(np.mean(list_loss_train))\n",
    "        final_loss_test.append(np.mean(list_loss_test))\n",
    "        print('Iteration number ' + str(count) + ' is done')\n",
    "        count += 1\n",
    "\n",
    "        print('The training loss is ' + str(np.mean(final_loss_train)))\n",
    "        print('The val loss is ' + str(np.mean(final_loss_test)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FOR L2=1e-08 AND PARAM=100\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.237874048842333 23.23341972581816\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.9479348057113 26.157321713565533\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.981932960411307 27.078980075564754\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.83734083837694 24.75818797324095\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "25.048375810811937 22.250640376579774\n",
      "Iteration number 1 is done\n",
      "The training loss is 24.21069169283076\n",
      "The val loss is 24.695709972953836\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.409911545943615 25.696468302269977\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.914025885186756 25.27968670748308\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.491426743687942 27.41497849815688\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "25.04801744343425 23.737501595685572\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.840972761755307 22.519360884915585\n",
      "Iteration number 2 is done\n",
      "The training loss is 24.175781284416168\n",
      "The val loss is 24.812654585328026\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.06753062668844 24.22790166546789\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.651189788229672 25.79158587162528\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "25.284821221944398 21.953105081490826\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.749722088035345 24.68789213229712\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.477664615712808 24.238219731449615\n",
      "Iteration number 3 is done\n",
      "The training loss is 24.199249412318153\n",
      "The val loss is 24.601683355707404\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.95285680421278 25.46032828059583\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "25.015205050810177 22.906559171017598\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.124220838300506 24.97177852471841\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.285612290078195 25.032162160496068\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.178594893507558 23.93328602863205\n",
      "Iteration number 4 is done\n",
      "The training loss is 24.177261553084076\n",
      "The val loss is 24.56646822505355\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.12008306340347 25.90726384418128\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.68871096569343 25.19042182370245\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.43281431011403 23.501193643455053\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.410471558256067 24.55619096137224\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.61553952599677 22.986515076314507\n",
      "Iteration number 5 is done\n",
      "The training loss is 24.192514019405813\n",
      "The val loss is 24.538837994003863\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.177886476658703 25.274806140924774\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.40100730724284 21.63756253379766\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.224781367341144 25.704732203405214\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.349082059379704 23.85571790851577\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.168990429248062 25.285321619069148\n",
      "Iteration number 6 is done\n",
      "The training loss is 24.20448660416719\n",
      "The val loss is 24.507636341860305\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.78030640368186 21.990800529148025\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.733173879629497 23.855830575518702\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.386205496274258 24.70268013775632\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "22.997208673946663 27.61288164538053\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.996090705399016 22.712784199418568\n",
      "Iteration number 7 is done\n",
      "The training loss is 24.229359522398486\n",
      "The val loss is 24.460116209800894\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "23.937982533763172 22.335851501518704\n",
      "Computing invmquadr Gram matrix for theta=100:\n",
      "24.27355572981974 24.058423979308085\n",
      "Computing invmquadr Gram matrix for theta=100:\n"
     ]
    }
   ],
   "source": [
    "for l2 in [1/10**8]:\n",
    "    for kernel_param in [100]:\n",
    "        print(\"FOR L2=\" + str(l2) + \" AND PARAM=\" + str(kernel_param))\n",
    "        instance_stacking(l2, kernel_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}