2.1 Main Experiment - Inter Subject Dependence Learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f10f819d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Subset, Dataset\n",
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import Subset\n",
    "import torch\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn as nn\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import csv\n",
    "from sklearn import svm\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "import sklearn\n",
    "from sklearn import datasets\n",
    "\n",
    "from torchvision.utils import save_image, make_grid\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42cd3aff",
   "metadata": {},
   "source": [
    "  # 1. Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d14f311c",
   "metadata": {},
   "outputs": [],
   "source": [
    "subject = np.load('../run_experiments_ssl_signal/subject.npy')\n",
    "label = np.load('../run_experiments_ssl_signal/label.npy')\n",
    "new_label = np.load('../run_experiments_ssl_signal/new_label.npy')\n",
    "eeg_data = np.load('../run_experiments_ssl_signal/eeg_data.npy')\n",
    "emg_data = np.load('../run_experiments_ssl_signal/emg_data.npy')\n",
    "\n",
    "subject = torch.from_numpy(subject).long()\n",
    "label = torch.from_numpy(label).long()\n",
    "new_label = torch.from_numpy(new_label).long()\n",
    "eeg_data = torch.from_numpy(eeg_data).float()\n",
    "emg_data = torch.from_numpy(emg_data).float()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6076cfe4",
   "metadata": {},
   "source": [
    "  # 2. Define Networks, Costs, Adaptive Filters for Gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a75fc780",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "# Define network for classification:\n",
    "class ComplexClassifier(nn.Module):\n",
    "    def __init__(self, dim_features=128, num_classes = 10):\n",
    "        super(ComplexClassifier, self).__init__()\n",
    "        self.fc1 = nn.Linear(dim_features, 256)\n",
    "        self.bn1 = nn.BatchNorm1d(256)\n",
    "        self.fc2 = nn.Linear(256, 512)\n",
    "        self.bn2 = nn.BatchNorm1d(512)\n",
    "        self.fc3 = nn.Linear(512, 256)\n",
    "        self.bn3 = nn.BatchNorm1d(256)\n",
    "        self.fc4 = nn.Linear(256, num_classes)  # CIFAR-10 has 10 classes\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = torch.relu(self.bn1(self.fc1(x)))\n",
    "        x = torch.relu(self.bn2(self.fc2(x)))\n",
    "        x = torch.relu(self.bn3(self.fc3(x)))\n",
    "        x = self.fc4(x)  # No activation, CrossEntropyLoss includes softmax\n",
    "        return x\n",
    "\n",
    "# Define network for temporal network: \n",
    "class NETWORK_F_MLP(nn.Module):\n",
    "    def __init__(self, input_dim = 784, HIDDEN = 200, out_dim = 200, how_many_layers = 2):\n",
    "        super(NETWORK_F_MLP, self).__init__()\n",
    "        self.dim = out_dim\n",
    "        self.many_layer = how_many_layers\n",
    "        \n",
    "        self.fc_list = []\n",
    "        self.bn_list = []\n",
    "        \n",
    "        self.fc_list.append(nn.Linear(input_dim, HIDDEN, bias=True))\n",
    "        self.bn_list.append(nn.BatchNorm1d(HIDDEN))\n",
    "\n",
    "        for i in range(0, self.many_layer-1):\n",
    "            self.fc_list.append(nn.Linear(HIDDEN, HIDDEN, bias=True))\n",
    "            self.bn_list.append(nn.BatchNorm1d(HIDDEN))\n",
    "            \n",
    "        self.fc_list = nn.ModuleList(self.fc_list)\n",
    "        self.bn_list = nn.ModuleList(self.bn_list)\n",
    "\n",
    "        self.fc_final = nn.Linear(HIDDEN, out_dim, bias=True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        x = x.reshape(x.shape[0], -1)\n",
    "        \n",
    "        for i in range(0, self.many_layer):\n",
    "            x = self.fc_list[i](x)\n",
    "            x = torch.relu(x)\n",
    "            x = self.bn_list[i](x)\n",
    "        \n",
    "        x = self.fc_final(x)\n",
    "        x = torch.sigmoid(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "# Define network for channel network: \n",
    "class Advanced1DCNN_channel(nn.Module):\n",
    "    def __init__(self, input_channel=1, num_classes=100, input_size=4000, num_channel=60):\n",
    "        super(Advanced1DCNN_channel, self).__init__()\n",
    "        \n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv1d(1, 32, kernel_size=11, padding=5),\n",
    "            nn.BatchNorm1d(32),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(kernel_size=4, stride=4)\n",
    "        )\n",
    "        \n",
    "        self.conv2 = nn.Sequential(\n",
    "            nn.Conv1d(32, 64, kernel_size=11, padding=5),\n",
    "            nn.BatchNorm1d(64),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(kernel_size=4, stride=4)\n",
    "        )\n",
    "\n",
    "        self.conv3 = nn.Sequential(\n",
    "            nn.Conv1d(64, 128, kernel_size=11, padding=5),\n",
    "            nn.BatchNorm1d(128),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(kernel_size=4, stride=4)\n",
    "        )\n",
    "\n",
    "        self.conv4 = nn.Sequential(\n",
    "            nn.Conv1d(128, 256, kernel_size=11, padding=5),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(kernel_size=4, stride=4)\n",
    "        )\n",
    "\n",
    "        self.fc1 = nn.Sequential(\n",
    "            nn.Linear(256 * 15, 1024),\n",
    "            nn.BatchNorm1d(1024),\n",
    "            nn.ReLU(),\n",
    "        )\n",
    "\n",
    "        self.fc2 = nn.Sequential(\n",
    "            nn.Linear(1024, 512),\n",
    "            nn.BatchNorm1d(512),\n",
    "            nn.ReLU(),\n",
    "        )\n",
    "\n",
    "        self.fc3 = nn.Linear(512, num_classes)\n",
    "        \n",
    "        self.MLP = NETWORK_F_MLP(input_dim = 128*input_channel, HIDDEN = 4000, out_dim = num_classes, how_many_layers = 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        bs, channel = x.shape[0], x.shape[1]\n",
    "        x = x.unsqueeze(2)\n",
    "        x = x.flatten(0, 1)\n",
    "        out = self.conv1(x)\n",
    "        out = self.conv2(out)\n",
    "        out = self.conv3(out)\n",
    "        out = self.conv4(out)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        out = self.fc1(out)\n",
    "        out = self.fc2(out)\n",
    "        out = self.fc3(out)\n",
    "        out = torch.sigmoid(out)\n",
    "        out = out.reshape(bs, channel, -1)\n",
    "        out = out.flatten(-2, -1)\n",
    "        out = self.MLP(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "# Construct the trace cost:\n",
    "def return_cost_trace(RFG, track_cov_estimate_final):\n",
    "\n",
    "    RF_E = track_cov_estimate_final[:128, :128]\n",
    "    RG_E = track_cov_estimate_final[128:, 128:]\n",
    "    P_E = track_cov_estimate_final[:128, 128:]\n",
    "\n",
    "    RF_EI = torch.inverse(RF_E)\n",
    "    RG_EI = torch.inverse(RG_E)\n",
    "\n",
    "    RF = RFG[:128, :128]\n",
    "    RG = RFG[128:, 128:]\n",
    "    P = RFG[:128, 128:]\n",
    "\n",
    "    COST = -RF_EI@RF@RF_EI@P_E@RG_EI@P_E.T \\\n",
    "            + RF_EI@P@RG_EI@P_E.T \\\n",
    "            - RF_EI@P_E@RG_EI@RG@RG_EI@P_E.T \\\n",
    "            + RF_EI@P_E@RG_EI@P.T\n",
    "\n",
    "    return -torch.trace(COST)\n",
    "\n",
    "# Define adaptive smoothing filters for estimating ACFs and CCFs\n",
    "def adaptive_estimation(v_t, beta, square_term, i):\n",
    "    v_t = beta*v_t + (1-beta)*square_term.detach()\n",
    "    return v_t, (v_t/(1-beta**i))\n",
    "\n",
    "# Define the loss function (returns log determinant costs)\n",
    "def MCA_LOSS_GIVEN_R(RP, track_cov, i, dim):\n",
    "    cov = RP + torch.eye((RP.shape[0])).cuda()*(.000001)\n",
    "    track_cov, cov_estimate = adaptive_estimation(track_cov, 0.5, cov, i)\n",
    "\n",
    "    cov_estimate_f = cov_estimate[:dim, :dim]\n",
    "    cov_f = cov[:dim, :dim]\n",
    "\n",
    "    cov_estimate_g = cov_estimate[dim:, dim:]\n",
    "    cov_g = cov[dim:, dim:]\n",
    "\n",
    "    LOSS = (torch.linalg.inv(cov_estimate)*cov).sum() - (torch.linalg.inv(cov_estimate_f)*cov_f).sum() -(torch.linalg.inv(cov_estimate_g)*cov_g).sum()\n",
    "    return track_cov, cov_estimate, LOSS\n",
    "\n",
    "def create_folder_if_not_exists(folder_path):\n",
    "    # Check if the folder exists\n",
    "    if not os.path.exists(folder_path):\n",
    "        # If the folder does not exist, create it\n",
    "        os.makedirs(folder_path)\n",
    "        print(f\"Folder '{folder_path}' created.\")\n",
    "    else:\n",
    "        print(f\"Folder '{folder_path}' already exists.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3a253dd",
   "metadata": {},
   "source": [
    "# 3. Run Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "59270b6b",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder './save_exp_inter_subj/' already exists.\n",
      "Iteration 100 Trace Cost 66.80906677246094 Classifer Cost (Trained Separately) 1.0191929340362549\n",
      "Iteration 200 Trace Cost 67.44324493408203 Classifer Cost (Trained Separately) 0.699934184551239\n",
      "Iteration 300 Trace Cost 66.84186553955078 Classifer Cost (Trained Separately) 0.7164157032966614\n",
      "Iteration 400 Trace Cost 67.8227310180664 Classifer Cost (Trained Separately) 0.5167750716209412\n",
      "Iteration 500 Trace Cost 68.00826263427734 Classifer Cost (Trained Separately) 0.4972356855869293\n",
      "Test Accuracy of Primary Movements: 0.6960013384641124\n",
      "Iteration 600 Trace Cost 67.86121368408203 Classifer Cost (Trained Separately) 0.4184565842151642\n",
      "Iteration 700 Trace Cost 67.99005889892578 Classifer Cost (Trained Separately) 0.38334381580352783\n",
      "Iteration 800 Trace Cost 68.77568054199219 Classifer Cost (Trained Separately) 0.3162824213504791\n",
      "Iteration 900 Trace Cost 68.3990249633789 Classifer Cost (Trained Separately) 0.3273831903934479\n",
      "Iteration 1000 Trace Cost 68.90419006347656 Classifer Cost (Trained Separately) 0.4686950445175171\n",
      "Test Accuracy of Primary Movements: 0.6458089342479505\n",
      "Iteration 1100 Trace Cost 68.87386322021484 Classifer Cost (Trained Separately) 0.21134743094444275\n",
      "Iteration 1200 Trace Cost 69.34857940673828 Classifer Cost (Trained Separately) 0.25435787439346313\n",
      "Iteration 1300 Trace Cost 69.51739501953125 Classifer Cost (Trained Separately) 0.3428570032119751\n",
      "Iteration 1400 Trace Cost 69.50138854980469 Classifer Cost (Trained Separately) 0.3225536346435547\n",
      "Iteration 1500 Trace Cost 69.42351531982422 Classifer Cost (Trained Separately) 0.262648344039917\n",
      "Test Accuracy of Primary Movements: 0.8340304500585578\n",
      "Iteration 1600 Trace Cost 69.46710205078125 Classifer Cost (Trained Separately) 0.21982668340206146\n",
      "Iteration 1700 Trace Cost 69.68278503417969 Classifer Cost (Trained Separately) 0.2501554489135742\n",
      "Iteration 1800 Trace Cost 70.05521392822266 Classifer Cost (Trained Separately) 0.2659931480884552\n",
      "Iteration 1900 Trace Cost 70.03067016601562 Classifer Cost (Trained Separately) 0.2845747768878937\n",
      "Iteration 2000 Trace Cost 70.07656860351562 Classifer Cost (Trained Separately) 0.1772652268409729\n",
      "Test Accuracy of Primary Movements: 0.8531035636606994\n",
      "Iteration 2100 Trace Cost 70.6051025390625 Classifer Cost (Trained Separately) 0.16714666783809662\n",
      "Iteration 2200 Trace Cost 70.01368713378906 Classifer Cost (Trained Separately) 0.22489669919013977\n",
      "Iteration 2300 Trace Cost 70.70735168457031 Classifer Cost (Trained Separately) 0.2023211121559143\n",
      "Iteration 2400 Trace Cost 70.318603515625 Classifer Cost (Trained Separately) 0.152475968003273\n",
      "Iteration 2500 Trace Cost 70.51840209960938 Classifer Cost (Trained Separately) 0.17217302322387695\n",
      "Test Accuracy of Primary Movements: 0.8653170486866321\n",
      "Iteration 2600 Trace Cost 70.52308654785156 Classifer Cost (Trained Separately) 0.16760139167308807\n",
      "Iteration 2700 Trace Cost 70.74063110351562 Classifer Cost (Trained Separately) 0.10306181013584137\n",
      "Iteration 2800 Trace Cost 70.68128967285156 Classifer Cost (Trained Separately) 0.15695704519748688\n",
      "Iteration 2900 Trace Cost 71.06710052490234 Classifer Cost (Trained Separately) 0.24982205033302307\n",
      "Iteration 3000 Trace Cost 71.79855346679688 Classifer Cost (Trained Separately) 0.17340883612632751\n",
      "Test Accuracy of Primary Movements: 0.8822151581060733\n",
      "Iteration 3100 Trace Cost 70.82514953613281 Classifer Cost (Trained Separately) 0.1676609367132187\n",
      "Iteration 3200 Trace Cost 71.48342895507812 Classifer Cost (Trained Separately) 0.13051481544971466\n",
      "Iteration 3300 Trace Cost 71.6098861694336 Classifer Cost (Trained Separately) 0.18409381806850433\n",
      "Iteration 3400 Trace Cost 71.78160095214844 Classifer Cost (Trained Separately) 0.0985836312174797\n",
      "Iteration 3500 Trace Cost 71.31495666503906 Classifer Cost (Trained Separately) 0.039545778185129166\n",
      "Test Accuracy of Primary Movements: 0.8900786347666053\n",
      "Iteration 3600 Trace Cost 71.2064208984375 Classifer Cost (Trained Separately) 0.1735718697309494\n",
      "Iteration 3700 Trace Cost 71.43980407714844 Classifer Cost (Trained Separately) 0.12639746069908142\n",
      "Iteration 3800 Trace Cost 71.50411987304688 Classifer Cost (Trained Separately) 0.08639399707317352\n",
      "Iteration 3900 Trace Cost 71.87066650390625 Classifer Cost (Trained Separately) 0.10242906212806702\n",
      "Iteration 4000 Trace Cost 71.84307861328125 Classifer Cost (Trained Separately) 0.09668929874897003\n",
      "Test Accuracy of Primary Movements: 0.9049690480174001\n",
      "Iteration 4100 Trace Cost 72.14430236816406 Classifer Cost (Trained Separately) 0.082498699426651\n",
      "Iteration 4200 Trace Cost 72.44184875488281 Classifer Cost (Trained Separately) 0.057380251586437225\n",
      "Iteration 4300 Trace Cost 72.07022857666016 Classifer Cost (Trained Separately) 0.11447964608669281\n",
      "Iteration 4400 Trace Cost 72.11181640625 Classifer Cost (Trained Separately) 0.12846192717552185\n",
      "Iteration 4500 Trace Cost 72.64924621582031 Classifer Cost (Trained Separately) 0.06069232150912285\n",
      "Test Accuracy of Primary Movements: 0.9145056048184708\n",
      "Iteration 4600 Trace Cost 72.97822570800781 Classifer Cost (Trained Separately) 0.09043948352336884\n",
      "Iteration 4700 Trace Cost 73.39712524414062 Classifer Cost (Trained Separately) 0.060621194541454315\n",
      "Iteration 4800 Trace Cost 72.39757537841797 Classifer Cost (Trained Separately) 0.11382163316011429\n",
      "Iteration 4900 Trace Cost 72.78752136230469 Classifer Cost (Trained Separately) 0.11869853734970093\n",
      "Iteration 5000 Trace Cost 73.68827056884766 Classifer Cost (Trained Separately) 0.026559555903077126\n",
      "Test Accuracy of Primary Movements: 0.9175171490714406\n",
      "Iteration 5100 Trace Cost 72.86478424072266 Classifer Cost (Trained Separately) 0.13187521696090698\n",
      "Iteration 5200 Trace Cost 72.87873077392578 Classifer Cost (Trained Separately) 0.07942859828472137\n",
      "Iteration 5300 Trace Cost 73.24476623535156 Classifer Cost (Trained Separately) 0.047810498625040054\n",
      "Iteration 5400 Trace Cost 72.5842056274414 Classifer Cost (Trained Separately) 0.06100701913237572\n",
      "Iteration 5500 Trace Cost 73.22844696044922 Classifer Cost (Trained Separately) 0.0590037927031517\n",
      "Test Accuracy of Primary Movements: 0.9268863978584574\n",
      "Iteration 5600 Trace Cost 73.8505859375 Classifer Cost (Trained Separately) 0.06280013918876648\n",
      "Iteration 5700 Trace Cost 74.12335205078125 Classifer Cost (Trained Separately) 0.1163276731967926\n",
      "Iteration 5800 Trace Cost 73.51370239257812 Classifer Cost (Trained Separately) 0.03783372789621353\n",
      "Iteration 5900 Trace Cost 73.17886352539062 Classifer Cost (Trained Separately) 0.06927548348903656\n",
      "Iteration 6000 Trace Cost 74.59768676757812 Classifer Cost (Trained Separately) 0.055777229368686676\n",
      "Test Accuracy of Primary Movements: 0.9344152584908817\n",
      "Iteration 6100 Trace Cost 73.4782485961914 Classifer Cost (Trained Separately) 0.02974458783864975\n",
      "Iteration 6200 Trace Cost 73.2222900390625 Classifer Cost (Trained Separately) 0.017341753467917442\n",
      "Iteration 6300 Trace Cost 74.33546447753906 Classifer Cost (Trained Separately) 0.018492721021175385\n",
      "Iteration 6400 Trace Cost 74.81619262695312 Classifer Cost (Trained Separately) 0.03389531001448631\n",
      "Iteration 6500 Trace Cost 74.8216323852539 Classifer Cost (Trained Separately) 0.04775109142065048\n",
      "Test Accuracy of Primary Movements: 0.9434498912497908\n",
      "Iteration 6600 Trace Cost 74.57316589355469 Classifer Cost (Trained Separately) 0.05592361465096474\n",
      "Iteration 6700 Trace Cost 74.86287689208984 Classifer Cost (Trained Separately) 0.0939989760518074\n",
      "Iteration 6800 Trace Cost 74.7437744140625 Classifer Cost (Trained Separately) 0.06999471038579941\n",
      "Iteration 6900 Trace Cost 75.0013427734375 Classifer Cost (Trained Separately) 0.020010000094771385\n",
      "Iteration 7000 Trace Cost 74.27981567382812 Classifer Cost (Trained Separately) 0.06603414565324783\n",
      "Test Accuracy of Primary Movements: 0.9437845072778986\n",
      "Iteration 7100 Trace Cost 75.03282165527344 Classifer Cost (Trained Separately) 0.024836372584104538\n",
      "Iteration 7200 Trace Cost 75.38443756103516 Classifer Cost (Trained Separately) 0.016551358625292778\n",
      "Iteration 7300 Trace Cost 75.59501647949219 Classifer Cost (Trained Separately) 0.09318583458662033\n",
      "Iteration 7400 Trace Cost 74.97962188720703 Classifer Cost (Trained Separately) 0.024662544950842857\n",
      "Iteration 7500 Trace Cost 75.8831787109375 Classifer Cost (Trained Separately) 0.06472016125917435\n",
      "Test Accuracy of Primary Movements: 0.9529864480508616\n",
      "Iteration 7600 Trace Cost 76.00019836425781 Classifer Cost (Trained Separately) 0.019202547147870064\n",
      "Iteration 7700 Trace Cost 75.97885131835938 Classifer Cost (Trained Separately) 0.043746594339609146\n",
      "Iteration 7800 Trace Cost 75.11583709716797 Classifer Cost (Trained Separately) 0.017132824286818504\n",
      "Iteration 7900 Trace Cost 75.83336639404297 Classifer Cost (Trained Separately) 0.015689082443714142\n",
      "Iteration 8000 Trace Cost 76.19561767578125 Classifer Cost (Trained Separately) 0.018247295171022415\n",
      "Test Accuracy of Primary Movements: 0.9491383637276225\n",
      "Iteration 8100 Trace Cost 76.27867889404297 Classifer Cost (Trained Separately) 0.040627047419548035\n",
      "Iteration 8200 Trace Cost 76.23536682128906 Classifer Cost (Trained Separately) 0.014629537239670753\n",
      "Iteration 8300 Trace Cost 76.59933471679688 Classifer Cost (Trained Separately) 0.012026757933199406\n",
      "Iteration 8400 Trace Cost 76.23126220703125 Classifer Cost (Trained Separately) 0.03394007310271263\n",
      "Iteration 8500 Trace Cost 76.91998291015625 Classifer Cost (Trained Separately) 0.01436810102313757\n",
      "Test Accuracy of Primary Movements: 0.9585076125146395\n",
      "Iteration 8600 Trace Cost 76.42835235595703 Classifer Cost (Trained Separately) 0.05328912287950516\n",
      "Iteration 8700 Trace Cost 76.4884033203125 Classifer Cost (Trained Separately) 0.01838632859289646\n",
      "Iteration 8800 Trace Cost 77.05348205566406 Classifer Cost (Trained Separately) 0.015166547149419785\n",
      "Iteration 8900 Trace Cost 77.07257843017578 Classifer Cost (Trained Separately) 0.01146132592111826\n",
      "Iteration 9000 Trace Cost 78.15400695800781 Classifer Cost (Trained Separately) 0.01068058516830206\n",
      "Test Accuracy of Primary Movements: 0.9531537560649155\n",
      "Iteration 9100 Trace Cost 77.37915802001953 Classifer Cost (Trained Separately) 0.015826089307665825\n",
      "Iteration 9200 Trace Cost 77.88790130615234 Classifer Cost (Trained Separately) 0.015554330311715603\n",
      "Iteration 9300 Trace Cost 77.33352661132812 Classifer Cost (Trained Separately) 0.01949298568069935\n",
      "Iteration 9400 Trace Cost 77.93792724609375 Classifer Cost (Trained Separately) 0.11459537595510483\n",
      "Iteration 9500 Trace Cost 78.4576187133789 Classifer Cost (Trained Separately) 0.01779130846261978\n",
      "Test Accuracy of Primary Movements: 0.9585076125146395\n",
      "Iteration 9600 Trace Cost 78.18060302734375 Classifer Cost (Trained Separately) 0.03630630671977997\n",
      "Iteration 9700 Trace Cost 78.51484680175781 Classifer Cost (Trained Separately) 0.06590581685304642\n",
      "Iteration 9800 Trace Cost 77.96449279785156 Classifer Cost (Trained Separately) 0.022167906165122986\n",
      "Iteration 9900 Trace Cost 78.95689392089844 Classifer Cost (Trained Separately) 0.04458337277173996\n",
      "Iteration 10000 Trace Cost 78.95626831054688 Classifer Cost (Trained Separately) 0.00475770840421319\n",
      "Test Accuracy of Primary Movements: 0.9598460766270704\n",
      "Iteration 10100 Trace Cost 78.49378204345703 Classifer Cost (Trained Separately) 0.021061284467577934\n",
      "Iteration 10200 Trace Cost 78.7161636352539 Classifer Cost (Trained Separately) 0.004017304629087448\n",
      "Iteration 10300 Trace Cost 79.86027526855469 Classifer Cost (Trained Separately) 0.008619439788162708\n",
      "Iteration 10400 Trace Cost 78.80204772949219 Classifer Cost (Trained Separately) 0.011993546038866043\n",
      "Iteration 10500 Trace Cost 79.62045288085938 Classifer Cost (Trained Separately) 0.003563285106793046\n",
      "Test Accuracy of Primary Movements: 0.9665383971892254\n",
      "Iteration 10600 Trace Cost 79.05401611328125 Classifer Cost (Trained Separately) 0.01047246903181076\n",
      "Iteration 10700 Trace Cost 79.1883773803711 Classifer Cost (Trained Separately) 0.008191073313355446\n",
      "Iteration 10800 Trace Cost 80.1561279296875 Classifer Cost (Trained Separately) 0.04024037718772888\n",
      "Iteration 10900 Trace Cost 79.96421813964844 Classifer Cost (Trained Separately) 0.028324615210294724\n",
      "Iteration 11000 Trace Cost 78.89488220214844 Classifer Cost (Trained Separately) 0.022739741951227188\n",
      "Test Accuracy of Primary Movements: 0.9660364731470638\n",
      "Iteration 11100 Trace Cost 79.7317123413086 Classifer Cost (Trained Separately) 0.05358527600765228\n",
      "Iteration 11200 Trace Cost 79.74644470214844 Classifer Cost (Trained Separately) 0.006425442174077034\n",
      "Iteration 11300 Trace Cost 80.01901245117188 Classifer Cost (Trained Separately) 0.018934672698378563\n",
      "Iteration 11400 Trace Cost 80.61837768554688 Classifer Cost (Trained Separately) 0.0027808155864477158\n",
      "Iteration 11500 Trace Cost 81.1640396118164 Classifer Cost (Trained Separately) 0.009330700151622295\n",
      "Test Accuracy of Primary Movements: 0.9678768613016564\n",
      "Iteration 11600 Trace Cost 81.4496841430664 Classifer Cost (Trained Separately) 0.012215281836688519\n",
      "Iteration 11700 Trace Cost 80.76057434082031 Classifer Cost (Trained Separately) 0.003393336897715926\n",
      "Iteration 11800 Trace Cost 80.96966552734375 Classifer Cost (Trained Separately) 0.01152895949780941\n",
      "Iteration 11900 Trace Cost 81.60952758789062 Classifer Cost (Trained Separately) 0.007709895260632038\n",
      "Iteration 12000 Trace Cost 82.19305419921875 Classifer Cost (Trained Separately) 0.021966923028230667\n",
      "Test Accuracy of Primary Movements: 0.9658691651330099\n",
      "Iteration 12100 Trace Cost 81.52159118652344 Classifer Cost (Trained Separately) 0.006672823801636696\n",
      "Iteration 12200 Trace Cost 82.19908142089844 Classifer Cost (Trained Separately) 0.012952804565429688\n",
      "Iteration 12300 Trace Cost 81.52012634277344 Classifer Cost (Trained Separately) 0.007462235167622566\n",
      "Iteration 12400 Trace Cost 81.7888412475586 Classifer Cost (Trained Separately) 0.0007857446908019483\n",
      "Iteration 12500 Trace Cost 82.24261474609375 Classifer Cost (Trained Separately) 0.00851152278482914\n",
      "Test Accuracy of Primary Movements: 0.9732307177513803\n",
      "Iteration 12600 Trace Cost 81.593017578125 Classifer Cost (Trained Separately) 0.006220730487257242\n",
      "Iteration 12700 Trace Cost 83.08574676513672 Classifer Cost (Trained Separately) 0.017041165381669998\n",
      "Iteration 12800 Trace Cost 82.32877349853516 Classifer Cost (Trained Separately) 0.01410050317645073\n",
      "Iteration 12900 Trace Cost 82.37775421142578 Classifer Cost (Trained Separately) 0.0048302835784852505\n",
      "Iteration 13000 Trace Cost 83.16407775878906 Classifer Cost (Trained Separately) 0.0025100847706198692\n",
      "Test Accuracy of Primary Movements: 0.9695499414421951\n",
      "Iteration 13100 Trace Cost 82.91059875488281 Classifer Cost (Trained Separately) 0.0957837775349617\n",
      "Iteration 13200 Trace Cost 83.12286376953125 Classifer Cost (Trained Separately) 0.002520972862839699\n",
      "Iteration 13300 Trace Cost 83.45249938964844 Classifer Cost (Trained Separately) 0.0016366811469197273\n",
      "Iteration 13400 Trace Cost 83.34481048583984 Classifer Cost (Trained Separately) 0.0003841468715108931\n",
      "Iteration 13500 Trace Cost 83.86288452148438 Classifer Cost (Trained Separately) 0.0033819694072008133\n",
      "Test Accuracy of Primary Movements: 0.9765768780324577\n",
      "Iteration 13600 Trace Cost 84.21041107177734 Classifer Cost (Trained Separately) 0.0009629415581002831\n",
      "Iteration 13700 Trace Cost 84.19619750976562 Classifer Cost (Trained Separately) 0.012164443731307983\n",
      "Iteration 13800 Trace Cost 84.864501953125 Classifer Cost (Trained Separately) 0.0012809992767870426\n",
      "Iteration 13900 Trace Cost 84.57964324951172 Classifer Cost (Trained Separately) 0.0015429817140102386\n",
      "Iteration 14000 Trace Cost 85.71797180175781 Classifer Cost (Trained Separately) 0.0021278264466673136\n",
      "Test Accuracy of Primary Movements: 0.9777480341308349\n",
      "Iteration 14100 Trace Cost 84.43164825439453 Classifer Cost (Trained Separately) 0.002183720702305436\n",
      "Iteration 14200 Trace Cost 85.40281677246094 Classifer Cost (Trained Separately) 0.01072603091597557\n",
      "Iteration 14300 Trace Cost 85.49949645996094 Classifer Cost (Trained Separately) 0.0013643953716382384\n",
      "Iteration 14400 Trace Cost 84.60234069824219 Classifer Cost (Trained Separately) 0.013300900347530842\n",
      "Iteration 14500 Trace Cost 84.80787658691406 Classifer Cost (Trained Separately) 0.01205350086092949\n",
      "Test Accuracy of Primary Movements: 0.9780826501589426\n",
      "Iteration 14600 Trace Cost 85.33810424804688 Classifer Cost (Trained Separately) 0.001675016712397337\n",
      "Iteration 14700 Trace Cost 86.31465148925781 Classifer Cost (Trained Separately) 0.002756711794063449\n",
      "Iteration 14800 Trace Cost 85.72264099121094 Classifer Cost (Trained Separately) 0.005635418929159641\n",
      "Iteration 14900 Trace Cost 85.71366882324219 Classifer Cost (Trained Separately) 0.008498266339302063\n",
      "Iteration 15000 Trace Cost 86.20592498779297 Classifer Cost (Trained Separately) 0.0014686232898384333\n",
      "Test Accuracy of Primary Movements: 0.9792538062573197\n",
      "Iteration 15100 Trace Cost 86.23567962646484 Classifer Cost (Trained Separately) 0.0019209061283618212\n",
      "Iteration 15200 Trace Cost 86.21110534667969 Classifer Cost (Trained Separately) 0.0008364731911569834\n",
      "Iteration 15300 Trace Cost 86.86283874511719 Classifer Cost (Trained Separately) 0.003200183156877756\n",
      "Iteration 15400 Trace Cost 86.93753051757812 Classifer Cost (Trained Separately) 0.00025485255173407495\n",
      "Iteration 15500 Trace Cost 87.406494140625 Classifer Cost (Trained Separately) 0.0001722125307423994\n",
      "Test Accuracy of Primary Movements: 0.9784172661870504\n",
      "Iteration 15600 Trace Cost 87.30511474609375 Classifer Cost (Trained Separately) 0.0007572192116640508\n",
      "Iteration 15700 Trace Cost 88.09334564208984 Classifer Cost (Trained Separately) 0.006681782193481922\n",
      "Iteration 15800 Trace Cost 88.1369400024414 Classifer Cost (Trained Separately) 0.0016664414433762431\n",
      "Iteration 15900 Trace Cost 88.01117706298828 Classifer Cost (Trained Separately) 0.001600713119842112\n",
      "Iteration 16000 Trace Cost 88.21912384033203 Classifer Cost (Trained Separately) 0.028072571381926537\n",
      "Test Accuracy of Primary Movements: 0.9807595783838046\n",
      "Iteration 16100 Trace Cost 88.75975036621094 Classifer Cost (Trained Separately) 0.0008405469125136733\n",
      "Iteration 16200 Trace Cost 87.89188385009766 Classifer Cost (Trained Separately) 0.006242737174034119\n",
      "Iteration 16300 Trace Cost 89.00904846191406 Classifer Cost (Trained Separately) 0.005257859826087952\n",
      "Iteration 16400 Trace Cost 89.65798950195312 Classifer Cost (Trained Separately) 0.00033437710953876376\n",
      "Iteration 16500 Trace Cost 88.49467468261719 Classifer Cost (Trained Separately) 0.004860932007431984\n",
      "Test Accuracy of Primary Movements: 0.9820980424962356\n",
      "Iteration 16600 Trace Cost 87.69561767578125 Classifer Cost (Trained Separately) 0.0026864828541874886\n",
      "Iteration 16700 Trace Cost 90.19720458984375 Classifer Cost (Trained Separately) 0.003494610544294119\n",
      "Iteration 16800 Trace Cost 88.85546875 Classifer Cost (Trained Separately) 0.002488878322765231\n",
      "Iteration 16900 Trace Cost 89.50782775878906 Classifer Cost (Trained Separately) 0.00032230382203124464\n",
      "Iteration 17000 Trace Cost 89.7933349609375 Classifer Cost (Trained Separately) 0.009968779049813747\n",
      "Test Accuracy of Primary Movements: 0.9836038146227204\n",
      "Iteration 17100 Trace Cost 90.3033676147461 Classifer Cost (Trained Separately) 0.002650847192853689\n",
      "Iteration 17200 Trace Cost 89.66926574707031 Classifer Cost (Trained Separately) 0.002640525810420513\n",
      "Iteration 17300 Trace Cost 90.27825927734375 Classifer Cost (Trained Separately) 0.0009313643095083535\n",
      "Iteration 17400 Trace Cost 90.69064331054688 Classifer Cost (Trained Separately) 0.003440934233367443\n",
      "Iteration 17500 Trace Cost 90.87740325927734 Classifer Cost (Trained Separately) 0.0010419312166050076\n",
      "Test Accuracy of Primary Movements: 0.9822653505102894\n",
      "Iteration 17600 Trace Cost 91.12921142578125 Classifer Cost (Trained Separately) 0.013253442011773586\n",
      "Iteration 17700 Trace Cost 90.03106689453125 Classifer Cost (Trained Separately) 0.0003895092522725463\n",
      "Iteration 17800 Trace Cost 91.10243225097656 Classifer Cost (Trained Separately) 0.005497426260262728\n",
      "Iteration 17900 Trace Cost 91.12451171875 Classifer Cost (Trained Separately) 0.0007058849441818893\n",
      "Iteration 18000 Trace Cost 92.01397705078125 Classifer Cost (Trained Separately) 0.00016595458146184683\n",
      "Test Accuracy of Primary Movements: 0.9832691985946127\n",
      "Iteration 18100 Trace Cost 92.46199798583984 Classifer Cost (Trained Separately) 0.00040450721280649304\n",
      "Iteration 18200 Trace Cost 92.03822326660156 Classifer Cost (Trained Separately) 0.001356669468805194\n",
      "Iteration 18300 Trace Cost 92.90762329101562 Classifer Cost (Trained Separately) 0.0005523521103896201\n",
      "Iteration 18400 Trace Cost 93.00057220458984 Classifer Cost (Trained Separately) 0.008421615697443485\n",
      "Iteration 18500 Trace Cost 92.6873779296875 Classifer Cost (Trained Separately) 2.2172735043568537e-05\n",
      "Test Accuracy of Primary Movements: 0.9832691985946127\n",
      "Iteration 18600 Trace Cost 93.10810852050781 Classifer Cost (Trained Separately) 8.570312638767064e-05\n",
      "Iteration 18700 Trace Cost 93.78136444091797 Classifer Cost (Trained Separately) 0.008296241983771324\n",
      "Iteration 18800 Trace Cost 93.52902221679688 Classifer Cost (Trained Separately) 0.0005899520474486053\n",
      "Iteration 18900 Trace Cost 93.33787536621094 Classifer Cost (Trained Separately) 0.0004466033715289086\n",
      "Iteration 19000 Trace Cost 93.59053039550781 Classifer Cost (Trained Separately) 0.010090465657413006\n",
      "Test Accuracy of Primary Movements: 0.9852768947632592\n",
      "Iteration 19100 Trace Cost 93.32144927978516 Classifer Cost (Trained Separately) 0.0024183313362300396\n",
      "Iteration 19200 Trace Cost 95.03599548339844 Classifer Cost (Trained Separately) 0.00019849672389682382\n",
      "Iteration 19300 Trace Cost 94.87451171875 Classifer Cost (Trained Separately) 0.0006276475032791495\n",
      "Iteration 19400 Trace Cost 94.58711242675781 Classifer Cost (Trained Separately) 0.0016260339180007577\n",
      "Iteration 19500 Trace Cost 95.68870544433594 Classifer Cost (Trained Separately) 0.00033307430567219853\n",
      "Test Accuracy of Primary Movements: 0.9847749707210975\n",
      "Iteration 19600 Trace Cost 94.97453308105469 Classifer Cost (Trained Separately) 0.0003931287210434675\n",
      "Iteration 19700 Trace Cost 93.47759246826172 Classifer Cost (Trained Separately) 0.0005639569717459381\n",
      "Iteration 19800 Trace Cost 94.66558837890625 Classifer Cost (Trained Separately) 0.033807720988988876\n",
      "Iteration 19900 Trace Cost 95.47975158691406 Classifer Cost (Trained Separately) 0.0002620498707983643\n",
      "Iteration 20000 Trace Cost 96.00375366210938 Classifer Cost (Trained Separately) 0.0004771943495143205\n",
      "Test Accuracy of Primary Movements: 0.9861134348335285\n",
      "Iteration 20100 Trace Cost 96.65686798095703 Classifer Cost (Trained Separately) 0.02261311002075672\n",
      "Iteration 20200 Trace Cost 96.58448791503906 Classifer Cost (Trained Separately) 0.002243989845737815\n",
      "Iteration 20300 Trace Cost 96.91937255859375 Classifer Cost (Trained Separately) 0.0039279223419725895\n",
      "Iteration 20400 Trace Cost 97.18999481201172 Classifer Cost (Trained Separately) 0.00047874715528450906\n",
      "Iteration 20500 Trace Cost 97.35611724853516 Classifer Cost (Trained Separately) 0.008809051476418972\n",
      "Test Accuracy of Primary Movements: 0.9861134348335285\n",
      "Iteration 20800 Trace Cost 97.79376220703125 Classifer Cost (Trained Separately) 0.0010649310424923897\n",
      "Iteration 20900 Trace Cost 97.60112762451172 Classifer Cost (Trained Separately) 0.002941895043477416\n",
      "Iteration 21000 Trace Cost 98.39006042480469 Classifer Cost (Trained Separately) 0.008128967136144638\n",
      "Test Accuracy of Primary Movements: 0.9829345825665049\n",
      "Iteration 21100 Trace Cost 98.44097137451172 Classifer Cost (Trained Separately) 0.00021951286180410534\n",
      "Iteration 21200 Trace Cost 98.74893188476562 Classifer Cost (Trained Separately) 0.0001692437072051689\n",
      "Iteration 21300 Trace Cost 97.61898040771484 Classifer Cost (Trained Separately) 0.0002671823895070702\n",
      "Iteration 21400 Trace Cost 99.24176788330078 Classifer Cost (Trained Separately) 0.0001161880572908558\n",
      "Iteration 21500 Trace Cost 100.77994537353516 Classifer Cost (Trained Separately) 0.0018432928482070565\n",
      "Test Accuracy of Primary Movements: 0.984105738664882\n",
      "Iteration 21600 Trace Cost 99.42323303222656 Classifer Cost (Trained Separately) 2.6739387976704165e-05\n",
      "Iteration 21700 Trace Cost 99.1213150024414 Classifer Cost (Trained Separately) 0.00011519376857904717\n",
      "Iteration 21800 Trace Cost 100.7759780883789 Classifer Cost (Trained Separately) 0.0005582750891335309\n",
      "Iteration 21900 Trace Cost 100.77349853515625 Classifer Cost (Trained Separately) 0.00028148668934591115\n",
      "Iteration 22000 Trace Cost 100.15816497802734 Classifer Cost (Trained Separately) 6.713443872286007e-05\n",
      "Test Accuracy of Primary Movements: 0.9871172829178517\n",
      "Iteration 22100 Trace Cost 100.37278747558594 Classifer Cost (Trained Separately) 0.0006270389421842992\n",
      "Iteration 22200 Trace Cost 101.56743621826172 Classifer Cost (Trained Separately) 0.0004175545182079077\n",
      "Iteration 22300 Trace Cost 101.36680603027344 Classifer Cost (Trained Separately) 0.0004928872804157436\n",
      "Iteration 22400 Trace Cost 100.5758285522461 Classifer Cost (Trained Separately) 0.00018746621208265424\n",
      "Iteration 22500 Trace Cost 101.69673156738281 Classifer Cost (Trained Separately) 6.910991214681417e-05\n",
      "Test Accuracy of Primary Movements: 0.9866153588756902\n",
      "Iteration 22600 Trace Cost 101.3575668334961 Classifer Cost (Trained Separately) 9.979908645618707e-05\n",
      "Iteration 22700 Trace Cost 100.91218566894531 Classifer Cost (Trained Separately) 0.0005696298321709037\n",
      "Iteration 22800 Trace Cost 101.03289794921875 Classifer Cost (Trained Separately) 0.00025928940158337355\n",
      "Iteration 22900 Trace Cost 100.69573974609375 Classifer Cost (Trained Separately) 4.003286812803708e-05\n",
      "Iteration 23000 Trace Cost 102.24657440185547 Classifer Cost (Trained Separately) 0.0010012544225901365\n",
      "Test Accuracy of Primary Movements: 0.9877865149740672\n",
      "Iteration 23100 Trace Cost 102.68002319335938 Classifer Cost (Trained Separately) 8.674332639202476e-05\n",
      "Iteration 23200 Trace Cost 103.06664276123047 Classifer Cost (Trained Separately) 0.00011408160207793117\n",
      "Iteration 23300 Trace Cost 103.29720306396484 Classifer Cost (Trained Separately) 0.00028435158310458064\n",
      "Iteration 23400 Trace Cost 102.74160766601562 Classifer Cost (Trained Separately) 0.00019005421199835837\n",
      "Iteration 23500 Trace Cost 102.41277313232422 Classifer Cost (Trained Separately) 0.005458401050418615\n",
      "Test Accuracy of Primary Movements: 0.9866153588756902\n",
      "Iteration 23600 Trace Cost 103.83926391601562 Classifer Cost (Trained Separately) 0.001099155517295003\n",
      "Iteration 23700 Trace Cost 103.98202514648438 Classifer Cost (Trained Separately) 0.00021608294628094882\n",
      "Iteration 23800 Trace Cost 103.73391723632812 Classifer Cost (Trained Separately) 0.00030837609665468335\n",
      "Iteration 23900 Trace Cost 103.44795989990234 Classifer Cost (Trained Separately) 3.534714414854534e-05\n",
      "Iteration 24000 Trace Cost 104.66708374023438 Classifer Cost (Trained Separately) 0.00012364907888695598\n",
      "Test Accuracy of Primary Movements: 0.9877865149740672\n",
      "Iteration 24100 Trace Cost 105.47384643554688 Classifer Cost (Trained Separately) 0.00059797044377774\n",
      "Iteration 24200 Trace Cost 105.6014404296875 Classifer Cost (Trained Separately) 0.0003126645169686526\n",
      "Iteration 24300 Trace Cost 103.95426177978516 Classifer Cost (Trained Separately) 0.0007721727597527206\n",
      "Iteration 24400 Trace Cost 105.40962219238281 Classifer Cost (Trained Separately) 0.00018415797967463732\n",
      "Iteration 24500 Trace Cost 106.15143585205078 Classifer Cost (Trained Separately) 0.004771076608449221\n",
      "Test Accuracy of Primary Movements: 0.988121131002175\n",
      "Iteration 24600 Trace Cost 105.91110229492188 Classifer Cost (Trained Separately) 0.00233938149176538\n",
      "Iteration 24700 Trace Cost 106.5185775756836 Classifer Cost (Trained Separately) 0.0005603451863862574\n",
      "Iteration 24800 Trace Cost 105.8240966796875 Classifer Cost (Trained Separately) 0.00048126690671779215\n",
      "Iteration 24900 Trace Cost 107.19294738769531 Classifer Cost (Trained Separately) 6.192186992848292e-05\n",
      "Iteration 25000 Trace Cost 107.5554428100586 Classifer Cost (Trained Separately) 0.0014641323359683156\n",
      "Test Accuracy of Primary Movements: 0.9887903630583905\n",
      "Iteration 25100 Trace Cost 106.11953735351562 Classifer Cost (Trained Separately) 0.0005217625293880701\n",
      "Iteration 25200 Trace Cost 107.63433837890625 Classifer Cost (Trained Separately) 7.962060044519603e-05\n",
      "Iteration 25300 Trace Cost 107.27948760986328 Classifer Cost (Trained Separately) 3.0895069357939065e-05\n",
      "Iteration 25400 Trace Cost 106.94963073730469 Classifer Cost (Trained Separately) 0.0003494463162496686\n",
      "Iteration 25500 Trace Cost 106.75988006591797 Classifer Cost (Trained Separately) 0.008178873918950558\n",
      "Test Accuracy of Primary Movements: 0.9887903630583905\n",
      "Iteration 25600 Trace Cost 107.70668029785156 Classifer Cost (Trained Separately) 0.019947117194533348\n",
      "Iteration 25700 Trace Cost 108.0750732421875 Classifer Cost (Trained Separately) 2.945683263533283e-05\n",
      "Iteration 25800 Trace Cost 107.99966430664062 Classifer Cost (Trained Separately) 0.00011618612188613042\n",
      "Iteration 25900 Trace Cost 108.11519622802734 Classifer Cost (Trained Separately) 0.00013587030116468668\n",
      "Iteration 26000 Trace Cost 109.16984558105469 Classifer Cost (Trained Separately) 0.00021284281683620065\n",
      "Test Accuracy of Primary Movements: 0.9879538229881212\n",
      "Iteration 26100 Trace Cost 109.00450134277344 Classifer Cost (Trained Separately) 0.00029400852508842945\n",
      "Iteration 26200 Trace Cost 108.63465118408203 Classifer Cost (Trained Separately) 0.00022946129320189357\n",
      "Iteration 26300 Trace Cost 109.26759338378906 Classifer Cost (Trained Separately) 0.0015866972971707582\n",
      "Iteration 26400 Trace Cost 108.98590087890625 Classifer Cost (Trained Separately) 5.030433021602221e-05\n",
      "Iteration 26500 Trace Cost 109.01087951660156 Classifer Cost (Trained Separately) 0.0004897902836091816\n",
      "Test Accuracy of Primary Movements: 0.9899615191567676\n",
      "Iteration 26600 Trace Cost 109.99413299560547 Classifer Cost (Trained Separately) 8.762285142438486e-05\n",
      "Iteration 26700 Trace Cost 109.68169403076172 Classifer Cost (Trained Separately) 1.87806708709104e-05\n",
      "Iteration 26800 Trace Cost 109.6258544921875 Classifer Cost (Trained Separately) 5.597813651547767e-05\n",
      "Iteration 26900 Trace Cost 108.61285400390625 Classifer Cost (Trained Separately) 2.9266162528074346e-05\n",
      "Iteration 27000 Trace Cost 110.45867919921875 Classifer Cost (Trained Separately) 4.7581459512002766e-05\n",
      "Test Accuracy of Primary Movements: 0.9901288271708215\n",
      "Iteration 27100 Trace Cost 109.61054229736328 Classifer Cost (Trained Separately) 9.906800551107153e-05\n",
      "Iteration 27200 Trace Cost 110.66952514648438 Classifer Cost (Trained Separately) 0.00021022134751547128\n",
      "Iteration 27300 Trace Cost 110.48976135253906 Classifer Cost (Trained Separately) 1.0974581527989358e-05\n",
      "Iteration 27400 Trace Cost 111.35713195800781 Classifer Cost (Trained Separately) 0.006066556088626385\n",
      "Iteration 27500 Trace Cost 111.33258056640625 Classifer Cost (Trained Separately) 6.154504080768675e-05\n",
      "Test Accuracy of Primary Movements: 0.9887903630583905\n",
      "Iteration 27600 Trace Cost 111.40872192382812 Classifer Cost (Trained Separately) 0.0001276123948628083\n",
      "Iteration 27700 Trace Cost 110.5289077758789 Classifer Cost (Trained Separately) 1.5199387235043105e-05\n",
      "Iteration 27800 Trace Cost 111.65418243408203 Classifer Cost (Trained Separately) 0.00025116471806541085\n",
      "Iteration 27900 Trace Cost 111.99748992919922 Classifer Cost (Trained Separately) 0.0001930658909259364\n",
      "Iteration 28000 Trace Cost 111.8056640625 Classifer Cost (Trained Separately) 7.200025720521808e-06\n",
      "Test Accuracy of Primary Movements: 0.9914672912832525\n",
      "Iteration 28100 Trace Cost 112.49344635009766 Classifer Cost (Trained Separately) 4.909971903543919e-05\n",
      "Iteration 28200 Trace Cost 112.15385437011719 Classifer Cost (Trained Separately) 2.39928049268201e-05\n",
      "Iteration 28300 Trace Cost 112.56285858154297 Classifer Cost (Trained Separately) 0.015510521829128265\n",
      "Iteration 28400 Trace Cost 113.19650268554688 Classifer Cost (Trained Separately) 0.0004987030406482518\n",
      "Iteration 28500 Trace Cost 112.59883117675781 Classifer Cost (Trained Separately) 1.447071281290846e-05\n",
      "Test Accuracy of Primary Movements: 0.9904634431989292\n",
      "Iteration 28600 Trace Cost 112.81710815429688 Classifer Cost (Trained Separately) 7.103376447048504e-06\n",
      "Iteration 28700 Trace Cost 112.54828643798828 Classifer Cost (Trained Separately) 2.4687520635779947e-05\n",
      "Iteration 28800 Trace Cost 113.14640808105469 Classifer Cost (Trained Separately) 0.0002545599709264934\n",
      "Iteration 28900 Trace Cost 112.54995727539062 Classifer Cost (Trained Separately) 0.0013024540385231376\n",
      "Iteration 29000 Trace Cost 114.17059326171875 Classifer Cost (Trained Separately) 5.372395753511228e-05\n",
      "Test Accuracy of Primary Movements: 0.9902961351848754\n",
      "Iteration 29100 Trace Cost 112.59329223632812 Classifer Cost (Trained Separately) 0.00021965189080219716\n",
      "Iteration 29200 Trace Cost 114.34756469726562 Classifer Cost (Trained Separately) 3.160549022140913e-05\n",
      "Iteration 29300 Trace Cost 113.80256652832031 Classifer Cost (Trained Separately) 3.796350574702956e-05\n",
      "Iteration 29400 Trace Cost 114.17134094238281 Classifer Cost (Trained Separately) 0.0008009973680600524\n",
      "Iteration 29500 Trace Cost 114.90956115722656 Classifer Cost (Trained Separately) 0.0001058366906363517\n",
      "Test Accuracy of Primary Movements: 0.9897942111427137\n",
      "Iteration 29600 Trace Cost 113.75482940673828 Classifer Cost (Trained Separately) 0.00015122214972507209\n",
      "Iteration 29700 Trace Cost 114.86815643310547 Classifer Cost (Trained Separately) 4.6564902731915936e-05\n",
      "Iteration 29800 Trace Cost 114.08912658691406 Classifer Cost (Trained Separately) 0.0011975582456216216\n",
      "Iteration 29900 Trace Cost 116.08866119384766 Classifer Cost (Trained Separately) 0.0001432000135537237\n",
      "Iteration 30000 Trace Cost 114.93659973144531 Classifer Cost (Trained Separately) 1.872145730885677e-05\n",
      "Test Accuracy of Primary Movements: 0.9902961351848754\n"
     ]
    }
   ],
   "source": [
    "### Change Total Iterartion to 30001 to run the full epoches. Here we run 501 iterartions for testing the codes. \n",
    "# total_iteration = 501\n",
    "total_iteration = 30001 \n",
    "\n",
    "# torch.cuda.set_device(2)\n",
    "\n",
    "# Measuring Dependence between EEG and EMG\n",
    "SAMPLE_X = eeg_data\n",
    "SAMPLE_Y = emg_data\n",
    "\n",
    "label_tensor = new_label\n",
    "\n",
    "indices = np.arange(SAMPLE_X.shape[0])\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "train_X, test_X, train_labels, test_labels, idx_train, idx_test = train_test_split(torch.cat((SAMPLE_X, SAMPLE_Y), 1), label_tensor, indices, test_size=0.2, random_state=42)\n",
    "train_X, train_Y, test_X, test_Y = train_X[:, :60], train_X[:, 60:], test_X[:, :60], test_X[:, 60:]\n",
    "\n",
    "NET_1 = Advanced1DCNN_channel(SAMPLE_X.shape[1], 128, 4000).cuda()\n",
    "NET_2 = Advanced1DCNN_channel(SAMPLE_Y.shape[1], 128, 4000).cuda()\n",
    "\n",
    "classifier = ComplexClassifier(dim_features = 128, num_classes = label_tensor.max()+1).cuda()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "beta1 = 0.9\n",
    "beta2 = 0.999\n",
    "\n",
    "lr1 = 0.0005\n",
    "lr2 = 0.0005\n",
    "\n",
    "optimizer_1 = optim.Adam([\n",
    "    {'params': NET_1.parameters(), 'lr': lr2, 'betas': (beta1, beta2)},\n",
    "], amsgrad = True)\n",
    "\n",
    "optimizer_2 = optim.Adam([\n",
    "    {'params': NET_2.parameters(), 'lr': lr2, 'betas': (beta1, beta2)},\n",
    "], amsgrad = True)\n",
    "\n",
    "optimizer_classifier = optim.Adam([\n",
    "    {'params': classifier.parameters(), 'lr': lr2, 'betas': (beta1, beta2)},\n",
    "], amsgrad = True)\n",
    "\n",
    "save_curve = []\n",
    "eig_list = []\n",
    "classifier_error = []\n",
    "\n",
    "track_cov_final = torch.zeros((128+128)).cuda()\n",
    "track_cov_estimate_final = torch.zeros((128+128)).cuda()\n",
    "\n",
    "save_path = './save_exp_inter_subj/'\n",
    "create_folder_if_not_exists(save_path)\n",
    "\n",
    "for i in range(1, total_iteration):\n",
    "    batch_size = 100\n",
    "    batch_indices = torch.randint(0, train_X.shape[0], (batch_size,))\n",
    "    input_x = train_X[batch_indices]\n",
    "    input_y = train_Y[batch_indices]\n",
    "    \n",
    "    feature_1 = NET_1(input_x.cuda())\n",
    "    feature_2 = NET_2(input_y.cuda())\n",
    "    \n",
    "    # Calculating ACFs and CCFs\n",
    "    RF = (feature_1.T@feature_1)/feature_1.shape[0]\n",
    "    RG = (feature_2.T@feature_2)/feature_2.shape[0]\n",
    "    P = (feature_1.T@feature_2)/feature_2.shape[0]\n",
    "    \n",
    "    input_dim, output_dim = RF.shape[1], RG.shape[1]\n",
    "    RFG = torch.zeros((input_dim+output_dim, input_dim+output_dim)).cuda()\n",
    "    RFG[:input_dim, :input_dim] = RF\n",
    "    RFG[input_dim:, input_dim:] = RG\n",
    "    RFG[:input_dim, input_dim:] = P\n",
    "    RFG[input_dim:, :input_dim] = P.T\n",
    "    \n",
    "    # Track the ACFs and CCFs and construct the cost\n",
    "    track_cov_final, track_cov_estimate_final, COST = MCA_LOSS_GIVEN_R(RFG, track_cov_final, i, 128)\n",
    "    COST = return_cost_trace(RFG, track_cov_estimate_final)\n",
    "    \n",
    "    labels_x = train_labels[batch_indices]\n",
    "    \n",
    "    # Train classifier (calling .detach() such that no graidents are passed from dependence networks)\n",
    "    output_class = classifier(feature_1.detach())\n",
    "    loss = criterion(output_class, labels_x.cuda())    \n",
    "    (COST+loss).backward()\n",
    "    \n",
    "    optimizer_1.step()  \n",
    "    optimizer_2.step()  \n",
    "    optimizer_classifier.step()  \n",
    "\n",
    "    optimizer_1.zero_grad()  \n",
    "    optimizer_2.zero_grad()      \n",
    "    optimizer_classifier.zero_grad()  \n",
    "\n",
    "    \n",
    "    E1,V1 = torch.linalg.eigh(track_cov_estimate_final[:128, :128])\n",
    "    E2,V2 = torch.linalg.eigh(track_cov_estimate_final[128:, 128:])\n",
    "\n",
    "    RF_NORM = V1@torch.diag(E1**(-1/2))@V1.T\n",
    "    RG_NORM = V2@torch.diag(E2**(-1/2))@V2.T\n",
    "\n",
    "    P = track_cov_estimate_final[:input_dim, input_dim:]\n",
    "    P_STAR = RF_NORM@P@RG_NORM    \n",
    "\n",
    "    U, S, V = torch.svd(P_STAR)\n",
    "    eig_list.append(S.detach().cpu().numpy())\n",
    "    \n",
    "    RF_NORM = RF_NORM.detach().cpu()\n",
    "    U = U.detach().cpu()\n",
    "    \n",
    "    RG_NORM = RG_NORM.detach().cpu()\n",
    "    V = V.detach().cpu()\n",
    "    \n",
    "    eig_list.append(S.detach().cpu().numpy())\n",
    "    \n",
    "    TSD = S.sum()\n",
    "    classifier_error.append(TSD.item())\n",
    "\n",
    "    if i%100 == 0:\n",
    "        print('Iteration', i, 'Trace Cost', TSD.item(), 'Classifer Cost (Trained Separately)', loss.item())\n",
    "        \n",
    "        if i in [200, 1000, 5000, 30000]:\n",
    "\n",
    "            np.save(save_path+'cost_iter{0}.npy'.format(i), classifier_error)\n",
    "            torch.save(NET_1.state_dict(), save_path+\"NET_1_iter{0}.pth\".format(i))\n",
    "            torch.save(NET_2.state_dict(), save_path+\"NET_2_iter{0}.pth\".format(i))\n",
    "\n",
    "        if i%500 == 0:\n",
    "            NET_1.eval()\n",
    "            classifier.eval()\n",
    "\n",
    "            val_loss = 0.0\n",
    "            mse_loss = 0.0\n",
    "            correct = 0\n",
    "            criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "            with torch.no_grad():\n",
    "                for index, k in enumerate(range(0, test_X.shape[0], 200)):\n",
    "\n",
    "                    output_class = classifier(NET_1(test_X[k:k+200].cuda()))\n",
    "                    val_loss += criterion(output_class, test_labels[k:k+200].cuda()).item()\n",
    "                    pred = output_class.argmax(dim=1, keepdim=True).detach().cpu()\n",
    "                    correct += pred.eq(test_labels[k:k+200].view_as(pred)).sum().item()\n",
    "\n",
    "            print('Test Accuracy of Primary Movements:', correct/test_X.shape[0])\n",
    "            \n",
    "            NET_1.train()\n",
    "            classifier.train()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}