Web/performance (#98)

* updated 78yo anon prediction for clf, electro, physionet * added results on test set * added performance metrics on test set * added openbci dataset intro * completed electro vs clf * added further dev section * added intro alex vs sleepedf * finish * Update web/src/views/performance/index.js Co-authored-by: William Harvey <[email protected]> * oups * updated youtube link!!! * Apply suggestions from code review Co-authored-by: Mathieu Bélanger <[email protected]> * update link to embed * Apply suggestions from code review Co-authored-by: Mathieu Bélanger <[email protected]> Co-authored-by: William Harvey <[email protected]> Co-authored-by: Mathieu Bélanger <[email protected]>
PolyCortex · Nov 23, 2020 · 9d0b40a · 9d0b40a
1 parent ba9b1a5
commit 9d0b40a
Show file tree

Hide file tree

Showing 10 changed files with 779 additions and 6,268 deletions.
diff --git a/ai/models/RF_HMM_smaller.ipynb b/ai/models/RF_HMM_smaller.ipynb
@@ -30,7 +30,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -164,7 +164,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -188,7 +188,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -225,7 +225,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -259,7 +259,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -533,15 +533,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 3min 41s, sys: 2.12 s, total: 3min 43s\n",
-      "Wall time: 1min 13s\n"
+      "CPU times: user 3min 41s, sys: 2.2 s, total: 3min 44s\n",
+      "Wall time: 1min 21s\n"
      ]
     }
    ],
@@ -620,7 +620,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -658,6 +658,51 @@
     "print(\"Agreement score (Cohen Kappa): \", cohen_kappa_score(y_test, y_test_pred))"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[[1457   66   41    5   55]\n",
+      " [ 186  259  318    3  217]\n",
+      " [   6   43 3195  245  114]\n",
+      " [   0    0   34  577    0]\n",
+      " [   7   10  146    0 1139]]\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           W       0.88      0.90      0.89      1624\n",
+      "          N1       0.69      0.26      0.38       983\n",
+      "          N2       0.86      0.89      0.87      3603\n",
+      "          N3       0.70      0.94      0.80       611\n",
+      "         REM       0.75      0.87      0.81      1302\n",
+      "\n",
+      "    accuracy                           0.82      8123\n",
+      "   macro avg       0.77      0.77      0.75      8123\n",
+      "weighted avg       0.81      0.82      0.80      8123\n",
+      "\n",
+      "Agreement score (Cohen Kappa):  0.7413141526163325\n"
+     ]
+    }
+   ],
+   "source": [
+    "hmm_model = MultinomialHMM(n_components=N_STAGES)\n",
+    "\n",
+    "MODELS_DIR = \"../trained_model\"\n",
+    "hmm_model.emissionprob_ = np.load(f\"{MODELS_DIR}/HMM_emissionprob.npy\", allow_pickle=True)\n",
+    "hmm_model.startprob_ = np.load(f\"{MODELS_DIR}/HMM_startprob.npy\", allow_pickle=True)\n",
+    "hmm_model.transmat_ = np.load(f\"{MODELS_DIR}/HMM_transmat.npy\", allow_pickle=True)\n",
+    "\n",
+    "y_test_pred = hmm_model.predict(y_test_pred.reshape(-1, 1))\n",
+    "\n",
+    "print(confusion_matrix(y_test, y_test_pred))\n",
+    "print(classification_report(y_test, y_test_pred, target_names=SLEEP_STAGES_VALUES.keys()))\n",
+    "print(\"Agreement score (Cohen Kappa): \", cohen_kappa_score(y_test, y_test_pred))"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

diff --git a/ai/models/model_utils.py b/ai/models/model_utils.py
@@ -114,7 +114,6 @@ def train_test_split_according_to_age(X, y, use_continuous_age, subjects_test=No
         [85,110] # 32 recordings
     ]
     age_categories = np.unique(X[:, AGE_CATEGORY_COL_IDX])
-    assert subjects_test is None or len(subjects_test) == len(age_categories), "If subjects are specified, they must be specified for all age groups"
 
     if subjects_test is None:
         unique_subject_with_age = np.array([

diff --git a/ai/prediction_anonymous.ipynb b/ai/prediction_anonymous.ipynb
@@ -18,32 +18,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The autoreload extension is already loaded. To reload it, use:\n",
-      "  %reload_ext autoreload\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "%load_ext autoreload\n",
     "%autoreload 2"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
     "%matplotlib inline\n",
     "\n",
     "import mne\n",
+    "import json\n",
     "import pandas as pd\n",
     "import numpy as np\n",
     "import matplotlib.pyplot as plt\n",
@@ -65,7 +57,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -81,7 +73,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -155,7 +147,7 @@
        "1          25620.0  1989-04-25 14:50:00           609519000.0  "
       ]
      },
-     "execution_count": 9,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -175,7 +167,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -231,7 +223,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -257,7 +249,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -282,7 +274,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -310,7 +302,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -357,6 +349,48 @@
     "print(\"Agreement score (Cohen Kappa): \", cohen_kappa_score(y_physionet, y_electrophysiologist))"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "electro_hypno = {\n",
+    "    'timestamps': [\n",
+    "        int((datetime.fromtimestamp(615514680) + timedelta(seconds=index*EPOCH_DURATION)).timestamp())\n",
+    "        for index, stage in enumerate(y_electrophysiologist)\n",
+    "    ],\n",
+    "    'stages': [\n",
+    "        str(list(SLEEP_STAGES_VALUES.keys())[list(SLEEP_STAGES_VALUES.values()).index(stage)])\n",
+    "        for index, stage in enumerate(y_electrophysiologist)\n",
+    "    ]\n",
+    "}\n",
+    "\n",
+    "with open('data/electrophysiologist_woman78yo_sleepedf.json', 'w') as fp:\n",
+    "    json.dump(dict(epochs=electro_hypno), fp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sleepedf_hypno = {\n",
+    "    'timestamps': [\n",
+    "        int((datetime.fromtimestamp(615514680) + timedelta(seconds=index*EPOCH_DURATION)).timestamp())\n",
+    "        for index, stage in enumerate(y_physionet)\n",
+    "    ],\n",
+    "    'stages': [\n",
+    "        str(list(SLEEP_STAGES_VALUES.keys())[list(SLEEP_STAGES_VALUES.values()).index(stage)])\n",
+    "        for index, stage in enumerate(y_physionet)\n",
+    "    ]\n",
+    "}\n",
+    "\n",
+    "with open('data/physionet_woman78yo_sleepedf.json', 'w') as fp:\n",
+    "    json.dump(dict(epochs=sleepedf_hypno), fp)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,