Code/Transferability experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tsfresh\n",
    "import os\n",
    "import json\n",
    "import scapy\n",
    "import numpy as np\n",
    "import warnings\n",
    "from timeit import default_timer as timer\n",
    "\n",
    "from scapy.all import *\n",
    "\n",
    "warnings.filterwarnings(\"ignore\") #ignore warnings caused by \n",
    "\n",
    "\n",
    "#################################################################\n",
    "#                                                               #\n",
    "#               malicious csv files import                      #\n",
    "#                                                               #\n",
    "#################################################################\n",
    "\n",
    "df1 = pd.read_csv('/malicious/WebOS_binary.csv') #\n",
    "df2 = pd.read_csv('/malicious/Server_Binary.csv') #\n",
    "df3 = pd.read_csv('/malicious/Raspberry_Webmine_Robust.csv')\n",
    "df4 = pd.read_csv('/malicious/Raspberry_Binary.csv') #\n",
    "df5 = pd.read_csv('/malicious/Raspberry_Webmine_Aggressive.csv')\n",
    "df6 = pd.read_csv('/malicious/Raspberry_WebminePool_Aggressive.csv')\n",
    "df7 = pd.read_csv('/malicious/Server_WebminePool_Aggressive.csv') #\n",
    "\n",
    "df32 = pd.read_csv('/malicious/Server_WebminePool_Robust.csv') #\n",
    "df33 = pd.read_csv('/malicious/Raspberry_WebminePool_Stealthy.csv') #\n",
    "df34 = pd.read_csv('/malicious/Raspberry_WebminePool_Robust.csv') #\n",
    "df35 = pd.read_csv('/malicious/Desktop_WebminePool_Aggressive.csv') #\n",
    "\n",
    "\n",
    "#################################################################\n",
    "#                                                               #\n",
    "#               benign csv files import                         #\n",
    "#                                                               #\n",
    "#################################################################\n",
    "\n",
    "############### LAPTOP #############\n",
    "\n",
    "df8 = pd.read_csv('/benign-2/Laptop/Laptop_download_benign.csv')\n",
    "df9 = pd.read_csv('/benign-2/Laptop/Laptop_idle_benign.csv')\n",
    "df10 = pd.read_csv('/benign-2/Laptop/Laptop_interactive_benign.csv')\n",
    "df11 = pd.read_csv('/benign-2/Laptop/Laptop_video_benign.csv')\n",
    "df12 = pd.read_csv('/benign-2/Laptop/Laptop_webbrowsing_benign.csv')\n",
    "\n",
    "############### Raspberry ##########\n",
    "\n",
    "df13 = pd.read_csv('/benign-2/Raspberry/Raspberry_download_benign.csv')\n",
    "df14 = pd.read_csv('/benign-2/Raspberry/Raspberry_idle_benign.csv')\n",
    "df15 = pd.read_csv('/benign-2/Raspberry/Raspberry_interactive_benign.csv')\n",
    "df16 = pd.read_csv('/benign-2/Raspberry/Raspberry_video_benign.csv')\n",
    "df17 = pd.read_csv('/benign-2/Raspberry/Raspberry_webbrowsing_benign.csv')\n",
    "\n",
    "############### Server ############\n",
    "\n",
    "\n",
    "df18 = pd.read_csv('/benign-2/Server/Server_download_benign.csv')\n",
    "df19 = pd.read_csv('/benign-2/Server/Server_idle_benign.csv')\n",
    "df20 = pd.read_csv('/benign-2/Server/Server_interactive_benign.csv')\n",
    "df21 = pd.read_csv('/benign-2/Server/Server_video_benign.csv')\n",
    "df22 = pd.read_csv('/benign-2/Server/Server_webbrowsing_benign.csv')\n",
    "\n",
    "############### WebOS ############\n",
    "\n",
    "df23 = pd.read_csv('/benign-2/WebOS/Webos_video(live&normal)_benign.csv')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "df_results = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df1 -->> 43173\n",
      "df2 -->> 1213354\n",
      "df3 -->> 3621\n",
      "df4 -->> 22111\n",
      "df5 -->> 14156\n",
      "df6 -->> 24476\n",
      "df7 -->> 3106\n",
      "df32 -->> 18460\n",
      "df33 -->> 10285\n",
      "df34 -->> 7708\n",
      "df35 -->> 234892\n",
      "/////////////////////////////////////////////////\n",
      " LAPTOP \n",
      "df8 -->> 442866\n",
      "df9 -->> 113602\n",
      "df10 -->> 81681\n",
      "df11 -->> 29010\n",
      "df12 -->> 99235\n",
      " Raspberry \n",
      "df13 -->> 276808\n",
      "df14 -->> 73\n",
      "df15 -->> 104241\n",
      "df16 -->> 57205\n",
      "df17 -->> 123298\n",
      " Server \n",
      "df18 -->> 564831\n",
      "df19 -->> 13459\n",
      "df20 -->> 123728\n",
      "df21 -->> 109497\n",
      "df22 -->> 43713\n",
      " WebOS \n",
      "df23 -->> 177704\n"
     ]
    }
   ],
   "source": [
    "print(\"df1 -->> {}\".format(len(df1)))\n",
    "print(\"df2 -->> {}\".format(len(df2)))\n",
    "print(\"df3 -->> {}\".format(len(df3)))\n",
    "print(\"df4 -->> {}\".format(len(df4)))\n",
    "print(\"df5 -->> {}\".format(len(df5)))\n",
    "print(\"df6 -->> {}\".format(len(df6)))\n",
    "print(\"df7 -->> {}\".format(len(df7)))\n",
    "print(\"df32 -->> {}\".format(len(df32)))\n",
    "print(\"df33 -->> {}\".format(len(df33)))\n",
    "print(\"df34 -->> {}\".format(len(df34)))\n",
    "print(\"df35 -->> {}\".format(len(df35)))\n",
    "print(\"/////////////////////////////////////////////////\")\n",
    "\n",
    "print(\" LAPTOP \")\n",
    "\n",
    "\n",
    "print(\"df8 -->> {}\".format(len(df8)))\n",
    "print(\"df9 -->> {}\".format(len(df9)))\n",
    "print(\"df10 -->> {}\".format(len(df10)))\n",
    "print(\"df11 -->> {}\".format(len(df11)))\n",
    "print(\"df12 -->> {}\".format(len(df12)))\n",
    "\n",
    "print(\" Raspberry \")\n",
    "\n",
    "\n",
    "print(\"df13 -->> {}\".format(len(df13)))\n",
    "print(\"df14 -->> {}\".format(len(df14)))\n",
    "print(\"df15 -->> {}\".format(len(df15)))\n",
    "print(\"df16 -->> {}\".format(len(df16)))\n",
    "print(\"df17 -->> {}\".format(len(df17)))\n",
    "\n",
    "print(\" Server \")\n",
    "\n",
    "print(\"df18 -->> {}\".format(len(df18)))\n",
    "print(\"df19 -->> {}\".format(len(df19)))\n",
    "print(\"df20 -->> {}\".format(len(df20)))\n",
    "print(\"df21 -->> {}\".format(len(df21)))\n",
    "print(\"df22 -->> {}\".format(len(df22)))\n",
    "\n",
    "print(\" WebOS \")\n",
    "\n",
    "print(\"df23 -->> {}\".format(len(df23)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prune the datasets for labeling process for malicious data\n",
    "\n",
    "\n",
    "# For WebOS = 18:56:80:17:d0:ef\n",
    "index_names = df1[((df1['HW_dst'] != '18:56:80:17:d0:ef') & (df1['Hw_src'] != '18:56:80:17:d0:ef'))].index\n",
    "df1.drop(index_names, inplace = True)\n",
    "\n",
    "# Big_Server_Monero_mining_data = a4:bb:6d:ac:e1:fd\n",
    "\n",
    "index_names = df2[((df2['HW_dst'] != 'a4:bb:6d:ac:e1:fd') & (df2['Hw_src'] != 'a4:bb:6d:ac:e1:fd'))].index\n",
    "df2.drop(index_names, inplace = True)\n",
    "\n",
    "# ege_data_rasberry = dc:a6:32:67:66:4b\t\n",
    "\n",
    "index_names = df3[((df3['HW_dst'] != 'dc:a6:32:67:66:4b') & (df3['Hw_src'] != 'dc:a6:32:67:66:4b'))].index\n",
    "df3.drop(index_names, inplace = True)\n",
    "\n",
    "# Rasberry_binary_monero_mining = dc:a6:32:68:35:8a\n",
    "\n",
    "index_names = df4[((df4['HW_dst'] != 'dc:a6:32:68:35:8a') & (df4['Hw_src'] != 'dc:a6:32:68:35:8a'))].index\n",
    "df4.drop(index_names, inplace = True)\n",
    "\n",
    "# Rasberry_network_data_2 = dc:a6:32:67:66:4b\n",
    "\n",
    "index_names = df5[((df5['HW_dst'] != 'dc:a6:32:67:66:4b') & (df5['Hw_src'] != 'dc:a6:32:67:66:4b'))].index\n",
    "df5.drop(index_names, inplace = True)\n",
    "\n",
    "# Rasberry-Webmine = dc:a6:32:67:66:4b\n",
    "index_names = df6[((df6['HW_dst'] != 'dc:a6:32:67:66:4b') & (df6['Hw_src'] != 'dc:a6:32:67:66:4b'))].index\n",
    "df6.drop(index_names, inplace = True)\n",
    "\n",
    "# Server_Webmine_Network_data = a4:bb:6d:ac:e1:fd\n",
    "\n",
    "index_names = df7[((df7['HW_dst'] != 'a4:bb:6d:ac:e1:fd') & (df7['Hw_src'] != 'a4:bb:6d:ac:e1:fd'))].index\n",
    "df7.drop(index_names, inplace = True)\n",
    "\n",
    "# Server_%50_Mining = a4:bb:6d:ac:e1:fd\n",
    "\n",
    "index_names = df32[((df32['HW_dst'] != 'a4:bb:6d:ac:e1:fd') & (df32['Hw_src'] != 'a4:bb:6d:ac:e1:fd'))].index\n",
    "df32.drop(index_names, inplace = True)\n",
    "\n",
    "# Rasberry_webmine_%10 = dc:a6:32:67:66:4b\n",
    "\n",
    "index_names = df33[((df33['HW_dst'] != 'dc:a6:32:67:66:4b') & (df33['Hw_src'] != 'dc:a6:32:67:66:4b'))].index\n",
    "df33.drop(index_names, inplace = True)\n",
    "\n",
    "# Rasberry_webmine_%50 = dc:a6:32:68:35:8a\n",
    "\n",
    "index_names = df34[((df34['HW_dst'] != 'dc:a6:32:68:35:8a') & (df34['Hw_src'] != 'dc:a6:32:68:35:8a'))].index\n",
    "df34.drop(index_names, inplace = True)\n",
    "\n",
    "# Desktop_Webmine_%100 = dc:a6:32:68:35:8a\n",
    "\n",
    "index_names = df35[((df35['HW_dst'] != 'd8:3b:bf:8f:ba:ba') & (df35['Hw_src'] != 'd8:3b:bf:8f:ba:ba'))].index\n",
    "df35.drop(index_names, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#################################################################\n",
    "#                                                               #\n",
    "#      Labeling Features for further calculations               #\n",
    "#                                                               #\n",
    "#################################################################\n",
    "\n",
    "df1.insert(7, \"Is_malicious\", 1)\n",
    "df2.insert(7, \"Is_malicious\", 1)\n",
    "df3.insert(7, \"Is_malicious\", 1)\n",
    "df4.insert(7, \"Is_malicious\", 1)\n",
    "df5.insert(7, \"Is_malicious\", 1)\n",
    "df6.insert(7, \"Is_malicious\", 1)\n",
    "df7.insert(7, \"Is_malicious\", 1)\n",
    "\n",
    "# ========================================================\n",
    "\n",
    "df8.insert(7, \"Is_malicious\", 0)\n",
    "df9.insert(7, \"Is_malicious\", 0)\n",
    "df10.insert(7, \"Is_malicious\", 0)\n",
    "df11.insert(7, \"Is_malicious\", 0)\n",
    "df12.insert(7, \"Is_malicious\", 0)\n",
    "df13.insert(7, \"Is_malicious\", 0)\n",
    "df14.insert(7, \"Is_malicious\", 0)\n",
    "df15.insert(7, \"Is_malicious\", 0)\n",
    "df16.insert(7, \"Is_malicious\", 0)\n",
    "df17.insert(7, \"Is_malicious\", 0)\n",
    "df18.insert(7, \"Is_malicious\", 0)\n",
    "df19.insert(7, \"Is_malicious\", 0)\n",
    "df20.insert(7, \"Is_malicious\", 0)\n",
    "df21.insert(7, \"Is_malicious\", 0)\n",
    "df22.insert(7, \"Is_malicious\", 0)\n",
    "df23.insert(7, \"Is_malicious\", 0)\n",
    "\n",
    "# ========================================================\n",
    "\n",
    "\n",
    "df32.insert(7, \"Is_malicious\", 1)\n",
    "df33.insert(7, \"Is_malicious\", 1)\n",
    "df34.insert(7, \"Is_malicious\", 1)\n",
    "df35.insert(7, \"Is_malicious\", 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_process(a,b,c,d,x):\n",
    "    #run_process(df_malicious_Train,df_benign_Train,df_malicious_Test,df_benign_Test,df_results)\n",
    " \n",
    "  \n",
    "    df_malicious_Train = a.copy()\n",
    "    df_benign_Train    = b.copy()\n",
    "    \n",
    "    df_malicious_Test = c.copy()\n",
    "    df_benign_Test    = d.copy()\n",
    "    \n",
    "    \n",
    "    \n",
    "    from tsfresh import extract_features, select_features\n",
    "    from tsfresh.utilities.dataframe_functions import impute\n",
    "    from tsfresh import extract_features\n",
    "    from tsfresh.feature_selection.relevance import calculate_relevance_table\n",
    "\n",
    "\n",
    "    df_malicious_Train.reset_index(drop=True, inplace=True) #reset index\n",
    "    df_malicious_Train['id']= np.floor(df_malicious_Train.index.array/10)\n",
    "    \n",
    "    df_malicious_Test.reset_index(drop=True, inplace=True) #reset index\n",
    "    df_malicious_Test['id']= np.floor(df_malicious_Test.index.array/10)\n",
    "    \n",
    "    \n",
    "    df_benign_Train.reset_index(drop=True, inplace=True) #reset index\n",
    "    df_benign_Train['id']= np.floor(df_benign_Train.index.array/10)\n",
    "    \n",
    "    df_benign_Test.reset_index(drop=True, inplace=True) #reset index\n",
    "    df_benign_Test['id']= np.floor(df_benign_Test.index.array/10)\n",
    "    \n",
    "\n",
    "\n",
    "    tf1=tsfresh.extract_features(df_malicious_Train,impute_function=impute, column_kind='Is_malicious',\n",
    "                                 column_id='id',column_sort=\"Time\",column_value = \"Length\")\n",
    "    tf1['class']= 1\n",
    "    \n",
    "    tf1_1=tsfresh.extract_features(df_malicious_Test,impute_function=impute, column_kind='Is_malicious',\n",
    "                                 column_id='id',column_sort=\"Time\",column_value = \"Length\")\n",
    "    tf1_1['class']= 1\n",
    "\n",
    "      \n",
    "    tf2=tsfresh.extract_features(df_benign_Train,impute_function=impute, column_kind='Is_malicious',\n",
    "                                 column_id='id',column_sort=\"Time\",column_value = \"Length\")\n",
    "    tf2['class']= 0\n",
    "    \n",
    "    tf2_1=tsfresh.extract_features(df_benign_Test,impute_function=impute, column_kind='Is_malicious',\n",
    "                                 column_id='id',column_sort=\"Time\",column_value = \"Length\")\n",
    "    tf2_1['class']= 0\n",
    "    \n",
    "\n",
    "    tf2.columns = tf1.columns\n",
    "    tf2_1.columns = tf1_1.columns\n",
    "    \n",
    "\n",
    "    features_train=pd.concat([tf1,tf2])\n",
    "    features_test=pd.concat([tf1_1,tf2_1])\n",
    "    \n",
    "    \n",
    "    features2_train = features_train.copy()\n",
    "    features2_train.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "\n",
    "    \n",
    "    y_train = pd.Series(data = features2_train['class'], index=features2_train.index)\n",
    "    \n",
    "    relevance_table_train = calculate_relevance_table(features2_train, y_train)\n",
    "    \n",
    "\n",
    "    relevance_table_train = relevance_table_train[relevance_table_train.relevant]\n",
    "    relevance_table_train.sort_values(\"p_value\", inplace=True)\n",
    "    best_features_train = relevance_table_train[relevance_table_train['p_value'] <= 0.05] \n",
    "    \n",
    "    \n",
    "    df_ML_train = pd.DataFrame()\n",
    "    df_ML_test  = pd.DataFrame()\n",
    "    \n",
    "\n",
    "    df_ML_train[best_features_train.feature] = features_train[best_features_train.feature]\n",
    "\n",
    "    df_ML_test[best_features_train.feature] = features_test[best_features_train.feature]\n",
    "\n",
    "\n",
    "    final = ML_Process(df_ML_train,df_ML_test,x)\n",
    "    \n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ML_Process(df_ML_train,df_ML_test,x):\n",
    "    df_results = x.copy() \n",
    "    print('let the ml starts')\n",
    "  \n",
    "    from sklearn import neighbors, metrics\n",
    "    from sklearn.preprocessing import LabelEncoder\n",
    "    \n",
    "\n",
    "    X_train = df_ML_train.drop('class',axis=1).to_numpy()\n",
    "    y_train = df_ML_train['class'].to_numpy()\n",
    "\n",
    "    X_test = df_ML_test.drop('class',axis=1).to_numpy()\n",
    "    y_test = df_ML_test['class'].to_numpy()\n",
    "\n",
    "    \n",
    "    from sklearn.model_selection import train_test_split\n",
    "    Le = LabelEncoder()\n",
    "    for i in range(len(X_train[0])):\n",
    "        X_train[:, i] = Le.fit_transform(X_train[:, i])\n",
    "    for i in range(len(X_train[0])):\n",
    "        X_test[:, i] = Le.fit_transform(X_test[:, i])\n",
    "\n",
    "\n",
    "    #X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=8675309)\n",
    "\n",
    "    \n",
    "    from sklearn.linear_model import LogisticRegression\n",
    "    from sklearn.neighbors import KNeighborsClassifier\n",
    "    from sklearn.svm import SVC\n",
    "    from sklearn.ensemble import RandomForestClassifier\n",
    "    from sklearn.naive_bayes import GaussianNB\n",
    "    #from xgboost import XGBClassifier\n",
    "    from sklearn import model_selection\n",
    "    from sklearn.utils import class_weight\n",
    "    from sklearn.metrics import classification_report\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    import numpy as np\n",
    "    import pandas as pd\n",
    "    y_train = y_train.ravel()\n",
    "    dfs = []\n",
    "    models = [\n",
    "          ('LogReg', LogisticRegression()), \n",
    "          #('RF', RandomForestClassifier()),\n",
    "          ('KNN', KNeighborsClassifier()),\n",
    "          ('SVM', SVC()), \n",
    "          ('GNB', GaussianNB())\n",
    "          #('XGB', XGBClassifier())\n",
    "            ]\n",
    "    results = []\n",
    "    names = []\n",
    "    scoring = ['accuracy', 'precision_weighted', 'recall_weighted', 'f1_weighted', 'roc_auc']\n",
    "    target_names = ['malignant', 'benign']\n",
    "    for name, model in models:\n",
    "        kfold = model_selection.KFold(n_splits=5, shuffle=True, random_state=90210)\n",
    "        cv_results = model_selection.cross_validate(model, X_train, y_train, cv=kfold, \n",
    "                                                    scoring=scoring)\n",
    "\n",
    "        clf = model.fit(X_train, y_train)\n",
    "\n",
    "        y_pred = clf.predict(X_test)\n",
    "        print(name)\n",
    "        print(classification_report(y_test, y_pred, target_names=target_names))\n",
    "        results.append(cv_results)\n",
    "        names.append(name)\n",
    "        this_df = pd.DataFrame(cv_results)\n",
    "        this_df['model'] = name\n",
    "        dfs.append(this_df)\n",
    "        df_resulta = df_results.append(dfs)\n",
    "        final = pd.concat(dfs, ignore_index=True)\n",
    "        print(final)\n",
    "\n",
    "\n",
    "    return(final)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malicious_train: 12871\n",
      "malicious_test: 19643\n",
      "benign_train: 450000\n",
      "benign_test: 111625\n",
      "After droppping NAN rows: \n",
      "malicious_train: 12871\n",
      "malicious_test: 19643\n",
      "benign_train: 450000\n",
      "benign_test: 111625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Feature Extraction: 100%|█████████████████████| 144/144 [00:02<00:00, 48.73it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 152/152 [00:03<00:00, 38.62it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:05<00:00,  2.45it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [00:18<00:00,  8.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "let the ml starts\n",
      "LogReg\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.97      0.99      0.98     11163\n",
      "      benign       0.92      0.81      0.86      1965\n",
      "\n",
      "    accuracy                           0.96     13128\n",
      "   macro avg       0.95      0.90      0.92     13128\n",
      "weighted avg       0.96      0.96      0.96     13128\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.032351    0.023884       0.992979                 0.992775   \n",
      "1  1.108005    0.026720       0.992223                 0.991944   \n",
      "2  0.846927    0.021305       0.993087                 0.992883   \n",
      "3  1.027660    0.021268       0.993410                 0.993221   \n",
      "4  1.035879    0.021311       0.992330                 0.992141   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.992979          0.992815      0.992410  LogReg  \n",
      "1              0.992223          0.992001      0.981768  LogReg  \n",
      "2              0.993087          0.992789      0.990833  LogReg  \n",
      "3              0.993410          0.993261      0.990344  LogReg  \n",
      "4              0.992330          0.992213      0.984112  LogReg  \n",
      "KNN\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.93      0.97      0.95     11163\n",
      "      benign       0.80      0.60      0.68      1965\n",
      "\n",
      "    accuracy                           0.92     13128\n",
      "   macro avg       0.87      0.79      0.82     13128\n",
      "weighted avg       0.91      0.92      0.91     13128\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.032351    0.023884       0.992979                 0.992775   \n",
      "1  1.108005    0.026720       0.992223                 0.991944   \n",
      "2  0.846927    0.021305       0.993087                 0.992883   \n",
      "3  1.027660    0.021268       0.993410                 0.993221   \n",
      "4  1.035879    0.021311       0.992330                 0.992141   \n",
      "5  0.072494   13.763048       0.986390                 0.985475   \n",
      "6  0.060058   13.312601       0.985310                 0.984344   \n",
      "7  0.060759   13.490714       0.987254                 0.986374   \n",
      "8  0.060975   13.630177       0.985308                 0.984434   \n",
      "9  0.063286   13.943345       0.987253                 0.986448   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.992979          0.992815      0.992410  LogReg  \n",
      "1              0.992223          0.992001      0.981768  LogReg  \n",
      "2              0.993087          0.992789      0.990833  LogReg  \n",
      "3              0.993410          0.993261      0.990344  LogReg  \n",
      "4              0.992330          0.992213      0.984112  LogReg  \n",
      "5              0.986390          0.985634      0.955271     KNN  \n",
      "6              0.985310          0.984662      0.934438     KNN  \n",
      "7              0.987254          0.986246      0.948531     KNN  \n",
      "8              0.985308          0.984740      0.947253     KNN  \n",
      "9              0.987253          0.986673      0.947472     KNN  \n",
      "SVM\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.98      0.86      0.92     11163\n",
      "      benign       0.53      0.91      0.67      1965\n",
      "\n",
      "    accuracy                           0.87     13128\n",
      "   macro avg       0.76      0.89      0.79     13128\n",
      "weighted avg       0.92      0.87      0.88     13128\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.032351    0.023884       0.992979                 0.992775   \n",
      "1    1.108005    0.026720       0.992223                 0.991944   \n",
      "2    0.846927    0.021305       0.993087                 0.992883   \n",
      "3    1.027660    0.021268       0.993410                 0.993221   \n",
      "4    1.035879    0.021311       0.992330                 0.992141   \n",
      "5    0.072494   13.763048       0.986390                 0.985475   \n",
      "6    0.060058   13.312601       0.985310                 0.984344   \n",
      "7    0.060759   13.490714       0.987254                 0.986374   \n",
      "8    0.060975   13.630177       0.985308                 0.984434   \n",
      "9    0.063286   13.943345       0.987253                 0.986448   \n",
      "10  13.890523    9.130271       0.987578                 0.987007   \n",
      "11  14.499660    9.273320       0.987470                 0.986739   \n",
      "12  14.051334    9.344368       0.987902                 0.987270   \n",
      "13  14.868187    9.345928       0.987901                 0.987117   \n",
      "14  14.548804    9.235632       0.988765                 0.988092   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.992979          0.992815      0.992410  LogReg  \n",
      "1               0.992223          0.992001      0.981768  LogReg  \n",
      "2               0.993087          0.992789      0.990833  LogReg  \n",
      "3               0.993410          0.993261      0.990344  LogReg  \n",
      "4               0.992330          0.992213      0.984112  LogReg  \n",
      "5               0.986390          0.985634      0.955271     KNN  \n",
      "6               0.985310          0.984662      0.934438     KNN  \n",
      "7               0.987254          0.986246      0.948531     KNN  \n",
      "8               0.985308          0.984740      0.947253     KNN  \n",
      "9               0.987253          0.986673      0.947472     KNN  \n",
      "10              0.987578          0.986399      0.923347     SVM  \n",
      "11              0.987470          0.986217      0.917140     SVM  \n",
      "12              0.987902          0.986754      0.925785     SVM  \n",
      "13              0.987901          0.987036      0.915624     SVM  \n",
      "14              0.988765          0.987885      0.933666     SVM  \n",
      "GNB\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.85      1.00      0.92     11163\n",
      "      benign       0.00      0.00      0.00      1965\n",
      "\n",
      "    accuracy                           0.85     13128\n",
      "   macro avg       0.43      0.50      0.46     13128\n",
      "weighted avg       0.72      0.85      0.78     13128\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.032351    0.023884       0.992979                 0.992775   \n",
      "1    1.108005    0.026720       0.992223                 0.991944   \n",
      "2    0.846927    0.021305       0.993087                 0.992883   \n",
      "3    1.027660    0.021268       0.993410                 0.993221   \n",
      "4    1.035879    0.021311       0.992330                 0.992141   \n",
      "5    0.072494   13.763048       0.986390                 0.985475   \n",
      "6    0.060058   13.312601       0.985310                 0.984344   \n",
      "7    0.060759   13.490714       0.987254                 0.986374   \n",
      "8    0.060975   13.630177       0.985308                 0.984434   \n",
      "9    0.063286   13.943345       0.987253                 0.986448   \n",
      "10  13.890523    9.130271       0.987578                 0.987007   \n",
      "11  14.499660    9.273320       0.987470                 0.986739   \n",
      "12  14.051334    9.344368       0.987902                 0.987270   \n",
      "13  14.868187    9.345928       0.987901                 0.987117   \n",
      "14  14.548804    9.235632       0.988765                 0.988092   \n",
      "15   0.201739    0.053596       0.999568                 0.999568   \n",
      "16   0.196661    0.051596       0.999352                 0.999352   \n",
      "17   0.201228    0.053146       0.999352                 0.999352   \n",
      "18   0.199077    0.054044       0.999244                 0.999244   \n",
      "19   0.200549    0.054707       0.999676                 0.999676   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.992979          0.992815      0.992410  LogReg  \n",
      "1               0.992223          0.992001      0.981768  LogReg  \n",
      "2               0.993087          0.992789      0.990833  LogReg  \n",
      "3               0.993410          0.993261      0.990344  LogReg  \n",
      "4               0.992330          0.992213      0.984112  LogReg  \n",
      "5               0.986390          0.985634      0.955271     KNN  \n",
      "6               0.985310          0.984662      0.934438     KNN  \n",
      "7               0.987254          0.986246      0.948531     KNN  \n",
      "8               0.985308          0.984740      0.947253     KNN  \n",
      "9               0.987253          0.986673      0.947472     KNN  \n",
      "10              0.987578          0.986399      0.923347     SVM  \n",
      "11              0.987470          0.986217      0.917140     SVM  \n",
      "12              0.987902          0.986754      0.925785     SVM  \n",
      "13              0.987901          0.987036      0.915624     SVM  \n",
      "14              0.988765          0.987885      0.933666     SVM  \n",
      "15              0.999568          0.999566      1.000000     GNB  \n",
      "16              0.999352          0.999348      0.996047     GNB  \n",
      "17              0.999352          0.999348      1.000000     GNB  \n",
      "18              0.999244          0.999239      1.000000     GNB  \n",
      "19              0.999676          0.999675      0.997967     GNB  \n",
      "411.2405991100004\n"
     ]
    }
   ],
   "source": [
    "## Train: Webmine, Test: WebminePool\n",
    " \n",
    "df_malicious_Train = pd.concat([df5])\n",
    "df_malicious_Test = pd.concat([df6])\n",
    "\n",
    "\n",
    "df_benign_Train = pd.concat([df13,df14,df15,df16,df17])[:450000]\n",
    "df_benign_Test = pd.concat([df13,df14,df15,df16,df17])[450000:]\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "#print(\"{} NAN in malicious!\".format(len(df_malicious[df_malicious.isna().any(axis=1)])))\n",
    "#print(\"{} NAN in benign!\".format(len(df_benign[df_benign.isna().any(axis=1)])))\n",
    "\n",
    "df_malicious_Test = df_malicious_Test.dropna()\n",
    "df_malicious_Train = df_malicious_Train.dropna()\n",
    "\n",
    "df_benign_Test = df_benign_Test.dropna()\n",
    "df_benign_Train = df_benign_Train.dropna()\n",
    "\n",
    "print(\"After droppping NAN rows: \")\n",
    "\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "start = timer()\n",
    "\n",
    "results_all_combined_s0 = run_process(df_malicious_Train,df_benign_Train,df_malicious_Test,df_benign_Test,df_results)\n",
    " \n",
    "end = timer()\n",
    "print(end - start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malicious_train: 19643\n",
      "malicious_test: 12871\n",
      "benign_train: 450000\n",
      "benign_test: 111625\n",
      "After droppping NAN rows: \n",
      "malicious_train: 19643\n",
      "malicious_test: 12871\n",
      "benign_train: 450000\n",
      "benign_test: 111625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Feature Extraction: 100%|█████████████████████| 152/152 [00:03<00:00, 38.83it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 144/144 [00:02<00:00, 55.50it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:04<00:00,  2.50it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [00:17<00:00,  9.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "let the ml starts\n",
      "LogReg\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.98      0.98      0.98     11163\n",
      "      benign       0.82      0.85      0.83      1288\n",
      "\n",
      "    accuracy                           0.97     12451\n",
      "   macro avg       0.90      0.91      0.91     12451\n",
      "weighted avg       0.97      0.97      0.97     12451\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.059858    0.022082       0.991589                 0.991401   \n",
      "1  1.116981    0.021914       0.992441                 0.992275   \n",
      "2  1.090642    0.021538       0.991696                 0.991504   \n",
      "3  1.115772    0.021677       0.991802                 0.991625   \n",
      "4  1.106091    0.021537       0.991057                 0.990843   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.991589          0.991439      0.992651  LogReg  \n",
      "1              0.992441          0.992248      0.995495  LogReg  \n",
      "2              0.991696          0.991530      0.990672  LogReg  \n",
      "3              0.991802          0.991540      0.995289  LogReg  \n",
      "4              0.991057          0.990896      0.992284  LogReg  \n",
      "KNN\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.97      0.94      0.95     11163\n",
      "      benign       0.57      0.71      0.63      1288\n",
      "\n",
      "    accuracy                           0.91     12451\n",
      "   macro avg       0.77      0.82      0.79     12451\n",
      "weighted avg       0.92      0.91      0.92     12451\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.059858    0.022082       0.991589                 0.991401   \n",
      "1  1.116981    0.021914       0.992441                 0.992275   \n",
      "2  1.090642    0.021538       0.991696                 0.991504   \n",
      "3  1.115772    0.021677       0.991802                 0.991625   \n",
      "4  1.106091    0.021537       0.991057                 0.990843   \n",
      "5  0.091391   14.013949       0.982221                 0.981293   \n",
      "6  0.069166   14.513196       0.981263                 0.980386   \n",
      "7  0.068184   14.355649       0.979559                 0.978249   \n",
      "8  0.068525   14.489534       0.980411                 0.979494   \n",
      "9  0.068572   14.530381       0.980304                 0.979746   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.991589          0.991439      0.992651  LogReg  \n",
      "1              0.992441          0.992248      0.995495  LogReg  \n",
      "2              0.991696          0.991530      0.990672  LogReg  \n",
      "3              0.991802          0.991540      0.995289  LogReg  \n",
      "4              0.991057          0.990896      0.992284  LogReg  \n",
      "5              0.982221          0.981521      0.959477     KNN  \n",
      "6              0.981263          0.980693      0.958469     KNN  \n",
      "7              0.979559          0.978559      0.935885     KNN  \n",
      "8              0.980411          0.979824      0.954742     KNN  \n",
      "9              0.980304          0.979989      0.963215     KNN  \n",
      "SVM\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       1.00      0.65      0.79     11163\n",
      "      benign       0.25      0.98      0.39      1288\n",
      "\n",
      "    accuracy                           0.69     12451\n",
      "   macro avg       0.62      0.82      0.59     12451\n",
      "weighted avg       0.92      0.69      0.75     12451\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.059858    0.022082       0.991589                 0.991401   \n",
      "1    1.116981    0.021914       0.992441                 0.992275   \n",
      "2    1.090642    0.021538       0.991696                 0.991504   \n",
      "3    1.115772    0.021677       0.991802                 0.991625   \n",
      "4    1.106091    0.021537       0.991057                 0.990843   \n",
      "5    0.091391   14.013949       0.982221                 0.981293   \n",
      "6    0.069166   14.513196       0.981263                 0.980386   \n",
      "7    0.068184   14.355649       0.979559                 0.978249   \n",
      "8    0.068525   14.489534       0.980411                 0.979494   \n",
      "9    0.068572   14.530381       0.980304                 0.979746   \n",
      "10  18.254371   10.409879       0.986799                 0.986265   \n",
      "11  18.968680   10.278474       0.985415                 0.984743   \n",
      "12  19.058550   10.121701       0.984989                 0.984357   \n",
      "13  18.624266   10.319038       0.984244                 0.983494   \n",
      "14  18.648865   10.988832       0.984137                 0.983289   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.991589          0.991439      0.992651  LogReg  \n",
      "1               0.992441          0.992248      0.995495  LogReg  \n",
      "2               0.991696          0.991530      0.990672  LogReg  \n",
      "3               0.991802          0.991540      0.995289  LogReg  \n",
      "4               0.991057          0.990896      0.992284  LogReg  \n",
      "5               0.982221          0.981521      0.959477     KNN  \n",
      "6               0.981263          0.980693      0.958469     KNN  \n",
      "7               0.979559          0.978559      0.935885     KNN  \n",
      "8               0.980411          0.979824      0.954742     KNN  \n",
      "9               0.980304          0.979989      0.963215     KNN  \n",
      "10              0.986799          0.986251      0.969953     SVM  \n",
      "11              0.985415          0.984547      0.960985     SVM  \n",
      "12              0.984989          0.984017      0.966017     SVM  \n",
      "13              0.984244          0.983157      0.954668     SVM  \n",
      "14              0.984137          0.983341      0.974065     SVM  \n",
      "GNB\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.94      1.00      0.97     11163\n",
      "      benign       0.95      0.44      0.60      1288\n",
      "\n",
      "    accuracy                           0.94     12451\n",
      "   macro avg       0.95      0.72      0.78     12451\n",
      "weighted avg       0.94      0.94      0.93     12451\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.059858    0.022082       0.991589                 0.991401   \n",
      "1    1.116981    0.021914       0.992441                 0.992275   \n",
      "2    1.090642    0.021538       0.991696                 0.991504   \n",
      "3    1.115772    0.021677       0.991802                 0.991625   \n",
      "4    1.106091    0.021537       0.991057                 0.990843   \n",
      "5    0.091391   14.013949       0.982221                 0.981293   \n",
      "6    0.069166   14.513196       0.981263                 0.980386   \n",
      "7    0.068184   14.355649       0.979559                 0.978249   \n",
      "8    0.068525   14.489534       0.980411                 0.979494   \n",
      "9    0.068572   14.530381       0.980304                 0.979746   \n",
      "10  18.254371   10.409879       0.986799                 0.986265   \n",
      "11  18.968680   10.278474       0.985415                 0.984743   \n",
      "12  19.058550   10.121701       0.984989                 0.984357   \n",
      "13  18.624266   10.319038       0.984244                 0.983494   \n",
      "14  18.648865   10.988832       0.984137                 0.983289   \n",
      "15   0.207602    0.057903       0.998829                 0.998845   \n",
      "16   0.205943    0.051379       0.999681                 0.999681   \n",
      "17   0.204634    0.053855       0.999042                 0.999046   \n",
      "18   0.209098    0.050953       0.999042                 0.999063   \n",
      "19   0.206887    0.050731       0.998935                 0.998943   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.991589          0.991439      0.992651  LogReg  \n",
      "1               0.992441          0.992248      0.995495  LogReg  \n",
      "2               0.991696          0.991530      0.990672  LogReg  \n",
      "3               0.991802          0.991540      0.995289  LogReg  \n",
      "4               0.991057          0.990896      0.992284  LogReg  \n",
      "5               0.982221          0.981521      0.959477     KNN  \n",
      "6               0.981263          0.980693      0.958469     KNN  \n",
      "7               0.979559          0.978559      0.935885     KNN  \n",
      "8               0.980411          0.979824      0.954742     KNN  \n",
      "9               0.980304          0.979989      0.963215     KNN  \n",
      "10              0.986799          0.986251      0.969953     SVM  \n",
      "11              0.985415          0.984547      0.960985     SVM  \n",
      "12              0.984989          0.984017      0.966017     SVM  \n",
      "13              0.984244          0.983157      0.954668     SVM  \n",
      "14              0.984137          0.983341      0.974065     SVM  \n",
      "15              0.998829          0.998834      0.999496     GNB  \n",
      "16              0.999681          0.999681      0.998649     GNB  \n",
      "17              0.999042          0.999044      0.998414     GNB  \n",
      "18              0.999042          0.999047      0.999556     GNB  \n",
      "19              0.998935          0.998938      0.999608     GNB  \n",
      "453.5430011240005\n"
     ]
    }
   ],
   "source": [
    "## Train: WebminePool, Test: Webmine\n",
    " \n",
    "df_malicious_Train = pd.concat([df6])\n",
    "df_malicious_Test = pd.concat([df5])\n",
    "\n",
    "\n",
    "df_benign_Train = pd.concat([df13,df14,df15,df16,df17])[:450000]\n",
    "df_benign_Test = pd.concat([df13,df14,df15,df16,df17])[450000:]\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "#print(\"{} NAN in malicious!\".format(len(df_malicious[df_malicious.isna().any(axis=1)])))\n",
    "#print(\"{} NAN in benign!\".format(len(df_benign[df_benign.isna().any(axis=1)])))\n",
    "\n",
    "df_malicious_Test = df_malicious_Test.dropna()\n",
    "df_malicious_Train = df_malicious_Train.dropna()\n",
    "\n",
    "df_benign_Test = df_benign_Test.dropna()\n",
    "df_benign_Train = df_benign_Train.dropna()\n",
    "\n",
    "print(\"After droppping NAN rows: \")\n",
    "\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "start = timer()\n",
    "\n",
    "results_all_combined_s0 = run_process(df_malicious_Train,df_benign_Train,df_malicious_Test,df_benign_Test,df_results)\n",
    " \n",
    "end = timer()\n",
    "print(end - start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malicious_train: 11745\n",
      "malicious_test: 41572\n",
      "benign_train: 284817\n",
      "benign_test: 284817\n",
      "After droppping NAN rows: \n",
      "malicious_train: 11745\n",
      "malicious_test: 41572\n",
      "benign_train: 284817\n",
      "benign_test: 284817\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Feature Extraction: 100%|█████████████████████| 147/147 [00:02<00:00, 59.63it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [00:08<00:00, 19.93it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [00:41<00:00,  3.89it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [00:41<00:00,  3.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "let the ml starts\n",
      "LogReg\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.88      1.00      0.93     28482\n",
      "      benign       0.57      0.02      0.05      4158\n",
      "\n",
      "    accuracy                           0.87     32640\n",
      "   macro avg       0.72      0.51      0.49     32640\n",
      "weighted avg       0.84      0.87      0.82     32640\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  0.670878    0.016850       0.995448                 0.995507   \n",
      "1  0.689840    0.015579       0.995954                 0.995939   \n",
      "2  0.664199    0.015765       0.994773                 0.994881   \n",
      "3  0.642392    0.015523       0.995448                 0.995504   \n",
      "4  0.653812    0.015522       0.996291                 0.996291   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.995448          0.995473      0.999135  LogReg  \n",
      "1              0.995954          0.995946      0.998848  LogReg  \n",
      "2              0.994773          0.994815      0.998983  LogReg  \n",
      "3              0.995448          0.995471      0.999215  LogReg  \n",
      "4              0.996291          0.996291      0.999594  LogReg  \n",
      "KNN\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.88      1.00      0.93     28482\n",
      "      benign       0.72      0.03      0.05      4158\n",
      "\n",
      "    accuracy                           0.87     32640\n",
      "   macro avg       0.80      0.51      0.49     32640\n",
      "weighted avg       0.86      0.87      0.82     32640\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  0.670878    0.016850       0.995448                 0.995507   \n",
      "1  0.689840    0.015579       0.995954                 0.995939   \n",
      "2  0.664199    0.015765       0.994773                 0.994881   \n",
      "3  0.642392    0.015523       0.995448                 0.995504   \n",
      "4  0.653812    0.015522       0.996291                 0.996291   \n",
      "5  0.058219    5.663634       0.992077                 0.992456   \n",
      "6  0.044289    5.645517       0.991571                 0.991812   \n",
      "7  0.044602    5.552185       0.987692                 0.988020   \n",
      "8  0.044366    5.773202       0.991570                 0.991687   \n",
      "9  0.044281    5.728148       0.991401                 0.991692   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.995448          0.995473      0.999135  LogReg  \n",
      "1              0.995954          0.995946      0.998848  LogReg  \n",
      "2              0.994773          0.994815      0.998983  LogReg  \n",
      "3              0.995448          0.995471      0.999215  LogReg  \n",
      "4              0.996291          0.996291      0.999594  LogReg  \n",
      "5              0.992077          0.992218      0.982661     KNN  \n",
      "6              0.991571          0.991669      0.984087     KNN  \n",
      "7              0.987692          0.987833      0.978624     KNN  \n",
      "8              0.991570          0.991622      0.980887     KNN  \n",
      "9              0.991401          0.991519      0.991423     KNN  \n",
      "SVM\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.87      1.00      0.93     28482\n",
      "      benign       0.57      0.01      0.01      4158\n",
      "\n",
      "    accuracy                           0.87     32640\n",
      "   macro avg       0.72      0.50      0.47     32640\n",
      "weighted avg       0.84      0.87      0.81     32640\n",
      "\n",
      "    fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0   0.670878    0.016850       0.995448                 0.995507   \n",
      "1   0.689840    0.015579       0.995954                 0.995939   \n",
      "2   0.664199    0.015765       0.994773                 0.994881   \n",
      "3   0.642392    0.015523       0.995448                 0.995504   \n",
      "4   0.653812    0.015522       0.996291                 0.996291   \n",
      "5   0.058219    5.663634       0.992077                 0.992456   \n",
      "6   0.044289    5.645517       0.991571                 0.991812   \n",
      "7   0.044602    5.552185       0.987692                 0.988020   \n",
      "8   0.044366    5.773202       0.991570                 0.991687   \n",
      "9   0.044281    5.728148       0.991401                 0.991692   \n",
      "10  9.712856    5.157125       0.990560                 0.990279   \n",
      "11  7.840397    4.748241       0.988200                 0.987775   \n",
      "12  7.549857    4.775530       0.983645                 0.982829   \n",
      "13  7.761525    5.202336       0.989209                 0.988831   \n",
      "14  7.754059    5.099404       0.990221                 0.990040   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.995448          0.995473      0.999135  LogReg  \n",
      "1               0.995954          0.995946      0.998848  LogReg  \n",
      "2               0.994773          0.994815      0.998983  LogReg  \n",
      "3               0.995448          0.995471      0.999215  LogReg  \n",
      "4               0.996291          0.996291      0.999594  LogReg  \n",
      "5               0.992077          0.992218      0.982661     KNN  \n",
      "6               0.991571          0.991669      0.984087     KNN  \n",
      "7               0.987692          0.987833      0.978624     KNN  \n",
      "8               0.991570          0.991622      0.980887     KNN  \n",
      "9               0.991401          0.991519      0.991423     KNN  \n",
      "10              0.990560          0.990343      0.996834     SVM  \n",
      "11              0.988200          0.987850      0.996591     SVM  \n",
      "12              0.983645          0.982844      0.996078     SVM  \n",
      "13              0.989209          0.988895      0.996760     SVM  \n",
      "14              0.990221          0.990112      0.997096     SVM  \n",
      "GNB\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.87      1.00      0.93     28482\n",
      "      benign       0.00      0.00      0.00      4158\n",
      "\n",
      "    accuracy                           0.87     32640\n",
      "   macro avg       0.44      0.50      0.47     32640\n",
      "weighted avg       0.76      0.87      0.81     32640\n",
      "\n",
      "    fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0   0.670878    0.016850       0.995448                 0.995507   \n",
      "1   0.689840    0.015579       0.995954                 0.995939   \n",
      "2   0.664199    0.015765       0.994773                 0.994881   \n",
      "3   0.642392    0.015523       0.995448                 0.995504   \n",
      "4   0.653812    0.015522       0.996291                 0.996291   \n",
      "5   0.058219    5.663634       0.992077                 0.992456   \n",
      "6   0.044289    5.645517       0.991571                 0.991812   \n",
      "7   0.044602    5.552185       0.987692                 0.988020   \n",
      "8   0.044366    5.773202       0.991570                 0.991687   \n",
      "9   0.044281    5.728148       0.991401                 0.991692   \n",
      "10  9.712856    5.157125       0.990560                 0.990279   \n",
      "11  7.840397    4.748241       0.988200                 0.987775   \n",
      "12  7.549857    4.775530       0.983645                 0.982829   \n",
      "13  7.761525    5.202336       0.989209                 0.988831   \n",
      "14  7.754059    5.099404       0.990221                 0.990040   \n",
      "15  0.139637    0.039705       1.000000                 1.000000   \n",
      "16  0.139058    0.035460       1.000000                 1.000000   \n",
      "17  0.140881    0.035009       1.000000                 1.000000   \n",
      "18  0.127767    0.035509       1.000000                 1.000000   \n",
      "19  0.127382    0.035117       1.000000                 1.000000   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.995448          0.995473      0.999135  LogReg  \n",
      "1               0.995954          0.995946      0.998848  LogReg  \n",
      "2               0.994773          0.994815      0.998983  LogReg  \n",
      "3               0.995448          0.995471      0.999215  LogReg  \n",
      "4               0.996291          0.996291      0.999594  LogReg  \n",
      "5               0.992077          0.992218      0.982661     KNN  \n",
      "6               0.991571          0.991669      0.984087     KNN  \n",
      "7               0.987692          0.987833      0.978624     KNN  \n",
      "8               0.991570          0.991622      0.980887     KNN  \n",
      "9               0.991401          0.991519      0.991423     KNN  \n",
      "10              0.990560          0.990343      0.996834     SVM  \n",
      "11              0.988200          0.987850      0.996591     SVM  \n",
      "12              0.983645          0.982844      0.996078     SVM  \n",
      "13              0.989209          0.988895      0.996760     SVM  \n",
      "14              0.990221          0.990112      0.997096     SVM  \n",
      "15              1.000000          1.000000      1.000000     GNB  \n",
      "16              1.000000          1.000000      1.000000     GNB  \n",
      "17              1.000000          1.000000      1.000000     GNB  \n",
      "18              1.000000          1.000000      1.000000     GNB  \n",
      "19              1.000000          1.000000      1.000000     GNB  \n",
      "325.581138775\n"
     ]
    }
   ],
   "source": [
    "## Train: Raspberry (server), Test: binary (WebOS)\n",
    " \n",
    "df_malicious_Train = pd.concat([df4])\n",
    "df_malicious_Test = pd.concat([df1])\n",
    "\n",
    "\n",
    "df_benign_Train = pd.concat([df14,df15,df16,df17])\n",
    "df_benign_Test = pd.concat([df14,df15,df16,df17])\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "#print(\"{} NAN in malicious!\".format(len(df_malicious[df_malicious.isna().any(axis=1)])))\n",
    "#print(\"{} NAN in benign!\".format(len(df_benign[df_benign.isna().any(axis=1)])))\n",
    "\n",
    "df_malicious_Test = df_malicious_Test.dropna()\n",
    "df_malicious_Train = df_malicious_Train.dropna()\n",
    "\n",
    "df_benign_Test = df_benign_Test.dropna()\n",
    "df_benign_Train = df_benign_Train.dropna()\n",
    "\n",
    "print(\"After droppping NAN rows: \")\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "start = timer()\n",
    "\n",
    "results_all_combined_s0 = run_process(df_malicious_Train,df_benign_Train,df_malicious_Test,df_benign_Test,df_results)\n",
    " \n",
    "end = timer()\n",
    "print(end - start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malicious_train: 11745\n",
      "malicious_test: 19643\n",
      "benign_train: 450000\n",
      "benign_test: 111625\n",
      "After droppping NAN rows: \n",
      "malicious_train: 11745\n",
      "malicious_test: 19643\n",
      "benign_train: 450000\n",
      "benign_test: 111625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Feature Extraction: 100%|█████████████████████| 147/147 [00:03<00:00, 46.21it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 152/152 [00:04<00:00, 35.07it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:03<00:00,  2.51it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [00:17<00:00,  9.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "let the ml starts\n",
      "LogReg\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.88      1.00      0.93     11163\n",
      "      benign       0.98      0.19      0.32      1965\n",
      "\n",
      "    accuracy                           0.88     13128\n",
      "   macro avg       0.93      0.60      0.63     13128\n",
      "weighted avg       0.89      0.88      0.84     13128\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.020993    0.021719       0.997943                 0.997928   \n",
      "1  1.047020    0.021428       0.997076                 0.997215   \n",
      "2  1.078097    0.021150       0.998159                 0.998156   \n",
      "3  1.093040    0.021149       0.996968                 0.997052   \n",
      "4  1.070723    0.021180       0.998592                 0.998603   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.997943          0.997933      0.999814  LogReg  \n",
      "1              0.997076          0.997121      0.999321  LogReg  \n",
      "2              0.998159          0.998157      0.999597  LogReg  \n",
      "3              0.996968          0.996999      0.999570  LogReg  \n",
      "4              0.998592          0.998597      0.999867  LogReg  \n",
      "KNN\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.88      0.98      0.93     11163\n",
      "      benign       0.74      0.26      0.38      1965\n",
      "\n",
      "    accuracy                           0.88     13128\n",
      "   macro avg       0.81      0.62      0.66     13128\n",
      "weighted avg       0.86      0.88      0.85     13128\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.020993    0.021719       0.997943                 0.997928   \n",
      "1  1.047020    0.021428       0.997076                 0.997215   \n",
      "2  1.078097    0.021150       0.998159                 0.998156   \n",
      "3  1.093040    0.021149       0.996968                 0.997052   \n",
      "4  1.070723    0.021180       0.998592                 0.998603   \n",
      "5  0.090640   13.968492       0.994911                 0.995069   \n",
      "6  0.069077   13.767529       0.995560                 0.995661   \n",
      "7  0.069181   13.971035       0.994153                 0.994293   \n",
      "8  0.068907   14.418993       0.994586                 0.994883   \n",
      "9  0.068923   14.039680       0.993395                 0.993640   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.997943          0.997933      0.999814  LogReg  \n",
      "1              0.997076          0.997121      0.999321  LogReg  \n",
      "2              0.998159          0.998157      0.999597  LogReg  \n",
      "3              0.996968          0.996999      0.999570  LogReg  \n",
      "4              0.998592          0.998597      0.999867  LogReg  \n",
      "5              0.994911          0.994973      0.991176     KNN  \n",
      "6              0.995560          0.995602      0.988139     KNN  \n",
      "7              0.994153          0.994213      0.983898     KNN  \n",
      "8              0.994586          0.994695      0.990281     KNN  \n",
      "9              0.993395          0.993495      0.990249     KNN  \n",
      "SVM\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.90      0.99      0.94     11163\n",
      "      benign       0.86      0.41      0.55      1965\n",
      "\n",
      "    accuracy                           0.90     13128\n",
      "   macro avg       0.88      0.70      0.75     13128\n",
      "weighted avg       0.90      0.90      0.89     13128\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.020993    0.021719       0.997943                 0.997928   \n",
      "1    1.047020    0.021428       0.997076                 0.997215   \n",
      "2    1.078097    0.021150       0.998159                 0.998156   \n",
      "3    1.093040    0.021149       0.996968                 0.997052   \n",
      "4    1.070723    0.021180       0.998592                 0.998603   \n",
      "5    0.090640   13.968492       0.994911                 0.995069   \n",
      "6    0.069077   13.767529       0.995560                 0.995661   \n",
      "7    0.069181   13.971035       0.994153                 0.994293   \n",
      "8    0.068907   14.418993       0.994586                 0.994883   \n",
      "9    0.068923   14.039680       0.993395                 0.993640   \n",
      "10  11.114666    7.101947       0.993611                 0.993736   \n",
      "11  11.253904    7.191035       0.993286                 0.993723   \n",
      "12  11.264299    7.218518       0.995777                 0.995849   \n",
      "13  11.357219    7.070672       0.992095                 0.992511   \n",
      "14  11.331483    7.122416       0.994261                 0.994516   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.997943          0.997933      0.999814  LogReg  \n",
      "1               0.997076          0.997121      0.999321  LogReg  \n",
      "2               0.998159          0.998157      0.999597  LogReg  \n",
      "3               0.996968          0.996999      0.999570  LogReg  \n",
      "4               0.998592          0.998597      0.999867  LogReg  \n",
      "5               0.994911          0.994973      0.991176     KNN  \n",
      "6               0.995560          0.995602      0.988139     KNN  \n",
      "7               0.994153          0.994213      0.983898     KNN  \n",
      "8               0.994586          0.994695      0.990281     KNN  \n",
      "9               0.993395          0.993495      0.990249     KNN  \n",
      "10              0.993611          0.993666      0.998595     SVM  \n",
      "11              0.993286          0.993449      0.998365     SVM  \n",
      "12              0.995777          0.995807      0.998916     SVM  \n",
      "13              0.992095          0.992262      0.997925     SVM  \n",
      "14              0.994261          0.994360      0.998921     SVM  \n",
      "GNB\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.85      1.00      0.92     11163\n",
      "      benign       0.00      0.00      0.00      1965\n",
      "\n",
      "    accuracy                           0.85     13128\n",
      "   macro avg       0.43      0.50      0.46     13128\n",
      "weighted avg       0.72      0.85      0.78     13128\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.020993    0.021719       0.997943                 0.997928   \n",
      "1    1.047020    0.021428       0.997076                 0.997215   \n",
      "2    1.078097    0.021150       0.998159                 0.998156   \n",
      "3    1.093040    0.021149       0.996968                 0.997052   \n",
      "4    1.070723    0.021180       0.998592                 0.998603   \n",
      "5    0.090640   13.968492       0.994911                 0.995069   \n",
      "6    0.069077   13.767529       0.995560                 0.995661   \n",
      "7    0.069181   13.971035       0.994153                 0.994293   \n",
      "8    0.068907   14.418993       0.994586                 0.994883   \n",
      "9    0.068923   14.039680       0.993395                 0.993640   \n",
      "10  11.114666    7.101947       0.993611                 0.993736   \n",
      "11  11.253904    7.191035       0.993286                 0.993723   \n",
      "12  11.264299    7.218518       0.995777                 0.995849   \n",
      "13  11.357219    7.070672       0.992095                 0.992511   \n",
      "14  11.331483    7.122416       0.994261                 0.994516   \n",
      "15   0.218139    0.061033       1.000000                 1.000000   \n",
      "16   0.212913    0.053934       1.000000                 1.000000   \n",
      "17   0.212575    0.053856       1.000000                 1.000000   \n",
      "18   0.212381    0.054008       1.000000                 1.000000   \n",
      "19   0.212418    0.053848       1.000000                 1.000000   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.997943          0.997933      0.999814  LogReg  \n",
      "1               0.997076          0.997121      0.999321  LogReg  \n",
      "2               0.998159          0.998157      0.999597  LogReg  \n",
      "3               0.996968          0.996999      0.999570  LogReg  \n",
      "4               0.998592          0.998597      0.999867  LogReg  \n",
      "5               0.994911          0.994973      0.991176     KNN  \n",
      "6               0.995560          0.995602      0.988139     KNN  \n",
      "7               0.994153          0.994213      0.983898     KNN  \n",
      "8               0.994586          0.994695      0.990281     KNN  \n",
      "9               0.993395          0.993495      0.990249     KNN  \n",
      "10              0.993611          0.993666      0.998595     SVM  \n",
      "11              0.993286          0.993449      0.998365     SVM  \n",
      "12              0.995777          0.995807      0.998916     SVM  \n",
      "13              0.992095          0.992262      0.997925     SVM  \n",
      "14              0.994261          0.994360      0.998921     SVM  \n",
      "15              1.000000          1.000000      1.000000     GNB  \n",
      "16              1.000000          1.000000      1.000000     GNB  \n",
      "17              1.000000          1.000000      1.000000     GNB  \n",
      "18              1.000000          1.000000      1.000000     GNB  \n",
      "19              1.000000          1.000000      1.000000     GNB  \n",
      "384.0026929760006\n"
     ]
    }
   ],
   "source": [
    "## Train: Binary (raspberry), Test: in-browser (aggressive) (raspberry)\n",
    " \n",
    "df_malicious_Train = pd.concat([df4])\n",
    "df_malicious_Test = pd.concat([df6])\n",
    "\n",
    "df_benign_Train = pd.concat([df13,df14,df15,df16,df17])[:450000]\n",
    "df_benign_Test = pd.concat([df13,df14,df15,df16,df17])[450000:]\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "#print(\"{} NAN in malicious!\".format(len(df_malicious[df_malicious.isna().any(axis=1)])))\n",
    "#print(\"{} NAN in benign!\".format(len(df_benign[df_benign.isna().any(axis=1)])))\n",
    "\n",
    "df_malicious_Test = df_malicious_Test.dropna()\n",
    "df_malicious_Train = df_malicious_Train.dropna()\n",
    "\n",
    "df_benign_Test = df_benign_Test.dropna()\n",
    "df_benign_Train = df_benign_Train.dropna()\n",
    "\n",
    "print(\"After droppping NAN rows: \")\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "start = timer()\n",
    "\n",
    "results_all_combined_s0 = run_process(df_malicious_Train,df_benign_Train,df_malicious_Test,df_benign_Test,df_results)\n",
    " \n",
    "end = timer()\n",
    "print(end - start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malicious_train: 11745\n",
      "malicious_test: 3519\n",
      "benign_train: 561625\n",
      "benign_test: 561625\n",
      "After droppping NAN rows: \n",
      "malicious_train: 11745\n",
      "malicious_test: 3519\n",
      "benign_train: 561625\n",
      "benign_test: 561625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Feature Extraction: 100%|█████████████████████| 147/147 [00:02<00:00, 59.34it/s]\n",
      "Feature Extraction: 100%|████████████████████| 118/118 [00:00<00:00, 144.47it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:23<00:00,  1.92it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:19<00:00,  2.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "let the ml starts\n",
      "LogReg\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.99      1.00      1.00     56163\n",
      "      benign       0.35      0.16      0.22       352\n",
      "\n",
      "    accuracy                           0.99     56515\n",
      "   macro avg       0.67      0.58      0.61     56515\n",
      "weighted avg       0.99      0.99      0.99     56515\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.232812    0.026706       0.997907                 0.997938   \n",
      "1  1.243682    0.025695       0.997384                 0.997352   \n",
      "2  1.412184    0.028950       0.997646                 0.997689   \n",
      "3  1.259485    0.025899       0.996512                 0.996601   \n",
      "4  1.342647    0.025527       0.998692                 0.998695   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.997907          0.997920      0.999684  LogReg  \n",
      "1              0.997384          0.997362      0.999594  LogReg  \n",
      "2              0.997646          0.997663      0.999734  LogReg  \n",
      "3              0.996512          0.996548      0.999050  LogReg  \n",
      "4              0.998692          0.998693      0.999825  LogReg  \n",
      "KNN\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.99      1.00      1.00     56163\n",
      "      benign       0.12      0.04      0.06       352\n",
      "\n",
      "    accuracy                           0.99     56515\n",
      "   macro avg       0.56      0.52      0.53     56515\n",
      "weighted avg       0.99      0.99      0.99     56515\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.232812    0.026706       0.997907                 0.997938   \n",
      "1  1.243682    0.025695       0.997384                 0.997352   \n",
      "2  1.412184    0.028950       0.997646                 0.997689   \n",
      "3  1.259485    0.025899       0.996512                 0.996601   \n",
      "4  1.342647    0.025527       0.998692                 0.998695   \n",
      "5  0.084887   20.717299       0.995204                 0.995302   \n",
      "6  0.081707   21.445543       0.995640                 0.995678   \n",
      "7  0.086478   21.311807       0.995291                 0.995434   \n",
      "8  0.086100   21.055004       0.994157                 0.994309   \n",
      "9  0.083949   21.108251       0.994070                 0.994096   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.997907          0.997920      0.999684  LogReg  \n",
      "1              0.997384          0.997362      0.999594  LogReg  \n",
      "2              0.997646          0.997663      0.999734  LogReg  \n",
      "3              0.996512          0.996548      0.999050  LogReg  \n",
      "4              0.998692          0.998693      0.999825  LogReg  \n",
      "5              0.995204          0.995247      0.986725     KNN  \n",
      "6              0.995640          0.995658      0.982532     KNN  \n",
      "7              0.995291          0.995351      0.983577     KNN  \n",
      "8              0.994157          0.994223      0.983750     KNN  \n",
      "9              0.994070          0.994082      0.979532     KNN  \n",
      "SVM\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.99      1.00      1.00     56163\n",
      "      benign       0.21      0.14      0.17       352\n",
      "\n",
      "    accuracy                           0.99     56515\n",
      "   macro avg       0.60      0.57      0.58     56515\n",
      "weighted avg       0.99      0.99      0.99     56515\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.232812    0.026706       0.997907                 0.997938   \n",
      "1    1.243682    0.025695       0.997384                 0.997352   \n",
      "2    1.412184    0.028950       0.997646                 0.997689   \n",
      "3    1.259485    0.025899       0.996512                 0.996601   \n",
      "4    1.342647    0.025527       0.998692                 0.998695   \n",
      "5    0.084887   20.717299       0.995204                 0.995302   \n",
      "6    0.081707   21.445543       0.995640                 0.995678   \n",
      "7    0.086478   21.311807       0.995291                 0.995434   \n",
      "8    0.086100   21.055004       0.994157                 0.994309   \n",
      "9    0.083949   21.108251       0.994070                 0.994096   \n",
      "10  16.586442   10.308514       0.993373                 0.993083   \n",
      "11  16.537410   10.296978       0.993460                 0.993190   \n",
      "12  16.666767   10.434226       0.994245                 0.994070   \n",
      "13  16.379793   10.246972       0.993896                 0.993781   \n",
      "14  16.274496   10.273878       0.995204                 0.995087   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.997907          0.997920      0.999684  LogReg  \n",
      "1               0.997384          0.997362      0.999594  LogReg  \n",
      "2               0.997646          0.997663      0.999734  LogReg  \n",
      "3               0.996512          0.996548      0.999050  LogReg  \n",
      "4               0.998692          0.998693      0.999825  LogReg  \n",
      "5               0.995204          0.995247      0.986725     KNN  \n",
      "6               0.995640          0.995658      0.982532     KNN  \n",
      "7               0.995291          0.995351      0.983577     KNN  \n",
      "8               0.994157          0.994223      0.983750     KNN  \n",
      "9               0.994070          0.994082      0.979532     KNN  \n",
      "10              0.993373          0.993097      0.998102     SVM  \n",
      "11              0.993460          0.993240      0.998274     SVM  \n",
      "12              0.994245          0.994128      0.998183     SVM  \n",
      "13              0.993896          0.993830      0.997540     SVM  \n",
      "14              0.995204          0.995125      0.998417     SVM  \n",
      "GNB\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       1.00      1.00      1.00     56163\n",
      "      benign       1.00      0.41      0.59       352\n",
      "\n",
      "    accuracy                           1.00     56515\n",
      "   macro avg       1.00      0.71      0.79     56515\n",
      "weighted avg       1.00      1.00      1.00     56515\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.232812    0.026706       0.997907                 0.997938   \n",
      "1    1.243682    0.025695       0.997384                 0.997352   \n",
      "2    1.412184    0.028950       0.997646                 0.997689   \n",
      "3    1.259485    0.025899       0.996512                 0.996601   \n",
      "4    1.342647    0.025527       0.998692                 0.998695   \n",
      "5    0.084887   20.717299       0.995204                 0.995302   \n",
      "6    0.081707   21.445543       0.995640                 0.995678   \n",
      "7    0.086478   21.311807       0.995291                 0.995434   \n",
      "8    0.086100   21.055004       0.994157                 0.994309   \n",
      "9    0.083949   21.108251       0.994070                 0.994096   \n",
      "10  16.586442   10.308514       0.993373                 0.993083   \n",
      "11  16.537410   10.296978       0.993460                 0.993190   \n",
      "12  16.666767   10.434226       0.994245                 0.994070   \n",
      "13  16.379793   10.246972       0.993896                 0.993781   \n",
      "14  16.274496   10.273878       0.995204                 0.995087   \n",
      "15   0.262778    0.075404       1.000000                 1.000000   \n",
      "16   0.259536    0.074755       1.000000                 1.000000   \n",
      "17   0.259792    0.075926       1.000000                 1.000000   \n",
      "18   0.259815    0.075633       1.000000                 1.000000   \n",
      "19   0.259942    0.075409       1.000000                 1.000000   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.997907          0.997920      0.999684  LogReg  \n",
      "1               0.997384          0.997362      0.999594  LogReg  \n",
      "2               0.997646          0.997663      0.999734  LogReg  \n",
      "3               0.996512          0.996548      0.999050  LogReg  \n",
      "4               0.998692          0.998693      0.999825  LogReg  \n",
      "5               0.995204          0.995247      0.986725     KNN  \n",
      "6               0.995640          0.995658      0.982532     KNN  \n",
      "7               0.995291          0.995351      0.983577     KNN  \n",
      "8               0.994157          0.994223      0.983750     KNN  \n",
      "9               0.994070          0.994082      0.979532     KNN  \n",
      "10              0.993373          0.993097      0.998102     SVM  \n",
      "11              0.993460          0.993240      0.998274     SVM  \n",
      "12              0.994245          0.994128      0.998183     SVM  \n",
      "13              0.993896          0.993830      0.997540     SVM  \n",
      "14              0.995204          0.995125      0.998417     SVM  \n",
      "15              1.000000          1.000000      1.000000     GNB  \n",
      "16              1.000000          1.000000      1.000000     GNB  \n",
      "17              1.000000          1.000000      1.000000     GNB  \n",
      "18              1.000000          1.000000      1.000000     GNB  \n",
      "19              1.000000          1.000000      1.000000     GNB  \n",
      "699.7130570959998\n"
     ]
    }
   ],
   "source": [
    "## Train: Binary  (raspberry), Test: in-browser (robust)  (raspberry)\n",
    "  \n",
    "df_malicious_Train = pd.concat([df4])\n",
    "df_malicious_Test = pd.concat([df3])\n",
    "\n",
    "df_benign_Train = pd.concat([df13,df14,df15,df16,df17])\n",
    "df_benign_Test = pd.concat([df13,df14,df15,df16,df17])\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "#print(\"{} NAN in malicious!\".format(len(df_malicious[df_malicious.isna().any(axis=1)])))\n",
    "#print(\"{} NAN in benign!\".format(len(df_benign[df_benign.isna().any(axis=1)])))\n",
    "\n",
    "df_malicious_Test = df_malicious_Test.dropna()\n",
    "df_malicious_Train = df_malicious_Train.dropna()\n",
    "\n",
    "df_benign_Test = df_benign_Test.dropna()\n",
    "df_benign_Train = df_benign_Train.dropna()\n",
    "\n",
    "print(\"After droppping NAN rows: \")\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "start = timer()\n",
    "\n",
    "results_all_combined_s0 = run_process(df_malicious_Train,df_benign_Train,df_malicious_Test,df_benign_Test,df_results)\n",
    "\n",
    "\n",
    "end = timer()\n",
    "print(end - start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malicious_train: 11745\n",
      "malicious_test: 9880\n",
      "benign_train: 561625\n",
      "benign_test: 561625\n",
      "After droppping NAN rows: \n",
      "malicious_train: 11745\n",
      "malicious_test: 9880\n",
      "benign_train: 561625\n",
      "benign_test: 561625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Feature Extraction: 100%|█████████████████████| 147/147 [00:02<00:00, 55.61it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 142/142 [00:02<00:00, 67.38it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:20<00:00,  1.99it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:21<00:00,  1.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "let the ml starts\n",
      "LogReg\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.99      1.00      0.99     56163\n",
      "      benign       0.88      0.45      0.59       988\n",
      "\n",
      "    accuracy                           0.99     57151\n",
      "   macro avg       0.94      0.72      0.79     57151\n",
      "weighted avg       0.99      0.99      0.99     57151\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.287373    0.026509       0.997907                 0.997938   \n",
      "1  1.352373    0.025498       0.997384                 0.997352   \n",
      "2  1.288956    0.027089       0.997646                 0.997689   \n",
      "3  1.306785    0.030216       0.996512                 0.996601   \n",
      "4  1.334100    0.025155       0.998692                 0.998695   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.997907          0.997920      0.999684  LogReg  \n",
      "1              0.997384          0.997362      0.999594  LogReg  \n",
      "2              0.997646          0.997663      0.999734  LogReg  \n",
      "3              0.996512          0.996548      0.999050  LogReg  \n",
      "4              0.998692          0.998693      0.999825  LogReg  \n",
      "KNN\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.98      1.00      0.99     56163\n",
      "      benign       0.58      0.13      0.21       988\n",
      "\n",
      "    accuracy                           0.98     57151\n",
      "   macro avg       0.78      0.56      0.60     57151\n",
      "weighted avg       0.98      0.98      0.98     57151\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.287373    0.026509       0.997907                 0.997938   \n",
      "1  1.352373    0.025498       0.997384                 0.997352   \n",
      "2  1.288956    0.027089       0.997646                 0.997689   \n",
      "3  1.306785    0.030216       0.996512                 0.996601   \n",
      "4  1.334100    0.025155       0.998692                 0.998695   \n",
      "5  0.082935   20.927298       0.995204                 0.995302   \n",
      "6  0.080564   21.571925       0.995640                 0.995678   \n",
      "7  0.081810   21.528514       0.995291                 0.995434   \n",
      "8  0.081178   20.896280       0.994157                 0.994309   \n",
      "9  0.082086   21.515686       0.994070                 0.994096   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.997907          0.997920      0.999684  LogReg  \n",
      "1              0.997384          0.997362      0.999594  LogReg  \n",
      "2              0.997646          0.997663      0.999734  LogReg  \n",
      "3              0.996512          0.996548      0.999050  LogReg  \n",
      "4              0.998692          0.998693      0.999825  LogReg  \n",
      "5              0.995204          0.995247      0.986725     KNN  \n",
      "6              0.995640          0.995658      0.982532     KNN  \n",
      "7              0.995291          0.995351      0.983577     KNN  \n",
      "8              0.994157          0.994223      0.983750     KNN  \n",
      "9              0.994070          0.994082      0.979532     KNN  \n",
      "SVM\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.99      1.00      0.99     56163\n",
      "      benign       0.80      0.39      0.53       988\n",
      "\n",
      "    accuracy                           0.99     57151\n",
      "   macro avg       0.89      0.70      0.76     57151\n",
      "weighted avg       0.99      0.99      0.99     57151\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.287373    0.026509       0.997907                 0.997938   \n",
      "1    1.352373    0.025498       0.997384                 0.997352   \n",
      "2    1.288956    0.027089       0.997646                 0.997689   \n",
      "3    1.306785    0.030216       0.996512                 0.996601   \n",
      "4    1.334100    0.025155       0.998692                 0.998695   \n",
      "5    0.082935   20.927298       0.995204                 0.995302   \n",
      "6    0.080564   21.571925       0.995640                 0.995678   \n",
      "7    0.081810   21.528514       0.995291                 0.995434   \n",
      "8    0.081178   20.896280       0.994157                 0.994309   \n",
      "9    0.082086   21.515686       0.994070                 0.994096   \n",
      "10  15.643508    9.985163       0.993373                 0.993083   \n",
      "11  15.741167    9.976577       0.993460                 0.993190   \n",
      "12  15.903341   10.104659       0.994245                 0.994070   \n",
      "13  15.659545    9.998727       0.993896                 0.993781   \n",
      "14  15.690814   10.036403       0.995204                 0.995087   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.997907          0.997920      0.999684  LogReg  \n",
      "1               0.997384          0.997362      0.999594  LogReg  \n",
      "2               0.997646          0.997663      0.999734  LogReg  \n",
      "3               0.996512          0.996548      0.999050  LogReg  \n",
      "4               0.998692          0.998693      0.999825  LogReg  \n",
      "5               0.995204          0.995247      0.986725     KNN  \n",
      "6               0.995640          0.995658      0.982532     KNN  \n",
      "7               0.995291          0.995351      0.983577     KNN  \n",
      "8               0.994157          0.994223      0.983750     KNN  \n",
      "9               0.994070          0.994082      0.979532     KNN  \n",
      "10              0.993373          0.993097      0.998102     SVM  \n",
      "11              0.993460          0.993240      0.998274     SVM  \n",
      "12              0.994245          0.994128      0.998183     SVM  \n",
      "13              0.993896          0.993830      0.997540     SVM  \n",
      "14              0.995204          0.995125      0.998417     SVM  \n",
      "GNB\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       1.00      1.00      1.00     56163\n",
      "      benign       1.00      0.86      0.92       988\n",
      "\n",
      "    accuracy                           1.00     57151\n",
      "   macro avg       1.00      0.93      0.96     57151\n",
      "weighted avg       1.00      1.00      1.00     57151\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.287373    0.026509       0.997907                 0.997938   \n",
      "1    1.352373    0.025498       0.997384                 0.997352   \n",
      "2    1.288956    0.027089       0.997646                 0.997689   \n",
      "3    1.306785    0.030216       0.996512                 0.996601   \n",
      "4    1.334100    0.025155       0.998692                 0.998695   \n",
      "5    0.082935   20.927298       0.995204                 0.995302   \n",
      "6    0.080564   21.571925       0.995640                 0.995678   \n",
      "7    0.081810   21.528514       0.995291                 0.995434   \n",
      "8    0.081178   20.896280       0.994157                 0.994309   \n",
      "9    0.082086   21.515686       0.994070                 0.994096   \n",
      "10  15.643508    9.985163       0.993373                 0.993083   \n",
      "11  15.741167    9.976577       0.993460                 0.993190   \n",
      "12  15.903341   10.104659       0.994245                 0.994070   \n",
      "13  15.659545    9.998727       0.993896                 0.993781   \n",
      "14  15.690814   10.036403       0.995204                 0.995087   \n",
      "15   0.273798    0.077471       1.000000                 1.000000   \n",
      "16   0.262822    0.076396       1.000000                 1.000000   \n",
      "17   0.263047    0.076608       1.000000                 1.000000   \n",
      "18   0.263232    0.076808       1.000000                 1.000000   \n",
      "19   0.262656    0.077140       1.000000                 1.000000   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.997907          0.997920      0.999684  LogReg  \n",
      "1               0.997384          0.997362      0.999594  LogReg  \n",
      "2               0.997646          0.997663      0.999734  LogReg  \n",
      "3               0.996512          0.996548      0.999050  LogReg  \n",
      "4               0.998692          0.998693      0.999825  LogReg  \n",
      "5               0.995204          0.995247      0.986725     KNN  \n",
      "6               0.995640          0.995658      0.982532     KNN  \n",
      "7               0.995291          0.995351      0.983577     KNN  \n",
      "8               0.994157          0.994223      0.983750     KNN  \n",
      "9               0.994070          0.994082      0.979532     KNN  \n",
      "10              0.993373          0.993097      0.998102     SVM  \n",
      "11              0.993460          0.993240      0.998274     SVM  \n",
      "12              0.994245          0.994128      0.998183     SVM  \n",
      "13              0.993896          0.993830      0.997540     SVM  \n",
      "14              0.995204          0.995125      0.998417     SVM  \n",
      "15              1.000000          1.000000      1.000000     GNB  \n",
      "16              1.000000          1.000000      1.000000     GNB  \n",
      "17              1.000000          1.000000      1.000000     GNB  \n",
      "18              1.000000          1.000000      1.000000     GNB  \n",
      "19              1.000000          1.000000      1.000000     GNB  \n",
      "703.5650659849998\n"
     ]
    }
   ],
   "source": [
    "## Train: Binary, Test: in-browser (stealthy)\n",
    " \n",
    "df_malicious_Train = pd.concat([df4])\n",
    "df_malicious_Test = pd.concat([df33])\n",
    "\n",
    "\n",
    "df_benign_Train = pd.concat([df13,df14,df15,df16,df17])\n",
    "df_benign_Test = pd.concat([df13,df14,df15,df16,df17])\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "#print(\"{} NAN in malicious!\".format(len(df_malicious[df_malicious.isna().any(axis=1)])))\n",
    "#print(\"{} NAN in benign!\".format(len(df_benign[df_benign.isna().any(axis=1)])))\n",
    "\n",
    "df_malicious_Test = df_malicious_Test.dropna()\n",
    "df_malicious_Train = df_malicious_Train.dropna()\n",
    "\n",
    "df_benign_Test = df_benign_Test.dropna()\n",
    "df_benign_Train = df_benign_Train.dropna()\n",
    "\n",
    "print(\"After droppping NAN rows: \")\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "start = timer()\n",
    "\n",
    "results_all_combined_s0 = run_process(df_malicious_Train,df_benign_Train,df_malicious_Test,df_benign_Test,df_results)\n",
    "\n",
    "end = timer()\n",
    "print(end - start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malicious_train: 12871\n",
      "malicious_test: 9880\n",
      "benign_train: 561625\n",
      "benign_test: 561625\n",
      "After droppping NAN rows: \n",
      "malicious_train: 12871\n",
      "malicious_test: 9880\n",
      "benign_train: 561625\n",
      "benign_test: 561625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Feature Extraction: 100%|█████████████████████| 144/144 [00:03<00:00, 43.05it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 142/142 [00:02<00:00, 68.91it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:19<00:00,  2.01it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:20<00:00,  1.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "let the ml starts\n",
      "LogReg\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       1.00      1.00      1.00     56163\n",
      "      benign       0.88      0.72      0.79       988\n",
      "\n",
      "    accuracy                           0.99     57151\n",
      "   macro avg       0.94      0.86      0.89     57151\n",
      "weighted avg       0.99      0.99      0.99     57151\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.067142    0.025759       0.992690                 0.992365   \n",
      "1  1.065518    0.025086       0.993037                 0.992786   \n",
      "2  1.354021    0.025290       0.993734                 0.993513   \n",
      "3  1.038317    0.025166       0.993473                 0.993209   \n",
      "4  1.177782    0.025202       0.993647                 0.993420   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.992690          0.992366      0.983442  LogReg  \n",
      "1              0.993037          0.992851      0.985584  LogReg  \n",
      "2              0.993734          0.993567      0.990189  LogReg  \n",
      "3              0.993473          0.993194      0.988586  LogReg  \n",
      "4              0.993647          0.993480      0.982865  LogReg  \n",
      "KNN\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.99      1.00      0.99     56163\n",
      "      benign       0.83      0.50      0.62       988\n",
      "\n",
      "    accuracy                           0.99     57151\n",
      "   macro avg       0.91      0.75      0.81     57151\n",
      "weighted avg       0.99      0.99      0.99     57151\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.067142    0.025759       0.992690                 0.992365   \n",
      "1  1.065518    0.025086       0.993037                 0.992786   \n",
      "2  1.354021    0.025290       0.993734                 0.993513   \n",
      "3  1.038317    0.025166       0.993473                 0.993209   \n",
      "4  1.177782    0.025202       0.993647                 0.993420   \n",
      "5  0.083230   21.559979       0.986946                 0.985756   \n",
      "6  0.079665   20.620398       0.986510                 0.985176   \n",
      "7  0.079589   20.999211       0.988425                 0.987418   \n",
      "8  0.080179   21.621862       0.987728                 0.986705   \n",
      "9  0.079423   20.812693       0.987119                 0.985846   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.992690          0.992366      0.983442  LogReg  \n",
      "1              0.993037          0.992851      0.985584  LogReg  \n",
      "2              0.993734          0.993567      0.990189  LogReg  \n",
      "3              0.993473          0.993194      0.988586  LogReg  \n",
      "4              0.993647          0.993480      0.982865  LogReg  \n",
      "5              0.986946          0.985973      0.924755     KNN  \n",
      "6              0.986510          0.985092      0.917734     KNN  \n",
      "7              0.988425          0.987463      0.915861     KNN  \n",
      "8              0.987728          0.986945      0.930298     KNN  \n",
      "9              0.987119          0.986106      0.920124     KNN  \n",
      "SVM\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.99      1.00      1.00     56163\n",
      "      benign       0.93      0.50      0.65       988\n",
      "\n",
      "    accuracy                           0.99     57151\n",
      "   macro avg       0.96      0.75      0.82     57151\n",
      "weighted avg       0.99      0.99      0.99     57151\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.067142    0.025759       0.992690                 0.992365   \n",
      "1    1.065518    0.025086       0.993037                 0.992786   \n",
      "2    1.354021    0.025290       0.993734                 0.993513   \n",
      "3    1.038317    0.025166       0.993473                 0.993209   \n",
      "4    1.177782    0.025202       0.993647                 0.993420   \n",
      "5    0.083230   21.559979       0.986946                 0.985756   \n",
      "6    0.079665   20.620398       0.986510                 0.985176   \n",
      "7    0.079589   20.999211       0.988425                 0.987418   \n",
      "8    0.080179   21.621862       0.987728                 0.986705   \n",
      "9    0.079423   20.812693       0.987119                 0.985846   \n",
      "10  19.531325   11.232444       0.988861                 0.988563   \n",
      "11  19.331001   11.228837       0.987903                 0.987565   \n",
      "12  19.986226   11.215172       0.988686                 0.988338   \n",
      "13  19.438970   11.148853       0.988860                 0.988477   \n",
      "14  19.558782   11.308556       0.989643                 0.989100   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.992690          0.992366      0.983442  LogReg  \n",
      "1               0.993037          0.992851      0.985584  LogReg  \n",
      "2               0.993734          0.993567      0.990189  LogReg  \n",
      "3               0.993473          0.993194      0.988586  LogReg  \n",
      "4               0.993647          0.993480      0.982865  LogReg  \n",
      "5               0.986946          0.985973      0.924755     KNN  \n",
      "6               0.986510          0.985092      0.917734     KNN  \n",
      "7               0.988425          0.987463      0.915861     KNN  \n",
      "8               0.987728          0.986945      0.930298     KNN  \n",
      "9               0.987119          0.986106      0.920124     KNN  \n",
      "10              0.988861          0.987329      0.908273     SVM  \n",
      "11              0.987903          0.986067      0.917491     SVM  \n",
      "12              0.988686          0.986986      0.892342     SVM  \n",
      "13              0.988860          0.987293      0.909611     SVM  \n",
      "14              0.989643          0.988394      0.921479     SVM  \n",
      "GNB\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       1.00      1.00      1.00     56163\n",
      "      benign       1.00      1.00      1.00       988\n",
      "\n",
      "    accuracy                           1.00     57151\n",
      "   macro avg       1.00      1.00      1.00     57151\n",
      "weighted avg       1.00      1.00      1.00     57151\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.067142    0.025759       0.992690                 0.992365   \n",
      "1    1.065518    0.025086       0.993037                 0.992786   \n",
      "2    1.354021    0.025290       0.993734                 0.993513   \n",
      "3    1.038317    0.025166       0.993473                 0.993209   \n",
      "4    1.177782    0.025202       0.993647                 0.993420   \n",
      "5    0.083230   21.559979       0.986946                 0.985756   \n",
      "6    0.079665   20.620398       0.986510                 0.985176   \n",
      "7    0.079589   20.999211       0.988425                 0.987418   \n",
      "8    0.080179   21.621862       0.987728                 0.986705   \n",
      "9    0.079423   20.812693       0.987119                 0.985846   \n",
      "10  19.531325   11.232444       0.988861                 0.988563   \n",
      "11  19.331001   11.228837       0.987903                 0.987565   \n",
      "12  19.986226   11.215172       0.988686                 0.988338   \n",
      "13  19.438970   11.148853       0.988860                 0.988477   \n",
      "14  19.558782   11.308556       0.989643                 0.989100   \n",
      "15   0.262566    0.074279       0.998956                 0.998957   \n",
      "16   0.259003    0.072757       0.999217                 0.999217   \n",
      "17   0.265922    0.072625       0.999826                 0.999826   \n",
      "18   0.258967    0.072964       0.999217                 0.999217   \n",
      "19   0.258928    0.072905       0.998869                 0.998870   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.992690          0.992366      0.983442  LogReg  \n",
      "1               0.993037          0.992851      0.985584  LogReg  \n",
      "2               0.993734          0.993567      0.990189  LogReg  \n",
      "3               0.993473          0.993194      0.988586  LogReg  \n",
      "4               0.993647          0.993480      0.982865  LogReg  \n",
      "5               0.986946          0.985973      0.924755     KNN  \n",
      "6               0.986510          0.985092      0.917734     KNN  \n",
      "7               0.988425          0.987463      0.915861     KNN  \n",
      "8               0.987728          0.986945      0.930298     KNN  \n",
      "9               0.987119          0.986106      0.920124     KNN  \n",
      "10              0.988861          0.987329      0.908273     SVM  \n",
      "11              0.987903          0.986067      0.917491     SVM  \n",
      "12              0.988686          0.986986      0.892342     SVM  \n",
      "13              0.988860          0.987293      0.909611     SVM  \n",
      "14              0.989643          0.988394      0.921479     SVM  \n",
      "15              0.998956          0.998944      1.000000     GNB  \n",
      "16              0.999217          0.999210      0.996283     GNB  \n",
      "17              0.999826          0.999826      1.000000     GNB  \n",
      "18              0.999217          0.999210      1.000000     GNB  \n",
      "19              0.998869          0.998854      0.997967     GNB  \n",
      "734.9613477069997\n"
     ]
    }
   ],
   "source": [
    "## Train: in-browser (aggressive), Test: in-browser (stealthy)\n",
    " \n",
    "df_malicious_Train = pd.concat([df5])\n",
    "df_malicious_Test = pd.concat([df33])\n",
    "\n",
    "\n",
    "df_benign_Train = pd.concat([df13,df14,df15,df16,df17])\n",
    "df_benign_Test = pd.concat([df13,df14,df15,df16,df17])\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "#print(\"{} NAN in malicious!\".format(len(df_malicious[df_malicious.isna().any(axis=1)])))\n",
    "#print(\"{} NAN in benign!\".format(len(df_benign[df_benign.isna().any(axis=1)])))\n",
    "\n",
    "df_malicious_Test = df_malicious_Test.dropna()\n",
    "df_malicious_Train = df_malicious_Train.dropna()\n",
    "\n",
    "df_benign_Test = df_benign_Test.dropna()\n",
    "df_benign_Train = df_benign_Train.dropna()\n",
    "\n",
    "print(\"After droppping NAN rows: \")\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "start = timer()\n",
    "\n",
    "results_all_combined_s0 = run_process(df_malicious_Train,df_benign_Train,df_malicious_Test,df_benign_Test,df_results)\n",
    "\n",
    "end = timer()\n",
    "print(end - start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malicious_train: 11745\n",
      "malicious_test: 16744\n",
      "benign_train: 561625\n",
      "benign_test: 561625\n",
      "After droppping NAN rows: \n",
      "malicious_train: 11745\n",
      "malicious_test: 16744\n",
      "benign_train: 561625\n",
      "benign_test: 561625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Feature Extraction: 100%|█████████████████████| 147/147 [00:02<00:00, 58.08it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 153/153 [00:03<00:00, 42.94it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:19<00:00,  2.02it/s]\n",
      "Feature Extraction: 100%|█████████████████████| 160/160 [01:20<00:00,  1.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "let the ml starts\n",
      "LogReg\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.97      1.00      0.99     56163\n",
      "      benign       0.74      0.13      0.22      1675\n",
      "\n",
      "    accuracy                           0.97     57838\n",
      "   macro avg       0.86      0.56      0.60     57838\n",
      "weighted avg       0.97      0.97      0.96     57838\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.320381    0.026651       0.997907                 0.997938   \n",
      "1  1.311617    0.025603       0.997384                 0.997352   \n",
      "2  1.206052    0.025622       0.997646                 0.997689   \n",
      "3  1.389016    0.025620       0.996512                 0.996601   \n",
      "4  1.317492    0.025485       0.998692                 0.998695   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.997907          0.997920      0.999684  LogReg  \n",
      "1              0.997384          0.997362      0.999594  LogReg  \n",
      "2              0.997646          0.997663      0.999734  LogReg  \n",
      "3              0.996512          0.996548      0.999050  LogReg  \n",
      "4              0.998692          0.998693      0.999825  LogReg  \n",
      "KNN\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.97      1.00      0.99     56163\n",
      "      benign       0.64      0.09      0.16      1675\n",
      "\n",
      "    accuracy                           0.97     57838\n",
      "   macro avg       0.81      0.54      0.57     57838\n",
      "weighted avg       0.96      0.97      0.96     57838\n",
      "\n",
      "   fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0  1.320381    0.026651       0.997907                 0.997938   \n",
      "1  1.311617    0.025603       0.997384                 0.997352   \n",
      "2  1.206052    0.025622       0.997646                 0.997689   \n",
      "3  1.389016    0.025620       0.996512                 0.996601   \n",
      "4  1.317492    0.025485       0.998692                 0.998695   \n",
      "5  0.083856   20.680910       0.995204                 0.995302   \n",
      "6  0.080384   21.398600       0.995640                 0.995678   \n",
      "7  0.080891   21.580201       0.995291                 0.995434   \n",
      "8  0.081256   20.511752       0.994157                 0.994309   \n",
      "9  0.085614   21.236640       0.994070                 0.994096   \n",
      "\n",
      "   test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0              0.997907          0.997920      0.999684  LogReg  \n",
      "1              0.997384          0.997362      0.999594  LogReg  \n",
      "2              0.997646          0.997663      0.999734  LogReg  \n",
      "3              0.996512          0.996548      0.999050  LogReg  \n",
      "4              0.998692          0.998693      0.999825  LogReg  \n",
      "5              0.995204          0.995247      0.986725     KNN  \n",
      "6              0.995640          0.995658      0.982532     KNN  \n",
      "7              0.995291          0.995351      0.983577     KNN  \n",
      "8              0.994157          0.994223      0.983750     KNN  \n",
      "9              0.994070          0.994082      0.979532     KNN  \n",
      "SVM\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.97      1.00      0.99     56163\n",
      "      benign       0.63      0.10      0.18      1675\n",
      "\n",
      "    accuracy                           0.97     57838\n",
      "   macro avg       0.80      0.55      0.58     57838\n",
      "weighted avg       0.96      0.97      0.96     57838\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.320381    0.026651       0.997907                 0.997938   \n",
      "1    1.311617    0.025603       0.997384                 0.997352   \n",
      "2    1.206052    0.025622       0.997646                 0.997689   \n",
      "3    1.389016    0.025620       0.996512                 0.996601   \n",
      "4    1.317492    0.025485       0.998692                 0.998695   \n",
      "5    0.083856   20.680910       0.995204                 0.995302   \n",
      "6    0.080384   21.398600       0.995640                 0.995678   \n",
      "7    0.080891   21.580201       0.995291                 0.995434   \n",
      "8    0.081256   20.511752       0.994157                 0.994309   \n",
      "9    0.085614   21.236640       0.994070                 0.994096   \n",
      "10  18.406569   10.305786       0.993373                 0.993083   \n",
      "11  16.902231   10.353427       0.993460                 0.993190   \n",
      "12  17.724654   10.361819       0.994245                 0.994070   \n",
      "13  16.654845   10.354887       0.993896                 0.993781   \n",
      "14  17.679086   10.495988       0.995204                 0.995087   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.997907          0.997920      0.999684  LogReg  \n",
      "1               0.997384          0.997362      0.999594  LogReg  \n",
      "2               0.997646          0.997663      0.999734  LogReg  \n",
      "3               0.996512          0.996548      0.999050  LogReg  \n",
      "4               0.998692          0.998693      0.999825  LogReg  \n",
      "5               0.995204          0.995247      0.986725     KNN  \n",
      "6               0.995640          0.995658      0.982532     KNN  \n",
      "7               0.995291          0.995351      0.983577     KNN  \n",
      "8               0.994157          0.994223      0.983750     KNN  \n",
      "9               0.994070          0.994082      0.979532     KNN  \n",
      "10              0.993373          0.993097      0.998102     SVM  \n",
      "11              0.993460          0.993240      0.998274     SVM  \n",
      "12              0.994245          0.994128      0.998183     SVM  \n",
      "13              0.993896          0.993830      0.997540     SVM  \n",
      "14              0.995204          0.995125      0.998417     SVM  \n",
      "GNB\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   malignant       0.97      1.00      0.99     56163\n",
      "      benign       0.00      0.00      0.00      1675\n",
      "\n",
      "    accuracy                           0.97     57838\n",
      "   macro avg       0.49      0.50      0.49     57838\n",
      "weighted avg       0.94      0.97      0.96     57838\n",
      "\n",
      "     fit_time  score_time  test_accuracy  test_precision_weighted  \\\n",
      "0    1.320381    0.026651       0.997907                 0.997938   \n",
      "1    1.311617    0.025603       0.997384                 0.997352   \n",
      "2    1.206052    0.025622       0.997646                 0.997689   \n",
      "3    1.389016    0.025620       0.996512                 0.996601   \n",
      "4    1.317492    0.025485       0.998692                 0.998695   \n",
      "5    0.083856   20.680910       0.995204                 0.995302   \n",
      "6    0.080384   21.398600       0.995640                 0.995678   \n",
      "7    0.080891   21.580201       0.995291                 0.995434   \n",
      "8    0.081256   20.511752       0.994157                 0.994309   \n",
      "9    0.085614   21.236640       0.994070                 0.994096   \n",
      "10  18.406569   10.305786       0.993373                 0.993083   \n",
      "11  16.902231   10.353427       0.993460                 0.993190   \n",
      "12  17.724654   10.361819       0.994245                 0.994070   \n",
      "13  16.654845   10.354887       0.993896                 0.993781   \n",
      "14  17.679086   10.495988       0.995204                 0.995087   \n",
      "15   0.270794    0.078204       1.000000                 1.000000   \n",
      "16   0.267109    0.080246       1.000000                 1.000000   \n",
      "17   0.270173    0.077326       1.000000                 1.000000   \n",
      "18   0.267945    0.080282       1.000000                 1.000000   \n",
      "19   0.265065    0.077471       1.000000                 1.000000   \n",
      "\n",
      "    test_recall_weighted  test_f1_weighted  test_roc_auc   model  \n",
      "0               0.997907          0.997920      0.999684  LogReg  \n",
      "1               0.997384          0.997362      0.999594  LogReg  \n",
      "2               0.997646          0.997663      0.999734  LogReg  \n",
      "3               0.996512          0.996548      0.999050  LogReg  \n",
      "4               0.998692          0.998693      0.999825  LogReg  \n",
      "5               0.995204          0.995247      0.986725     KNN  \n",
      "6               0.995640          0.995658      0.982532     KNN  \n",
      "7               0.995291          0.995351      0.983577     KNN  \n",
      "8               0.994157          0.994223      0.983750     KNN  \n",
      "9               0.994070          0.994082      0.979532     KNN  \n",
      "10              0.993373          0.993097      0.998102     SVM  \n",
      "11              0.993460          0.993240      0.998274     SVM  \n",
      "12              0.994245          0.994128      0.998183     SVM  \n",
      "13              0.993896          0.993830      0.997540     SVM  \n",
      "14              0.995204          0.995125      0.998417     SVM  \n",
      "15              1.000000          1.000000      1.000000     GNB  \n",
      "16              1.000000          1.000000      1.000000     GNB  \n",
      "17              1.000000          1.000000      1.000000     GNB  \n",
      "18              1.000000          1.000000      1.000000     GNB  \n",
      "19              1.000000          1.000000      1.000000     GNB  \n",
      "719.1135704599983\n"
     ]
    }
   ],
   "source": [
    "## Train: in-browser (aggressive), Test: in-browser (robust)\n",
    " \n",
    "df_malicious_Train = pd.concat([df4])\n",
    "df_malicious_Test = pd.concat([df32])\n",
    "\n",
    "\n",
    "df_benign_Train = pd.concat([df13,df14,df15,df16,df17])\n",
    "df_benign_Test = pd.concat([df13,df14,df15,df16,df17])\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "#print(\"{} NAN in malicious!\".format(len(df_malicious[df_malicious.isna().any(axis=1)])))\n",
    "#print(\"{} NAN in benign!\".format(len(df_benign[df_benign.isna().any(axis=1)])))\n",
    "\n",
    "df_malicious_Test = df_malicious_Test.dropna()\n",
    "df_malicious_Train = df_malicious_Train.dropna()\n",
    "\n",
    "df_benign_Test = df_benign_Test.dropna()\n",
    "df_benign_Train = df_benign_Train.dropna()\n",
    "\n",
    "print(\"After droppping NAN rows: \")\n",
    "\n",
    "print(\"malicious_train: {}\".format(len(df_malicious_Train)))\n",
    "print(\"malicious_test: {}\".format(len(df_malicious_Test)))\n",
    "\n",
    "print(\"benign_train: {}\".format(len(df_benign_Train)))\n",
    "print(\"benign_test: {}\".format(len(df_benign_Test)))\n",
    "\n",
    "start = timer()\n",
    "\n",
    "results_all_combined_s0 = run_process(df_malicious_Train,df_benign_Train,df_malicious_Test,df_benign_Test,df_results)\n",
    "\n",
    "end = timer()\n",
    "print(end - start)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}