feature_importance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "null_counts = df.isnull().sum()\n",
    "print(\"Null/Empty counts for each column:\")\n",
    "print(null_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['timestamp_call_key', 'retailer_code', 'serial', 'reason', 'mos',\n",
       "       'resolved', 'no_of_accounts_with_syf_13_march',\n",
       "       'account_balance_13_march', 'delinquency_history_13_march',\n",
       "       'account_open_date_13_march', 'account_status_13_march',\n",
       "       'card_activation_status_13_march', 'eservice_ind_13_march',\n",
       "       'ebill_enrolled_status_13_march', 'auto_pay_enrolled_status_13_march',\n",
       "       'no_of_accounts_with_syf_18_march', 'account_balance_18_march',\n",
       "       'delinquency_history_18_march', 'account_open_date_18_march',\n",
       "       'account_status_18_march', 'card_activation_status_18_march',\n",
       "       'eservice_ind_18_march', 'ebill_enrolled_status_18_march',\n",
       "       'auto_pay_enrolled_status_18_march', 'date_of_call', 'time_of_call'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Code-level Importance Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Identifying the \"starting\" codes and their counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting codes and their counts:\n",
      "mos\n",
      "IA    1384460\n",
      "mn     409496\n",
      "mm       4842\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Split the 'mos' column by spaces and get the first element of each split to find the starting codes\n",
    "starting_codes = df['mos'].str.split().str[0]\n",
    "\n",
    "# Count the occurrences of each starting code\n",
    "starting_codes_count = starting_codes.value_counts()\n",
    "\n",
    "print(\"Starting codes and their counts:\")\n",
    "print(starting_codes_count)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique cases where 'mos' starts with 'mn' and their counts:\n",
      "mos\n",
      "IA PP    176820\n",
      "IA BA    132921\n",
      "IA PI     40559\n",
      "IA IA     22252\n",
      "RS IA     17087\n",
      "IA TR      4593\n",
      "IA Ba      3959\n",
      "IA DR      3155\n",
      "mn IA      2069\n",
      "m- IA      1803\n",
      "IA CB      1515\n",
      "IA AA      1220\n",
      "IA DP       821\n",
      "IA nl       183\n",
      "IA FI       179\n",
      "IA LW        87\n",
      "mn RS        79\n",
      "IA mm        69\n",
      "IA           57\n",
      "mn mn        43\n",
      "IA OC        25\n",
      "Name: count, dtype: int64\n",
      "Total unique cases: 21\n"
     ]
    }
   ],
   "source": [
    "# Filter rows where 'mos' starts with \"mn\" and extract the next two codes\n",
    "mn_sequences = df['mos'][df['mos'].str.startswith('mn')].str.split().str[1:3]\n",
    "\n",
    "# Convert the list of codes to a string for easier counting\n",
    "mn_sequences_str = mn_sequences.apply(lambda x: ' '.join(x))\n",
    "\n",
    "# Count unique sequences and their occurrences\n",
    "mn_unique_counts = mn_sequences_str.value_counts()\n",
    "\n",
    "print(\"Unique cases where 'mos' starts with 'mn' and their counts:\")\n",
    "print(mn_unique_counts)\n",
    "print(f\"Total unique cases: {mn_unique_counts.count()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter rows where 'mos' starts with \"mn mn RS\"\n",
    "mn_mn_rs_rows = df[df['mos'].str.startswith('mn mn RS')]\n",
    "\n",
    "# Save these rows to a CSV file\n",
    "mn_mn_rs_rows.to_csv('mn_mn_rs_rows.csv', index=False)\n",
    "\n",
    "print(f\"Saved {len(mn_mn_rs_rows)} rows to 'mn_mn_rs_rows.csv'.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter rows where 'mos' starts with \"mn mn mn\"\n",
    "mn_mn_mn_rows = df[df['mos'].str.startswith('mn mn mn')]\n",
    "\n",
    "# Save these rows to a CSV file\n",
    "mn_mn_mn_rows.to_csv('mn_mn_mn_rows.csv', index=False)\n",
    "\n",
    "print(f\"Saved {len(mn_mn_mn_rows)} rows to 'mn_mn_mn_rows.csv'.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique cases where 'mos' starts with 'mm' and their counts:\n",
      "mos\n",
      "IA IA    4652\n",
      "mm IA     175\n",
      "mm mm      15\n",
      "Name: count, dtype: int64\n",
      "Total unique cases: 3\n"
     ]
    }
   ],
   "source": [
    "# Filter rows where 'mos' starts with \"mm\" and extract the next two codes\n",
    "mm_sequences = df['mos'][df['mos'].str.startswith('mm')].str.split().str[1:3]\n",
    "\n",
    "# Convert the list of codes to a string for easier counting\n",
    "mm_sequences_str = mm_sequences.apply(lambda x: ' '.join(x))\n",
    "\n",
    "# Count unique sequences and their occurrences\n",
    "mm_unique_counts = mm_sequences_str.value_counts()\n",
    "\n",
    "print(\"Unique cases where 'mos' starts with 'mm' and their counts:\")\n",
    "print(mm_unique_counts)\n",
    "print(f\"Total unique cases: {mm_unique_counts.count()}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter rows where 'mos' starts with \"mm\" followed by \"mm mm\"\n",
    "mm_mm_mm_rows = df[df['mos'].str.startswith('mm mm mm')]\n",
    "\n",
    "# Save these rows to a CSV file\n",
    "mm_mm_mm_rows.to_csv('mm_mm_mm_rows.csv', index=False)\n",
    "\n",
    "print(f\"Saved {len(mm_mm_mm_rows)} rows to 'mm_mm_mm_rows.csv'.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Count of \"IA\" followed by \"BA\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"IA\" followed by \"BA\" occurs 584268 times.\n"
     ]
    }
   ],
   "source": [
    "# Filter rows where 'mos' has \"IA BA\" sequence and count them\n",
    "ia_ba_count = df[df['mos'].str.contains('IA BA')].shape[0]\n",
    "\n",
    "print(f'\"IA\" followed by \"BA\" occurs {ia_ba_count} times.')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rows containing IA, BA, and TR in any order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows containing IA, BA, and TR in any order: 498342\n"
     ]
    }
   ],
   "source": [
    "# Using regular expressions to find rows where IA, BA, and TR appear in any order\n",
    "rows_with_ia_ba_tr = df[df['mos'].str.contains('IA') & df['mos'].str.contains('BA') & df['mos'].str.contains('TR')].shape[0]\n",
    "\n",
    "print(f'Rows containing IA, BA, and TR in any order: {rows_with_ia_ba_tr}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rows ending with TR, divided by \"resolved\" value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows ending with TR and resolved: 853475\n",
      "Rows ending with TR and floor: 366217\n"
     ]
    }
   ],
   "source": [
    "# Count of rows ending with TR and resolved status\n",
    "rows_ending_with_tr_resolved = df[(df['mos'].str.endswith('TR')) & (df['resolved'] == 'resolved')].shape[0]\n",
    "\n",
    "# Count of rows ending with TR and floor status\n",
    "rows_ending_with_tr_floor = df[(df['mos'].str.endswith('TR')) & (df['resolved'] == 'floor')].shape[0]\n",
    "\n",
    "print(f'Rows ending with TR and resolved: {rows_ending_with_tr_resolved}')\n",
    "print(f'Rows ending with TR and floor: {rows_ending_with_tr_floor}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Count of Resolved values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counts of unique values in the 'resolved' column:\n",
      "resolved\n",
      "resolved    1432581\n",
      "floor        366217\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Count the occurrences of each unique value in the 'resolved' column\n",
    "resolved_counts = df['resolved'].value_counts()\n",
    "\n",
    "print(\"Counts of unique values in the 'resolved' column:\")\n",
    "print(resolved_counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Analyzing most impactful codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_mos_sequence(mos):\n",
    "    # Split the sequence into codes\n",
    "    codes = mos.split()\n",
    "    \n",
    "    # Remove everything up to the first \"IA\" including that \"IA\"\n",
    "    if \"IA\" in codes:\n",
    "        first_ia_index = codes.index(\"IA\") + 1  # Move past the first \"IA\"\n",
    "        codes = codes[first_ia_index:]\n",
    "    \n",
    "    # Now, remove consecutive \"IAs\" that follow immediately after\n",
    "    while codes and codes[0] == \"IA\":\n",
    "        codes.pop(0)  # Keep removing \"IA\" until we encounter a different code\n",
    "    \n",
    "    # Remove all \"BA\" and \"TR\" codes from what remains\n",
    "    codes = [code for code in codes if code not in (\"BA\", \"TR\")]\n",
    "    \n",
    "    return ' '.join(codes)\n",
    "\n",
    "# Apply the adjusted cleaning function to the 'mos' column\n",
    "df['cleaned_mos'] = df['mos'].apply(clean_mos_sequence)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data_cleaned_mos.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\tansw\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\sklearn\\feature_extraction\\text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Feature  Coefficient  Absolute Coefficient\n",
      "39      RS    10.079166             10.079166\n",
      "47      TN    -5.391877              5.391877\n",
      "42      TB    -5.041593              5.041593\n",
      "29      PC     4.310617              4.310617\n",
      "36      RC     4.281609              4.281609\n",
      "..     ...          ...                   ...\n",
      "59      eS     0.045563              0.045563\n",
      "30      PI    -0.044125              0.044125\n",
      "69      iT     0.024342              0.024342\n",
      "56      eB    -0.021953              0.021953\n",
      "48      TP    -0.009868              0.009868\n",
      "\n",
      "[78 rows x 3 columns]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Assuming df and clean_mos_sequence_v2 as before and df['cleaned_mos_v2'] is ready\n",
    "\n",
    "# Encoding the 'cleaned_mos_v2' sequences\n",
    "vectorizer = TfidfVectorizer(tokenizer=lambda x: x.split(), lowercase=False)\n",
    "X = vectorizer.fit_transform(df['cleaned_mos'].values)\n",
    "\n",
    "# Encoding the 'resolved' column (1 for 'floor', 0 otherwise)\n",
    "y = (df['resolved'] == 'floor').astype(int)\n",
    "\n",
    "# Splitting the data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Training a Logistic Regression model\n",
    "model = LogisticRegression(max_iter=1000)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Extracting feature importance\n",
    "feature_names = vectorizer.get_feature_names_out()\n",
    "coefficients = model.coef_[0]\n",
    "\n",
    "# Creating a DataFrame to display feature importance\n",
    "feature_importance = pd.DataFrame({'Feature': feature_names, 'Coefficient': coefficients})\n",
    "\n",
    "# Sorting features by absolute importance\n",
    "feature_importance['Absolute Coefficient'] = feature_importance['Coefficient'].abs()\n",
    "feature_importance_sorted = feature_importance.sort_values(by='Absolute Coefficient', ascending=False)\n",
    "\n",
    "# Displaying the top influential features\n",
    "print(feature_importance_sorted)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance_sorted.to_csv('logistic_features.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ML - XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\tansw\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\sklearn\\feature_extraction\\text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Feature  Importance\n",
      "75      mt    0.313048\n",
      "39      RS    0.142852\n",
      "35      Pd    0.055902\n",
      "33      PT    0.055104\n",
      "14      FI    0.038958\n",
      "..     ...         ...\n",
      "56      eB    0.000000\n",
      "27      OC    0.000000\n",
      "51      Te    0.000000\n",
      "45      TE    0.000000\n",
      "62      iF    0.000000\n",
      "\n",
      "[78 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "# Assuming the data preparation steps are the same\n",
    "vectorizer = TfidfVectorizer(tokenizer=lambda x: x.split(), lowercase=False)\n",
    "X = vectorizer.fit_transform(df['cleaned_mos'].values)\n",
    "y = (df['resolved'] == 'floor').astype(int)\n",
    "\n",
    "# Splitting the dataset\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Initialize and fit the model\n",
    "xgb_model = XGBClassifier(use_label_encoder=False, eval_metric='logloss')\n",
    "xgb_model.fit(X_train, y_train)\n",
    "\n",
    "# Extract feature importance\n",
    "feature_importances = xgb_model.feature_importances_\n",
    "feature_names = vectorizer.get_feature_names_out()\n",
    "ml_feature_importance_df = pd.DataFrame({'Feature': feature_names, 'Importance': feature_importances})\n",
    "ml_feature_importance_df = ml_feature_importance_df.sort_values(by='Importance', ascending=False)\n",
    "\n",
    "print(ml_feature_importance_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ML - Linear Support Vector Machine (SVM) with Linear Kernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\tansw\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\sklearn\\feature_extraction\\text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
      "  warnings.warn(\n",
      "c:\\Users\\tansw\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\sklearn\\svm\\_classes.py:31: FutureWarning: The default value of `dual` will change from `True` to `'auto'` in 1.5. Set the value of `dual` explicitly to suppress the warning.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Feature  Coefficient  Absolute Coefficient\n",
      "39      RS     2.589569              2.589569\n",
      "47      TN    -1.857296              1.857296\n",
      "10      DE     1.735202              1.735202\n",
      "42      TB    -1.667842              1.667842\n",
      "4       BL     1.517775              1.517775\n",
      "36      RC     1.513772              1.513772\n",
      "14      FI     1.468856              1.468856\n",
      "15      FM     1.456148              1.456148\n",
      "29      PC     1.453462              1.453462\n",
      "66      iP     1.447207              1.447207\n",
      "9       CT     1.419551              1.419551\n",
      "25      NU     1.406764              1.406764\n",
      "44      TD    -1.397162              1.397162\n",
      "68      iS     1.357198              1.357198\n",
      "43      TC    -1.301690              1.301690\n",
      "21      LS     1.284685              1.284685\n",
      "71      me     1.281550              1.281550\n",
      "8       CD     1.246708              1.246708\n",
      "35      Pd     1.234342              1.234342\n",
      "46      TL    -1.228536              1.228536\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "\n",
    "vectorizer = TfidfVectorizer(tokenizer=lambda x: x.split(), lowercase=False)\n",
    "X = vectorizer.fit_transform(df['cleaned_mos'].values)\n",
    "y = (df['resolved'] == 'floor').astype(int)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "svm_model = LinearSVC(max_iter=10000)\n",
    "svm_model.fit(X_train, y_train)\n",
    "\n",
    "# Extracting feature importance\n",
    "feature_names = vectorizer.get_feature_names_out()\n",
    "coefficients = svm_model.coef_[0]\n",
    "feature_importance_svm = pd.DataFrame({'Feature': feature_names, 'Coefficient': coefficients})\n",
    "feature_importance_svm['Absolute Coefficient'] = feature_importance_svm['Coefficient'].abs()\n",
    "feature_importance_svm_sorted = feature_importance_svm.sort_values(by='Absolute Coefficient', ascending=False)\n",
    "\n",
    "print(feature_importance_svm_sorted.head(20))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance_svm_sorted.to_csv('svm_features.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ridge Classifier (with L2 Regularization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\tansw\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\sklearn\\feature_extraction\\text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Feature  Coefficient  Absolute Coefficient\n",
      "39      RS     2.026062              2.026062\n",
      "47      TN    -1.779387              1.779387\n",
      "42      TB    -1.631254              1.631254\n",
      "10      DE     1.562741              1.562741\n",
      "4       BL     1.497005              1.497005\n",
      "14      FI     1.473131              1.473131\n",
      "66      iP     1.452929              1.452929\n",
      "29      PC     1.452430              1.452430\n",
      "36      RC     1.449985              1.449985\n",
      "15      FM     1.419483              1.419483\n",
      "68      iS     1.388167              1.388167\n",
      "44      TD    -1.387542              1.387542\n",
      "43      TC    -1.277350              1.277350\n",
      "21      LS     1.266163              1.266163\n",
      "71      me     1.263555              1.263555\n",
      "8       CD     1.239214              1.239214\n",
      "35      Pd     1.228700              1.228700\n",
      "46      TL    -1.214026              1.214026\n",
      "60      eY     1.141885              1.141885\n",
      "20      LC     1.063283              1.063283\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import RidgeClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "\n",
    "vectorizer = TfidfVectorizer(tokenizer=lambda x: x.split(), lowercase=False)\n",
    "X = vectorizer.fit_transform(df['cleaned_mos'].values)\n",
    "y = (df['resolved'] == 'floor').astype(int)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "ridge_model = RidgeClassifier()\n",
    "ridge_model.fit(X_train, y_train)\n",
    "\n",
    "# Extracting feature importance\n",
    "feature_names = vectorizer.get_feature_names_out()\n",
    "coefficients = ridge_model.coef_[0]\n",
    "feature_importance_ridge = pd.DataFrame({'Feature': feature_names, 'Coefficient': coefficients})\n",
    "feature_importance_ridge['Absolute Coefficient'] = feature_importance_ridge['Coefficient'].abs()\n",
    "feature_importance_ridge_sorted = feature_importance_ridge.sort_values(by='Absolute Coefficient', ascending=False)\n",
    "\n",
    "print(feature_importance_ridge_sorted.head(20))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance_ridge_sorted.to_csv('ridge_features.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating mean coefficients, considering values from Logistic Regression, SVM, and Ridge Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Load the CSV files into DataFrames\n",
    "df_logreg = pd.read_csv('logistic_features.csv')\n",
    "df_svm = pd.read_csv('svm_features.csv')\n",
    "df_ridge = pd.read_csv('ridge_features.csv')\n",
    "\n",
    "# Normalize the Coefficients\n",
    "df_logreg['Normalized Coefficient'] = df_logreg['Coefficient'] / df_logreg['Absolute Coefficient'].max()\n",
    "df_svm['Normalized Coefficient'] = df_svm['Coefficient'] / df_svm['Absolute Coefficient'].max()\n",
    "df_ridge['Normalized Coefficient'] = df_ridge['Coefficient'] / df_ridge['Absolute Coefficient'].max()\n",
    "\n",
    "# Rank the Features based on Absolute Normalized Coefficients\n",
    "df_logreg['Rank'] = df_logreg['Absolute Coefficient'].rank(method='min', ascending=False)\n",
    "df_svm['Rank'] = df_svm['Absolute Coefficient'].rank(method='min', ascending=False)\n",
    "df_ridge['Rank'] = df_ridge['Absolute Coefficient'].rank(method='min', ascending=False)\n",
    "\n",
    "# Merge the dataframes on 'Feature', including normalized coefficients\n",
    "df_merged = pd.merge(df_logreg[['Feature', 'Rank', 'Normalized Coefficient']], df_svm[['Feature', 'Rank', 'Normalized Coefficient']], on='Feature', how='inner', suffixes=('_logreg', '_svm'))\n",
    "df_merged = pd.merge(df_merged, df_ridge[['Feature', 'Rank', 'Normalized Coefficient']], on='Feature', how='inner', suffixes=('', '_ridge'))\n",
    "\n",
    "# Calculate Average Rank and Average Coefficient\n",
    "df_merged['Average Rank'] = df_merged[['Rank_logreg', 'Rank_svm', 'Rank']].mean(axis=1)\n",
    "df_merged['Average Coefficient'] = df_merged[['Normalized Coefficient_logreg', 'Normalized Coefficient_svm', 'Normalized Coefficient']].mean(axis=1)\n",
    "\n",
    "# Prepare the final dataframe\n",
    "df_final = df_merged[['Feature', 'Average Rank', 'Average Coefficient']].sort_values(by='Average Rank')\n",
    "\n",
    "# Saving the \"median\" list with average coefficient to a CSV\n",
    "df_final.to_csv('features_median.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sequence-level Importance Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Identifying most common paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "mos_sequences = df['cleaned_mos'].str.split().apply(tuple) \n",
    "most_common_paths = Counter(mos_sequences).most_common()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some pre-processing with the cleaned_mos column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_data_types = df['cleaned_mos'].apply(type).unique()\n",
    "print(unique_data_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "float_rows = df[df['cleaned_mos'].apply(type) == float]\n",
    "print(float_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nan_count = df['cleaned_mos'].isna().sum()\n",
    "print(\"Number of NaN values in 'cleaned_mos' column:\", nan_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(subset=['cleaned_mos'], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualizing most common paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mos_labels = [' '.join(mos) for mos, count in most_common_paths[:10]]\n",
    "counts = [count for mos, count in most_common_paths[:10]]\n",
    "\n",
    "plt.figure(figsize=(10, 8))\n",
    "plt.barh(mos_labels, counts)\n",
    "plt.xlabel('Frequency')\n",
    "plt.ylabel('MOS Path')\n",
    "plt.title('Top 10 Most Common MOS Paths')\n",
    "plt.gca().invert_yaxis() \n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Grouping cleaned_mos by value and calculating Floor Percentage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mos_sequences_df = df['cleaned_mos'].str.split().apply(tuple).to_frame(name='mos_sequence_tuple').copy()\n",
    "mos_sequences_df['resolved_status'] = df['resolved']\n",
    "\n",
    "grouped_counts = mos_sequences_df.groupby(['mos_sequence_tuple', 'resolved_status']).size().unstack(fill_value=0)\n",
    "\n",
    "grouped_counts['total_occurrences'] = grouped_counts['resolved'] + grouped_counts['floor']\n",
    "\n",
    "grouped_counts['floor_percentage'] = (grouped_counts['floor'] / grouped_counts['total_occurrences']) * 100\n",
    "\n",
    "sorted_grouped_counts = grouped_counts.sort_values(by='floor_percentage', ascending=False)\n",
    "\n",
    "sorted_grouped_counts.reset_index(inplace=True)\n",
    "\n",
    "sorted_grouped_counts['mos_sequence'] = sorted_grouped_counts['mos_sequence_tuple'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "mos_df = sorted_grouped_counts[['mos_sequence', 'floor_percentage', 'total_occurrences', 'resolved', 'floor']].copy()\n",
    "\n",
    "mos_df.rename(columns={\n",
    "    'mos_sequence': 'MOS Sequence',\n",
    "    'floor_percentage': 'Floor Call Percentage',\n",
    "    'total_occurrences': 'Total Occurrences',\n",
    "    'resolved': 'Number of Resolved Calls',\n",
    "    'floor': 'Number of Floor Calls'\n",
    "}, inplace=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating weighted scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w_fp = 0.6 \n",
    "w_to = 0.4\n",
    "\n",
    "mos_df['weighted_score'] = (w_fp + w_to) / ((w_fp / (mos_df['Floor Call Percentage'] + 1)) + (w_to / (mos_df['Total Occurrences'] + 1)))\n",
    "\n",
    "mos_df_sorted = mos_df.sort_values(by='weighted_score', ascending=False)\n",
    "\n",
    "print(mos_df_sorted[['MOS Sequence', 'Floor Call Percentage', 'Total Occurrences', 'weighted_score']].head(10))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing occurrences of \"nl\" \"mt\" \"mo\" \"mm\" \"mn\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "keywords_to_remove = ['nl', 'mt', 'mo', 'mm', 'mn']\n",
    "\n",
    "def remove_keywords(sequence):\n",
    "    words = sequence.split()\n",
    "    cleaned_words = [word for word in words if word not in keywords_to_remove]\n",
    "    return ' '.join(cleaned_words)\n",
    "\n",
    "mos_df_sorted['MOS Sequence'] = mos_df_sorted['MOS Sequence'].apply(remove_keywords)\n",
    "\n",
    "print(mos_df_sorted['MOS Sequence'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mos_df_sorted.dropna(subset=['MOS Sequence'], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalizing the weighted score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "\n",
    "weighted_score = mos_df_sorted[['weighted_score']]\n",
    "\n",
    "mos_df_sorted['normalized_weighted_score'] = scaler.fit_transform(weighted_score)\n",
    "\n",
    "print(mos_df_sorted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mos_df_sorted.drop(columns=['Floor Call Percentage', 'weighted_score', 'normalized_weighted_score'], inplace=True)\n",
    "\n",
    "grouped_df = mos_df_sorted.groupby('MOS Sequence').sum().reset_index()\n",
    "\n",
    "grouped_df['Floor Percentage'] = (grouped_df['Number of Floor Calls'] / grouped_df['Total Occurrences']) * 100\n",
    "\n",
    "\n",
    "grouped_df.sort_values(by='Floor Percentage', ascending=False, inplace=True)\n",
    "\n",
    "print(grouped_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w_fp = 0.6\n",
    "w_to = 0.4\n",
    "grouped_df['weighted_score'] = (w_fp + w_to) / ((w_fp / (grouped_df['Floor Percentage'] + 1)) + (w_to / (grouped_df['Total Occurrences'] + 1)))\n",
    "\n",
    "grouped_df = grouped_df.sort_values(by='weighted_score', ascending=False)\n",
    "\n",
    "print(grouped_df[['MOS Sequence', 'Floor Percentage', 'Total Occurrences', 'weighted_score']].head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "scaler = MinMaxScaler()\n",
    "\n",
    "weighted_score = grouped_df[['weighted_score']]\n",
    "\n",
    "grouped_df['normalized_weighted_score'] = scaler.fit_transform(weighted_score)\n",
    "\n",
    "print(grouped_df[['MOS Sequence', 'Floor Percentage', 'Total Occurrences', 'weighted_score', 'normalized_weighted_score']].head(10))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_df.to_csv('cleaned_mos_analysis_weighted.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualizing most common codes, derived from the most impactful Sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patheffects as path_effects \n",
    "\n",
    "file_path = 'cleaned_mos_analysis_weighted.csv' \n",
    "data = pd.read_csv(file_path)\n",
    "\n",
    "data_subset = data.head(100)\n",
    "\n",
    "G = nx.Graph()\n",
    "\n",
    "node_sizes = {}\n",
    "\n",
    "for seq in data_subset['MOS Sequence']:\n",
    "    nodes = seq.split()\n",
    "    for i in range(len(nodes) - 1):\n",
    "        G.add_edge(nodes[i], nodes[i + 1])\n",
    "        node_sizes[nodes[i]] = node_sizes.get(nodes[i], 0) + 1\n",
    "    node_sizes[nodes[-1]] = node_sizes.get(nodes[-1], 0) + 1\n",
    "\n",
    "max_size = max(node_sizes.values())\n",
    "min_size = min(node_sizes.values())\n",
    "normalized_sizes = {node: ((size - min_size) / (max_size - min_size) * 200 + 20) for node, size in node_sizes.items()}\n",
    "\n",
    "pos = nx.spring_layout(G, seed=42)\n",
    "\n",
    "plt.figure(figsize=(12, 9), facecolor='white')\n",
    "ax = plt.gca()\n",
    "\n",
    "nodes = nx.draw_networkx_nodes(G, pos, node_size=[normalized_sizes[node] * 20 for node in G.nodes()],\n",
    "                               node_color=[normalized_sizes[node] for node in G.nodes()],\n",
    "                               cmap=plt.cm.Pastel2, alpha=1.0, edgecolors='#141414', linewidths=0)\n",
    "\n",
    "edges = nx.draw_networkx_edges(G, pos, edge_color='#c4c3c3', alpha=0.5, width=1)\n",
    "\n",
    "labels = nx.draw_networkx_labels(G, pos, font_size=8, font_color=\"black\")\n",
    "\n",
    "plt.title(\"MOS Codes which lead to greater Floor calls\", fontsize=20)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving nodes and edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "file_path = 'cleaned_mos_analysis_weighted.csv' \n",
    "data = pd.read_csv(file_path)\n",
    "\n",
    "data_subset = data.head(100)\n",
    "\n",
    "nodes = set()\n",
    "edges = []\n",
    "\n",
    "for index, row in data_subset.iterrows():\n",
    "    sequence = row['MOS Sequence'].split()\n",
    "    normalized_weighted_score = row['normalized_weighted_score']\n",
    "    for i in range(len(sequence) - 1):\n",
    "        source = sequence[i]\n",
    "        target = sequence[i + 1]\n",
    "        edges.append((source, target, normalized_weighted_score))\n",
    "        nodes.add(source)\n",
    "        nodes.add(target)\n",
    "\n",
    "nodes_df = pd.DataFrame({'Id': list(nodes)})\n",
    "edges_df = pd.DataFrame(edges, columns=['Source', 'Target', 'Weight'])\n",
    "\n",
    "nodes_file_path = 'nodes.csv'\n",
    "edges_file_path = 'edges.csv'\n",
    "\n",
    "nodes_df.to_csv(nodes_file_path, index=False)\n",
    "edges_df.to_csv(edges_file_path, index=False)\n",
    "\n",
    "print(f\"Nodes and edges CSV files have been saved to {nodes_file_path} and {edges_file_path}, respectively.\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}