train_tfkld.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from tfkld import TFKLD\n",
    "import pandas as pd\n",
    "from scipy.sparse import vstack, lil_matrix\n",
    "from cPickle import load, dump\n",
    "from scipy.sparse.linalg import svds\n",
    "import numpy, gzip\n",
    "import cPickle as pickle\n",
    "import os\n",
    "import numpy as np\n",
    "from sklearn.externals import joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Helper Functions\n",
    "\n",
    "This code was pulled from this git repo https://github.com/smujjiga/tfkld I've included the fe_quora.py file in this repository"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class DimReduction(object):\n",
    "    def __init__(self, M, K):\n",
    "        \"\"\" Initialization\n",
    "        \"\"\"\n",
    "        self.M = M\n",
    "        self.K = K\n",
    "\n",
    "\n",
    "    def svd(self):\n",
    "        U, s, Vt = svds(self.M, k=self.K)\n",
    "        W = U.dot(numpy.diag(s))\n",
    "        H = Vt\n",
    "        return (W, H)\n",
    "\n",
    "\n",
    "    def nmf(self):\n",
    "        pass\n",
    "\n",
    "\n",
    "def dr(trainX, devX, testX, K=200):\n",
    "    n_train = trainX.shape[0]\n",
    "    n_dev = devX.shape[0]\n",
    "    n_test = testX.shape[0]\n",
    "\n",
    "    M = vstack([trainX, devX, testX])\n",
    "    print 'M.shape = {}'.format(M.shape)\n",
    "    dr = DimReduction(M, K)\n",
    "    W, H = dr.svd()\n",
    "    # Split data\n",
    "    trainX = W[:n_train, :]\n",
    "    devX = W[n_train:n_train+n_dev, :]\n",
    "    testX = W[n_train+n_dev:, :]\n",
    "\n",
    "    return trainX, devX, testX\n",
    "\n",
    "\n",
    "def split_train(train):\n",
    "    if os.path.exists(\"dev_data.csv\"):\n",
    "        return \n",
    "\n",
    "    print (\"Splitting train data into train and dev sets\")\n",
    "    df = pd.read_csv(train)\n",
    "    n = len(df)\n",
    "    # Shuffle and split\n",
    "    df = df.iloc[np.random.permutation(n)]\n",
    "    per = .75\n",
    "    df[:int(per*n)].to_csv(\"train_data.csv\", index=False)\n",
    "    df[int(per*n):].to_csv(\"dev_data.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Training Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "split_train(\"train.csv\")\n",
    "    \n",
    "TRAIN_DATA = \"train_data.csv\"\n",
    "DEV_DATA = \"dev_data.csv\"\n",
    "TEST_DATA = \"test.csv\"\n",
    "\n",
    "print \"Loading Train data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(TRAIN_DATA, usecols=['question1', 'question2', 'is_duplicate'])\n",
    "train_df = train_df.fillna(' ')\n",
    "trainY = train_df['is_duplicate']\n",
    "trainX = list()\n",
    "for i, row in train_df.iterrows():\n",
    "    trainX.append(row['question1'])\n",
    "    trainX.append(row['question2'])\n",
    "print (\"Train: {0}\".format(train_df.shape))\n",
    "del train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tfkld = TFKLD()\n",
    "print (\"Fitting the model\")\n",
    "tfkld.fit(trainX, trainY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "pickle.dump(tfkld, open(\"tfkld.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "my_vec = tfkld.transform([\"Hello what is your name?\"])\n",
    "print my_vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Load and use Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "my_file = open(\"tfkld.pkl\", \"r\")\n",
    "tfkld_model = pickle.load(my_file)\n",
    "my_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "my_vec = tfkld_model.transform([\"test this sentence\"])\n",
    "print my_vec[0]\n",
    "print \"\\n next vec \\n\"\n",
    "# print my_vec[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}