Notebook_7_CV_Google_News.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook 2: CNN\n",
    "\n",
    "## Overview: \n",
    "\n",
    "1. Begin by importing and getting the embeddings and word to index mappings we created in [Notebook 1: Embed Words](Notebook_1_Embed_Words.ipynb)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "# ==================================================\n",
    "\n",
    "# Model Hyperparameters\n",
    "embedding_dim = 300  #Dimensionality of character embedding (default: 128)\n",
    "filter_sizes =  \"3,4,5\" #\"Comma-separated filter sizes (default: '3,4,5')\")\n",
    "num_filters = 50  #\"Number of filters per filter size (default: 128)\")\n",
    "dropout_keep_prob = 0.5 #\"Dropout keep probability (default: 0.5)\")\n",
    "l2_reg_lambda = 3.0 #\"L2 regularizaion lambda (default: 0.0)\")\n",
    "\n",
    "# Training parameters\n",
    "batch_size = 64 # \"Batch Size (default: 64)\")\n",
    "num_epochs = 500 #\"Number of training epochs (default: 200)\")\n",
    "evaluate_every = 100  #\"Evaluate model on dev set after this many steps (default: 100)\")\n",
    "checkpoint_every = 10000 # \"Save model after this many steps (default: 100)\")\n",
    "\n",
    "# Evaluation Parameters\n",
    "num_folds = 10 # number of cross validation folds \n",
    "\n",
    "# Misc Parameters\n",
    "allow_soft_placement = True # \"Allow device soft device placement\")\n",
    "log_device_placement = False  #\"Log placement of ops on devices\")\n",
    "display_train_steps = False # toggles output of training step results\n",
    "\n",
    "run_name = \"google-news\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cPickle as pickle\n",
    "from tensorflow.models.rnn.rnn_cell import BasicLSTMCell, LSTMCell \n",
    "import itertools\n",
    "from collections import Counter\n",
    "import csv\n",
    "import re\n",
    "import numpy as np\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "embeddings = None\n",
    "mappings = None\n",
    "rows = None\n",
    "\n",
    "    \n",
    "\n",
    "urlFinder = re.compile('\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*')\n",
    "atNameFinder = re.compile(r'@([A-Za-z0-9_]+)')\n",
    "atNameCounter = 0\n",
    "\n",
    "exclude_punc = set([\n",
    "        \"!\",\n",
    "        \"?\",\n",
    "        \".\",\n",
    "        \",\",\n",
    "        \":\",\n",
    "        \";\",\n",
    "        \"'\",\n",
    "        \"\\\"\",\n",
    "        \"“\",\n",
    "        \"’\",\n",
    "        \"-\"\n",
    "])\n",
    "\n",
    "sentences = []\n",
    "labels = []\n",
    "x = []\n",
    "y = []\n",
    "_y = []\n",
    "\n",
    "with open('data.csv', 'rb') as f:\n",
    "    reader = csv.reader(f, delimiter=',')   \n",
    "    \n",
    "    for row in reader:\n",
    "        words = []\n",
    "        \n",
    "        for word in row[1] \\\n",
    "            .strip() \\\n",
    "            .replace(\"&amp;\", \"\") \\\n",
    "            .replace(\"&gt;\",\"\") \\\n",
    "            .replace(\"&lt;\", \"\") \\\n",
    "            .lower().split():\n",
    "            \n",
    "            if urlFinder.match(word):\n",
    "                words.append(\"<URL/>\")\n",
    "            #elif atNameFinder.search(word):\n",
    "            #    words.append(\"<AT_NAME_%s/>\" % atNameCounter)\n",
    "            #    atNameCounter +=1\n",
    "            else:\n",
    "                word = ''.join(ch for ch in word if ch not in exclude_punc)\n",
    "                words.append(word)\n",
    "        sentences.append(words)\n",
    "        labels.append(([0, 1] if row[0] == \"example\" else [1, 0]))\n",
    "        _y.append(1 if row[0] == \"example\" else 0)\n",
    "\n",
    "\n",
    "sequence_length = max(len(i) for i in sentences)\n",
    "padded_sentences = []\n",
    "for i in range(len(sentences)):\n",
    "    sentence = sentences[i]\n",
    "    num_padding = sequence_length - len(sentence)\n",
    "    new_sentence = sentence + [\"<PAD/>\"] * num_padding\n",
    "    padded_sentences.append(new_sentence)\n",
    "    \n",
    " \n",
    "word_counts = Counter(itertools.chain(*padded_sentences))\n",
    "\n",
    "# Mapping from index to word\n",
    "vocabulary_inv = [x[0] for x in word_counts.most_common()]\n",
    "# Mapping from word to index\n",
    "vocabulary = {x: i for i, x in enumerate(vocabulary_inv)}\n",
    "\n",
    "x = np.array([[vocabulary[word] for word in sentence] for sentence in padded_sentences])\n",
    "y = np.array(labels)\n",
    "_y = np.array(_y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Embeddings.\n",
      "relevant word embeddings: 5841\n",
      "out of 8313 words\n",
      "Finished Loading 8313 Embeddings.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import sys\n",
    "\n",
    "# load embeddings\n",
    "print(\"Loading Embeddings.\")\n",
    "sys.stdout.flush()\n",
    "word_vecs = {}\n",
    "with open(\"GoogleNews-vectors-negative300.bin\", \"rb\") as f:\n",
    "    header = f.readline()\n",
    "    vocab_size, layer1_size = map(int, header.split())\n",
    "    binary_len = np.dtype('float32').itemsize * layer1_size\n",
    "    for line in xrange(vocab_size):\n",
    "        word = []\n",
    "        while True:\n",
    "            ch = f.read(1)\n",
    "            if ch == ' ':\n",
    "                word = ''.join(word)\n",
    "                break\n",
    "            if ch != '\\n':\n",
    "                word.append(ch)   \n",
    "        if word in vocabulary:\n",
    "            word_vecs[word] = np.fromstring(f.read(binary_len), dtype='float32')  \n",
    "        else:\n",
    "            f.read(binary_len)\n",
    "            \n",
    "print(\"relevant word embeddings: %s\" % len(word_vecs.keys()) )\n",
    "print(\"out of %s words\" % len(vocabulary))\n",
    "            \n",
    "for word in vocabulary:\n",
    "    if(word not in word_vecs.keys()):\n",
    "        word_vecs[word] = np.random.uniform(-1.0, 1.0, embedding_dim)\n",
    "embeddings = [ word_vecs[word] for word in word_vecs]\n",
    "\n",
    "for word in embeddings:\n",
    "    if len(word) != 300:\n",
    "        print(len(word))\n",
    "    \n",
    "\n",
    "print(\"Finished Loading %s Embeddings.\" % len(word_vecs.keys()))\n",
    "sys.stdout.flush()\n",
    "\n",
    "embeddings = np.array(embeddings, dtype=np.float32)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "class TextCNN(object):\n",
    "    \"\"\"\n",
    "    A CNN for text classification.\n",
    "    Uses an embedding layer, followed by a convolutional, max-pooling and softmax layer.\n",
    "    \"\"\"\n",
    "    def __init__(\n",
    "      self, sequence_length, num_classes, embeddings_tensor, embedding_size, filter_sizes, num_filters, l2_reg_lambda=0.0):\n",
    "\n",
    "        # Placeholders for input, output and dropout\n",
    "        self.input_x = tf.placeholder(tf.int32, [None, sequence_length], name=\"input_x\")\n",
    "        self.input_y = tf.placeholder(tf.float32, [None, num_classes], name=\"input_y\")\n",
    "        self.dropout_keep_prob = tf.placeholder(tf.float32, name=\"dropout_keep_prob\")\n",
    "        self.t = tf.placeholder(tf.int32, [None], name=\"t\")\n",
    "        self.f = tf.placeholder(tf.int32, [None], name=\"f\")\n",
    "\n",
    "        # Keeping track of l2 regularization loss (optional)\n",
    "        l2_loss = tf.constant(0.0)\n",
    "\n",
    "        # Embedding layer\n",
    "        print(\"Adding embedding layer\")\n",
    "        sys.stdout.flush()\n",
    "        with tf.device('/cpu:0'), tf.name_scope(\"embedding\"):\n",
    "            W = tf.Variable(embeddings_tensor,name=\"W\")\n",
    "            self.embedded_chars = tf.nn.embedding_lookup(W, self.input_x)\n",
    "            self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1)\n",
    "\n",
    "        # Create a convolution + maxpool layer for each filter size\n",
    "        pooled_outputs = []\n",
    "        for i, filter_size in enumerate(filter_sizes):\n",
    "            with tf.name_scope(\"conv-maxpool-%s\" % filter_size):\n",
    "                # Convolution Layer\n",
    "                filter_shape = [filter_size, embedding_size, 1, num_filters]\n",
    "                W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name=\"W\")\n",
    "                b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name=\"b\")\n",
    "                conv = tf.nn.conv2d(\n",
    "                    self.embedded_chars_expanded,\n",
    "                    W,\n",
    "                    strides=[1, 1, 1, 1],\n",
    "                    padding=\"VALID\",\n",
    "                    name=\"conv\")\n",
    "                # Apply nonlinearity\n",
    "                h = tf.nn.relu(tf.nn.bias_add(conv, b), name=\"relu\")\n",
    "                # Maxpooling over the outputs\n",
    "                pooled = tf.nn.max_pool(\n",
    "                    h,\n",
    "                    ksize=[1, sequence_length - filter_size + 1, 1, 1],\n",
    "                    strides=[1, 1, 1, 1],\n",
    "                    padding='VALID',\n",
    "                    name=\"pool\")\n",
    "                pooled_outputs.append(pooled)\n",
    "\n",
    "        # Combine all the pooled features\n",
    "        num_filters_total = num_filters * len(filter_sizes)\n",
    "        self.h_pool = tf.concat(3, pooled_outputs)\n",
    "        self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total])\n",
    "\n",
    "        # Add dropout\n",
    "        with tf.name_scope(\"dropout\"):\n",
    "            self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob)\n",
    "\n",
    "        # Final (unnormalized) scores and predictions\n",
    "        with tf.name_scope(\"output\"):\n",
    "            W = tf.Variable(tf.truncated_normal([num_filters_total, num_classes], stddev=0.1), name=\"W\")\n",
    "            b = tf.Variable(tf.constant(0.1, shape=[num_classes]), name=\"b\")\n",
    "            l2_loss += tf.nn.l2_loss(W)\n",
    "            l2_loss += tf.nn.l2_loss(b)\n",
    "            self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name=\"scores\")\n",
    "            self.predictions = tf.argmax(self.scores, 1, name=\"predictions\")\n",
    "\n",
    "        # CalculateMean cross-entropy loss\n",
    "        with tf.name_scope(\"loss\"):\n",
    "            losses = tf.nn.softmax_cross_entropy_with_logits(self.scores, self.input_y)\n",
    "            self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss\n",
    "\n",
    "        # Accuracy\n",
    "        with tf.name_scope(\"accuracy\"):\n",
    "            correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))\n",
    "            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, \"float\"), name=\"accuracy\")\n",
    "            \n",
    "        with tf.name_scope(\"sensitivity_and_specificity\"):\n",
    "            t = tf.fill(tf.shape(self.predictions), tf.cast(1, \"int64\"))\n",
    "            f = tf.fill(tf.shape(self.predictions), tf.cast(0, \"int64\"))\n",
    "            \n",
    "            TN = tf.size( \n",
    "                tf.where(\n",
    "                    tf.logical_and(\n",
    "                        tf.equal(\n",
    "                            tf.argmax(self.input_y, 1),\n",
    "                            f,\n",
    "                            name=\"equal_a\"\n",
    "                        ),\n",
    "                        tf.equal(\n",
    "                            self.predictions,\n",
    "                            f,\n",
    "                            name=\"equal_b\"\n",
    "                        )\n",
    "                    )\n",
    "                )\n",
    "            )\n",
    "\n",
    "            FP = tf.size(\n",
    "                tf.where(\n",
    "                    tf.logical_and(\n",
    "                        tf.equal(\n",
    "                            tf.argmax(self.input_y, 1),\n",
    "                            f,\n",
    "                            name=\"equal_c\"\n",
    "                        ),\n",
    "                        tf.equal(\n",
    "                            self.predictions,\n",
    "                           t,\n",
    "                            name=\"equal_d\"\n",
    "                        )\n",
    "                    )\n",
    "                )\n",
    "            )    \n",
    "\n",
    "            FN = tf.size(\n",
    "                tf.where(\n",
    "                    tf.logical_and(\n",
    "                        tf.equal(\n",
    "                            tf.argmax(self.input_y, 1),\n",
    "                            t,\n",
    "                            name=\"equal_f\"\n",
    "                        ),\n",
    "                        tf.equal(\n",
    "                            self.predictions,\n",
    "                            f,\n",
    "                            name=\"equal_g\"\n",
    "                        )\n",
    "                    )\n",
    "                )\n",
    "            )\n",
    "\n",
    "\n",
    "            TP = tf.size(\n",
    "                tf.where(\n",
    "                    tf.logical_and(\n",
    "                        tf.equal(\n",
    "                            tf.argmax(self.input_y, 1),\n",
    "                            t,\n",
    "                            name=\"equal_h\"\n",
    "                        ),\n",
    "                        tf.equal(\n",
    "                            self.predictions,\n",
    "                            t,\n",
    "                            name=\"equal_i\"\n",
    "                        )\n",
    "                    )\n",
    "                )\n",
    "            )\n",
    "\n",
    "            self.sensitivity = tf.div(\n",
    "                tf.cast(TP, \"float64\"),\n",
    "                tf.cast(\n",
    "                    tf.add(\n",
    "                        TP,\n",
    "                        FN\n",
    "                    ),\n",
    "                    \"float64\"\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            self.specificity = tf.div(\n",
    "                tf.cast(TN, \"float64\"),\n",
    "                tf.cast(\n",
    "                    tf.add(\n",
    "                        TN,\n",
    "                        FP\n",
    "                    ),\n",
    "                    \"float64\"\n",
    "                )\n",
    "            )\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def batch_iter(data, batch_size, num_epochs):\n",
    "    \"\"\"\n",
    "    Generates a batch iterator for a dataset.\n",
    "    \"\"\"\n",
    "    data = np.array(data)\n",
    "    data_size = len(data)\n",
    "    num_batches_per_epoch = int(len(data)/batch_size) + 1\n",
    "    for epoch in range(num_epochs):\n",
    "        # Shuffle the data at each epoch\n",
    "        shuffle_indices = np.random.permutation(np.arange(data_size))\n",
    "        shuffled_data = data[shuffle_indices]\n",
    "        for batch_num in range(num_batches_per_epoch):\n",
    "            start_index = batch_num * batch_size\n",
    "            end_index = min((batch_num + 1) * batch_size, data_size)\n",
    "            yield shuffled_data[start_index:end_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n",
      "Adding embedding layer\n",
      "Writing to /notebooks/AdHClassification/Expiriment #1.3 - Non-Candidate Tweets Included/runs/google-news-10f-cv/1458529001\n",
      "\n",
      "\n",
      "Evaluation:\n",
      "2016-03-21T02:57:11.088502: step 100, loss 3.34659, acc 0.911704"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "import datetime\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Data Preparatopn\n",
    "# ==================================================\n",
    "\n",
    "# Load data\n",
    "print(\"Loading data...\")\n",
    "np.random.seed(10)\n",
    "shuffle_indices = np.random.permutation(np.arange(len(y)))\n",
    "x_shuffled = x[shuffle_indices]\n",
    "y_shuffled = y[shuffle_indices]\n",
    "_y_shuffled = _y[shuffle_indices]\n",
    "\n",
    "skf = StratifiedKFold(_y_shuffled, n_folds=num_folds)\n",
    "\n",
    "fold_accuracies = []\n",
    "fold_specificities = []\n",
    "fold_sensitivities = []\n",
    "\n",
    "# Split train/test set\n",
    "# TODO: This is very crude, should use cross-validation\n",
    "#x_train, x_dev = x_shuffled[:len(x_shuffled)-1], x_shuffled[-len(x_shuffled)-1:]\n",
    "#y_train, y_dev = y_shuffled[:len(y_shuffled)-1], y_shuffled[-len(y_shuffled)-1:]\n",
    "#print(\"Vocabulary Size: {:d}\".format(len(vocabulary)))\n",
    "#print(\"Train/Dev split: {:d}/{:d}\".format(len(y_train), len(y_dev)))\n",
    "#print(\"Train Pos/Dev Pos Split {:d}/{:d}\"\n",
    "#      .format(\n",
    "#        len(\n",
    "#            [y for y in y_train if y[0] == 0 and y[1] == 1]\n",
    "#        ), len(\n",
    "#            [y for y in y_dev if y[0] == 0 and y[1] == 1]\n",
    "#        )))\n",
    "#print(\"Train Neg/Dev Neg Split {:d}/{:d}\"\n",
    "#      .format(\n",
    "#        len(\n",
    "#            [y for y in y_train if y[0] == 1 and y[1] == 0]\n",
    "#        ), len(\n",
    "#            [y for y in y_dev if y[0] == 1 and y[1] == 0]\n",
    "#        )))\n",
    "\n",
    "\n",
    "# Training\n",
    "# ==================================================\n",
    "\n",
    "for idx, fold in zip(skf, range(num_folds)):\n",
    "    x_train = x[idx[0]]\n",
    "    y_train = y[idx[0]]\n",
    "    \n",
    "    x_dev = x[idx[1]]\n",
    "    y_dev = y[idx[1]]\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    with tf.Graph().as_default():\n",
    "        embeddings_tensor = tf.convert_to_tensor(embeddings)\n",
    "        session_conf = tf.ConfigProto(\n",
    "          allow_soft_placement=allow_soft_placement,\n",
    "          log_device_placement=log_device_placement)\n",
    "        sess = tf.Session(config=session_conf)\n",
    "        with sess.as_default():\n",
    "            cnn = TextCNN(\n",
    "                sequence_length=x_train.shape[1],\n",
    "                num_classes=2,\n",
    "                embeddings_tensor=embeddings_tensor,\n",
    "                embedding_size=embedding_dim,\n",
    "                filter_sizes=map(int, filter_sizes.split(\",\")),\n",
    "                num_filters=num_filters,\n",
    "                l2_reg_lambda=l2_reg_lambda)\n",
    "\n",
    "            # Define Training procedure\n",
    "            global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "            optimizer = tf.train.AdamOptimizer(1e-4)\n",
    "            grads_and_vars = optimizer.compute_gradients(cnn.loss)\n",
    "            train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)\n",
    "\n",
    "            # Keep track of gradient values and sparsity (optional)\n",
    "            grad_summaries = []\n",
    "            for g, v in grads_and_vars:\n",
    "                if g is not None:\n",
    "                    grad_hist_summary = tf.histogram_summary(\"{}/grad/hist\".format(v.name), g)\n",
    "                    sparsity_summary = tf.scalar_summary(\"{}/grad/sparsity\".format(v.name), tf.nn.zero_fraction(g))\n",
    "                    grad_summaries.append(grad_hist_summary)\n",
    "                    grad_summaries.append(sparsity_summary)\n",
    "            grad_summaries_merged = tf.merge_summary(grad_summaries)\n",
    "\n",
    "            # Output directory for models and summaries\n",
    "            timestamp = str(int(time.time()))\n",
    "            out_dir = os.path.abspath(os.path.join(os.path.curdir, \"runs\", run_name,timestamp))\n",
    "            print(\"Writing to {}\\n\".format(out_dir))\n",
    "\n",
    "            # Summaries for loss and accuracy\n",
    "            loss_summary = tf.scalar_summary(\"loss\", cnn.loss)\n",
    "            acc_summary = tf.scalar_summary(\"accuracy\", cnn.accuracy)\n",
    "            \n",
    "            sens_summary = tf.scalar_summary(\"sensitivity\", cnn.sensitivity)\n",
    "            spec_summary = tf.scalar_summary(\"specificity\", cnn.specificity)\n",
    "\n",
    "            # Train Summaries\n",
    "            train_summary_op = tf.merge_summary([loss_summary, acc_summary,sens_summary, spec_summary, grad_summaries_merged])\n",
    "            train_summary_dir = os.path.join(out_dir, \"summaries\", \"train\", str(fold))\n",
    "            train_summary_writer = tf.train.SummaryWriter(train_summary_dir, sess.graph_def)\n",
    "\n",
    "            # Dev summaries\n",
    "            dev_summary_op = tf.merge_summary([loss_summary, acc_summary, sens_summary, spec_summary])\n",
    "            dev_summary_dir = os.path.join(out_dir, \"summaries\", \"dev\", str(fold))\n",
    "            dev_summary_writer = tf.train.SummaryWriter(dev_summary_dir, sess.graph_def)\n",
    "\n",
    "            # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it\n",
    "            checkpoint_dir = os.path.abspath(os.path.join(out_dir, \"checkpoints\"))\n",
    "            checkpoint_prefix = os.path.join(checkpoint_dir, \"model\")\n",
    "            if not os.path.exists(checkpoint_dir):\n",
    "                os.makedirs(checkpoint_dir)\n",
    "            saver = tf.train.Saver(tf.all_variables())\n",
    "\n",
    "            # Initialize all variables\n",
    "            sess.run(tf.initialize_all_variables())\n",
    "    \n",
    "            def train_step(x_batch, y_batch):\n",
    "                \"\"\"\n",
    "                A single training step\n",
    "                \"\"\"\n",
    "                feed_dict = {\n",
    "                  cnn.input_x: x_batch,\n",
    "                  cnn.input_y: y_batch,\n",
    "                  cnn.dropout_keep_prob: dropout_keep_prob,\n",
    "                  cnn.t: [ len(y_batch)],\n",
    "                  cnn.f: [ len(y_batch)]\n",
    "                }\n",
    "                _, step, summaries, loss, accuracy = sess.run(\n",
    "                    [train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy],\n",
    "                    feed_dict)\n",
    "                time_str = datetime.datetime.now().isoformat()\n",
    "                if(display_train_steps):\n",
    "                    print(\"{}: step {}, loss {:g}, acc {:g}\".format(time_str, step, loss, accuracy))\n",
    "                train_summary_writer.add_summary(summaries, step)\n",
    "\n",
    "            def dev_step(x_batch, y_batch, writer=None):\n",
    "                \"\"\"\n",
    "                Evaluates model on a dev set\n",
    "                \"\"\"\n",
    "                feed_dict = {\n",
    "                  cnn.input_x: x_batch,\n",
    "                  cnn.input_y: y_batch,\n",
    "                  cnn.dropout_keep_prob: 1.0,\n",
    "                  cnn.t: [ len(y_batch)],\n",
    "                  cnn.f: [ len(y_batch)]\n",
    "                }\n",
    "                step, summaries, loss, accuracy, sensitivity, specificity = sess.run(\n",
    "                    [global_step, dev_summary_op, cnn.loss, cnn.accuracy, cnn.sensitivity, cnn.specificity],\n",
    "                    feed_dict)\n",
    "                time_str = datetime.datetime.now().isoformat()\n",
    "                print(\"{}: step {}, loss {:g}, acc {:g} sens {:g} spec {:g}\".format(time_str, step, loss, accuracy, sensitivity, specificity))\n",
    "                \n",
    "                if writer:\n",
    "                    writer.add_summary(summaries, step)\n",
    "                return accuracy, sensitivity, specificity\n",
    "\n",
    "            # Generate batches\n",
    "            batches = batch_iter(\n",
    "                zip(x_train, y_train), batch_size, num_epochs)\n",
    "            # Training loop. For each batch...\n",
    "            for batch in batches:\n",
    "                x_batch, y_batch = zip(*batch)\n",
    "                train_step(x_batch, y_batch)\n",
    "                current_step = tf.train.global_step(sess, global_step)\n",
    "                if current_step % checkpoint_every == 0:\n",
    "                    path = saver.save(sess, checkpoint_prefix, global_step=current_step)\n",
    "                    print(\"Saved model checkpoint to {}\\n\".format(path))\n",
    "                if current_step % evaluate_every == 0:\n",
    "                    print(\"\\nEvaluation:\")\n",
    "                    dev_step(x_dev, y_dev, writer=dev_summary_writer)\n",
    "                    print(\"\")\n",
    "            \n",
    "            print(\"\\nFinal Evaluation for fold %s:\" % fold)\n",
    "            acc, sens, spec = dev_step(x_dev, y_dev, writer=dev_summary_writer)\n",
    "            fold_accuracies.append(acc)\n",
    "            fold_sensitivities.append(sens)\n",
    "            print(\"Sensitivity: %s\" % sens)\n",
    "            fold_specificities.append(spec)\n",
    "            print(\"Specificity: %s\" % spec)\n",
    "  \n",
    "        \n",
    "        print(\"\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "final_accuracy = sum(fold_accuracies) / float(len(fold_accuracies))\n",
    "print(\"10-fold final accuracy: %s\" % final_accuracy)\n",
    "final_specificity = sum(fold_specificities) / float(len(fold_specificities))\n",
    "print(\"10-fold final specificity: %s\" % final_specificity)\n",
    "final_sensitivities = sum(fold_sensitivities) / float(len(fold_sensitivities))\n",
    "print(\"10-fold final sensitivity: %s\" % final_sensitivities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}