2. Textbook Exploration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import spacy\n",
    "import spotlight\n",
    "from collections import Counter\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.decomposition import NMF, LatentDirichletAllocation\n",
    "\n",
    "input_dir = 'textbooks/'\n",
    "data_dir = 'data/'\n",
    "metadata_file = 'data/metadata.csv'\n",
    "toc_file = 'toc.pkl'\n",
    "text_file = 'text.pkl'\n",
    "spotlight_server = 'http://192.168.99.101:2222/rest/annotate'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data and spacy model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "isbns = os.listdir(input_dir)\n",
    "\n",
    "with open(os.path.join(data_dir, toc_file), 'rb') as fp:\n",
    "    all_toc = pickle.load(fp) \n",
    "\n",
    "with open(os.path.join(data_dir, text_file), 'rb') as fp:\n",
    "    all_text = pickle.load(fp) \n",
    "    \n",
    "nlp = spacy.load('en')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load metadata and calculate number of pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISBN</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>subject</th>\n",
       "      <th>imprint</th>\n",
       "      <th>sold_by</th>\n",
       "      <th>start_page</th>\n",
       "      <th>end_page</th>\n",
       "      <th>num_pages</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9781429219617</td>\n",
       "      <td>BIOLOGY OF PLANTS</td>\n",
       "      <td>PETER H RAVEN</td>\n",
       "      <td>BIOLOGY</td>\n",
       "      <td>FREEMAN/WORTH</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>21</td>\n",
       "      <td>747</td>\n",
       "      <td>863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9781429242301</td>\n",
       "      <td>INTRODUCING PSYCHOLOGY</td>\n",
       "      <td>DANIEL L SCHACTER</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>FREEMAN/WORTH</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>38</td>\n",
       "      <td>526</td>\n",
       "      <td>616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9781429298643</td>\n",
       "      <td>LIFE: THE SCIENCE OF BIOLOGY</td>\n",
       "      <td>DAVID E SADAVA</td>\n",
       "      <td>BIOLOGY</td>\n",
       "      <td>FREEMAN/WORTH</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>51</td>\n",
       "      <td>1297</td>\n",
       "      <td>1447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9781429298902</td>\n",
       "      <td>PSYCHOLOGY: A CONCISE INTRODUCTION</td>\n",
       "      <td>RICHARD A GRIGGS</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>WORTH PUBLISHERS</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>22</td>\n",
       "      <td>464</td>\n",
       "      <td>545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9781464126147</td>\n",
       "      <td>MOLECULAR BIOLOGY: PRINCIPLES AND PRACTICE</td>\n",
       "      <td>MICHAEL M COX</td>\n",
       "      <td>BIOLOGY</td>\n",
       "      <td>W. H. FREEMAN</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>30</td>\n",
       "      <td>828</td>\n",
       "      <td>934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9781464135958</td>\n",
       "      <td>WHAT IS LIFE? A GUIDE TO BIOLOGY</td>\n",
       "      <td>JAY PHELAN</td>\n",
       "      <td>BIOLOGY</td>\n",
       "      <td>FREEMAN/WORTH</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>34</td>\n",
       "      <td>718</td>\n",
       "      <td>773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9781464140815</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>DAVID G MYERS</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>FREEMAN/WORTH</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>59</td>\n",
       "      <td>751</td>\n",
       "      <td>985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9781464154072</td>\n",
       "      <td>EXPLORING PSYCHOLOGY</td>\n",
       "      <td>DAVID G MYERS</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>WORTH PUBLISHERS</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>59</td>\n",
       "      <td>662</td>\n",
       "      <td>892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9781464171703</td>\n",
       "      <td>ABNORMAL PSYCHOLOGY</td>\n",
       "      <td>RONALD J COMER</td>\n",
       "      <td>PSYCHOLOGY</td>\n",
       "      <td>WORTH PUBLISHERS</td>\n",
       "      <td>Macmillan Higher Education</td>\n",
       "      <td>33</td>\n",
       "      <td>699</td>\n",
       "      <td>852</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            ISBN                                       title  \\\n",
       "0  9781429219617                           BIOLOGY OF PLANTS   \n",
       "1  9781429242301                      INTRODUCING PSYCHOLOGY   \n",
       "2  9781429298643                LIFE: THE SCIENCE OF BIOLOGY   \n",
       "3  9781429298902          PSYCHOLOGY: A CONCISE INTRODUCTION   \n",
       "4  9781464126147  MOLECULAR BIOLOGY: PRINCIPLES AND PRACTICE   \n",
       "5  9781464135958            WHAT IS LIFE? A GUIDE TO BIOLOGY   \n",
       "6  9781464140815                                  PSYCHOLOGY   \n",
       "7  9781464154072                        EXPLORING PSYCHOLOGY   \n",
       "8  9781464171703                         ABNORMAL PSYCHOLOGY   \n",
       "\n",
       "              author     subject           imprint  \\\n",
       "0      PETER H RAVEN     BIOLOGY     FREEMAN/WORTH   \n",
       "1  DANIEL L SCHACTER  PSYCHOLOGY     FREEMAN/WORTH   \n",
       "2     DAVID E SADAVA     BIOLOGY     FREEMAN/WORTH   \n",
       "3   RICHARD A GRIGGS  PSYCHOLOGY  WORTH PUBLISHERS   \n",
       "4      MICHAEL M COX     BIOLOGY     W. H. FREEMAN   \n",
       "5         JAY PHELAN     BIOLOGY     FREEMAN/WORTH   \n",
       "6      DAVID G MYERS  PSYCHOLOGY     FREEMAN/WORTH   \n",
       "7      DAVID G MYERS  PSYCHOLOGY  WORTH PUBLISHERS   \n",
       "8     RONALD J COMER  PSYCHOLOGY  WORTH PUBLISHERS   \n",
       "\n",
       "                      sold_by  start_page  end_page  num_pages  \n",
       "0  Macmillan Higher Education          21       747        863  \n",
       "1  Macmillan Higher Education          38       526        616  \n",
       "2  Macmillan Higher Education          51      1297       1447  \n",
       "3  Macmillan Higher Education          22       464        545  \n",
       "4  Macmillan Higher Education          30       828        934  \n",
       "5  Macmillan Higher Education          34       718        773  \n",
       "6  Macmillan Higher Education          59       751        985  \n",
       "7  Macmillan Higher Education          59       662        892  \n",
       "8  Macmillan Higher Education          33       699        852  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata = pd.read_csv(metadata_file, dtype = {'ISBN': 'str'})\n",
    "metadata['num_pages'] = [len(all_text[isbn]) for isbn in metadata['ISBN']]\n",
    "metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example: *Introducing Psychology*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Psychology\n",
      "1 䉱 Psychology’s Roots:\n",
      "['1', '䉱', 'psychology', '’s', 'root', ':']\n"
     ]
    }
   ],
   "source": [
    "isbn = '9781429242301'\n",
    "\n",
    "start_page = metadata.loc[metadata['ISBN'] == isbn, 'start_page'].values[0]\n",
    "end_page = metadata.loc[metadata['ISBN'] == isbn, 'end_page'].values[0]\n",
    "\n",
    "# Extract the content portion of the textbook, and combine the pages\n",
    "text = all_text[isbn][(start_page-1):(end_page)]\n",
    "text = ' '.join(text)\n",
    "\n",
    "# Parse the textbook with spacy\n",
    "doc = nlp(text)\n",
    "\n",
    "# Some spacy examples\n",
    "token = doc[2]\n",
    "print(token)\n",
    "sentence = next(doc.sents)\n",
    "print(sentence)\n",
    "print([word.lemma_ for word in sentence])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use the DBpedia Spotlight server to annotate the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Confidence = confidence score for disambiguation / linking\n",
    "# Support = number of inlinks to the wikipedia entry\n",
    "\n",
    "# Low support with high confidence\n",
    "\n",
    "annotations = spotlight.annotate(spotlight_server,\n",
    "                                 doc.string,\n",
    "                                 confidence=0.9, support=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'URI': 'http://dbpedia.org/resource/Brain',\n",
       " 'offset': 793,\n",
       " 'percentageOfSecondRank': 4.2737093380401676e-18,\n",
       " 'similarityScore': 1.0,\n",
       " 'support': 10026,\n",
       " 'surfaceForm': 'The Brain',\n",
       " 'types': ''}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The French Connection',\n",
       " 'The Brain',\n",
       " 'ABRAHAM LINCOLN',\n",
       " 'Pony Express',\n",
       " 'Missouri',\n",
       " 'California',\n",
       " 'Harvard Medical School',\n",
       " 'sci',\n",
       " 'Harvard University',\n",
       " 'Abraham Lincoln',\n",
       " 'Pony Express',\n",
       " 'Kellogg Company',\n",
       " 'rela',\n",
       " 'science fiction',\n",
       " 'functional magnetic resonance imaging',\n",
       " 'risky business',\n",
       " 'emo',\n",
       " 'empiricism',\n",
       " 'phrenology',\n",
       " 'tics']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotation_names = [ann['surfaceForm'] for ann in annotations\n",
    "                   if ann['surfaceForm'] != '/12']\n",
    "# TODO:\n",
    "# Exclude '/12' or certain 'types': 'DBpedia:TimePeriod,DBpedia:Year'\n",
    "\n",
    "annotation_names[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic NLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "people - 1179\n",
      "brain - 725\n",
      "person - 497\n",
      "behavior - 453\n",
      "memory - 440\n",
      "example - 429\n",
      "time - 393\n",
      "information - 383\n",
      "children - 373\n",
      "disorder - 275\n",
      "way - 267\n",
      "study - 263\n",
      "mind - 255\n",
      "personality - 253\n",
      "self - 250\n",
      "system - 234\n",
      "disorders - 228\n",
      "body - 225\n",
      "life - 224\n",
      "psychology - 223\n",
      "experience - 217\n",
      "group - 209\n",
      "theory - 208\n",
      "cortex - 203\n",
      "activity - 200\n",
      "intelligence - 200\n",
      "response - 200\n",
      "world - 199\n",
      "fact - 190\n",
      "participants - 188\n"
     ]
    }
   ],
   "source": [
    "words = [token.text for token in doc if \n",
    "         token.is_stop != True and \n",
    "         token.is_punct != True and \n",
    "         token.is_digit != True and \n",
    "         token.pos_ == \"NOUN\" and\n",
    "         len(token) > 2]\n",
    "\n",
    "# five most common tokens\n",
    "word_freq = Counter(words)\n",
    "common_words = word_freq.most_common(30)\n",
    "\n",
    "for word, count in common_words:\n",
    "    print('{} - {}'.format(word, count))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LDA for Topic Modelling (all textbooks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "textbook_text = []\n",
    "\n",
    "for isbn in isbns:\n",
    "    start_page = metadata.loc[metadata['ISBN'] == isbn, 'start_page'].values[0]\n",
    "    end_page = metadata.loc[metadata['ISBN'] == isbn, 'end_page'].values[0]\n",
    "\n",
    "    # Extract the content portion of the textbook, and combine the pages\n",
    "    text = all_text[isbn][(start_page-1):(end_page)]\n",
    "    text = ' '.join(text)\n",
    "\n",
    "    # Parse the textbook with spacy\n",
    "    doc = nlp(text)\n",
    "\n",
    "    text_clean = [token.text for token in doc if \n",
    "                  token.is_stop != True and \n",
    "                  token.is_punct != True and \n",
    "                  token.is_digit != True and \n",
    "                  token.pos_ == \"NOUN\" and\n",
    "                  len(token) > 2]\n",
    "    \n",
    "    textbook_text.append(' '.join(text_clean))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_features = 1000\n",
    "n_topics = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting tf features for LDA...\n"
     ]
    }
   ],
   "source": [
    "# Use tf (raw term count) features for LDA\n",
    "print(\"Extracting tf features for LDA...\")\n",
    "tf_vectorizer = TfidfVectorizer(min_df=1,\n",
    "                                max_features=n_features,\n",
    "                                stop_words='english')\n",
    "\n",
    "\n",
    "tf = tf_vectorizer.fit_transform(textbook_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting LDA models with tf features\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "             evaluate_every=-1, learning_decay=0.7,\n",
       "             learning_method='online', learning_offset=50.0,\n",
       "             max_doc_update_iter=100, max_iter=10, mean_change_tol=0.001,\n",
       "             n_jobs=1, n_topics=10, perp_tol=0.1, random_state=0,\n",
       "             topic_word_prior=None, total_samples=1000000.0, verbose=0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Fitting LDA models with tf features\")\n",
    "lda = LatentDirichletAllocation(n_topics=n_topics, \n",
    "                                max_iter=10,\n",
    "                                learning_method='online',\n",
    "                                learning_offset=50.,\n",
    "                                random_state=0)\n",
    "lda.fit(tf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Topics in LDA model:\n",
      "\n",
      "Topic #0:\n",
      "water cells cell plants cid species 132 time plant blood sporophyte environments changes heart rate ions tasks genomes segment energy\n",
      "\n",
      "Topic #1:\n",
      "rem reaction communication separation rule repair time definition proteins sample potentials bird issues ratio carbohydrates thing job populations failure individuals\n",
      "\n",
      "Topic #2:\n",
      "people brain disorder memory person life behavior personality children study disorders arousal age scores family time environment performance activity information\n",
      "\n",
      "Topic #3:\n",
      "polypeptide shape complex glands dna male concentrations evidence humans question reactions answers mating death hemisphere chemicals height lack pro core\n",
      "\n",
      "Topic #4:\n",
      "atoms branch fear mechanisms resources layers domain skin trees orientation algae schizophrenia past face steps salt methods transport survival pre\n",
      "\n",
      "Topic #5:\n",
      "zygote words walls cues environment disorders females ments view matrix source theory mass polypeptide flowering turn energy therapists humans variables\n",
      "\n",
      "Topic #6:\n",
      "habitats respiration receptors appearance mammals hormones section decision race type mutation access carbon childhood connections book synthesis air decades today\n",
      "\n",
      "Topic #7:\n",
      "consciousness testing stages woman stomach opening ers organs structures cells fixation value techniques lack names key middle origin risk seconds\n",
      "\n",
      "Topic #8:\n",
      "membrane sleep interactions origin babies theory responses traits nutrients father consciousness systems lines target boys value influence residues sugar measure\n",
      "\n",
      "Topic #9:\n",
      "conditions males recognition mother story stomach sion frequencies fish chromatin needs testing forces mass ecosystems layer rna ent elements abilities\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nTopics in LDA model:\")\n",
    "print()\n",
    "\n",
    "def print_top_words(model, feature_names, n_top_words):\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        print(\"Topic #%d:\" % topic_idx)\n",
    "        print(\" \".join([feature_names[i]\n",
    "                        for i in topic.argsort()[:-n_top_words - 1:-1]]))\n",
    "        print()\n",
    "    \n",
    "tf_feature_names = tf_vectorizer.get_feature_names()\n",
    "print_top_words(lda, tf_feature_names, 20)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}