CCD_word_movers_distance.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Code to calculate average Word Movers Distance between adjacent sentences in a text file. Uses word2vec pretrained Google News embeddings, available from https://github.com/mmihaltz/word2vec-GoogleNews-vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyemd import emd\n",
    "from gensim.similarities import WmdSimilarity\n",
    "from gensim.models import KeyedVectors\n",
    "from gensim.test.utils import get_tmpfile\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import numpy\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import spatial\n",
    "import numpy as np\n",
    "from pandas import DataFrame, Series, ExcelWriter\n",
    "import more_itertools\n",
    "import os\n",
    "import glob\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk import sent_tokenize\n",
    "from nltk import pos_tag\n",
    "import pandas as pd\n",
    "stopWords = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'GoogleNews-vectors-negative300.bin.gz'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KeyedVectors.load_word2vec_format(filename, binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# play around with vectors\n",
    "\n",
    "model.most_similar_cosmul(positive=['woman', 'king'], negative=['man'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "text1 = ''''''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "text2 = ''''''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculate distance between two sentences using WMD algorithm\n",
    "distance = model.wmdistance(text1, text2)\n",
    "print ('distance = %.3f' % distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# sentence to sentence word movers distance. if sentence has no words in vocab, returns infinity\n",
    "\n",
    "tags = ['FW','JJ','JJR','JJS', 'NN', 'NNS', 'NNP', 'NNPS', 'RB', 'RBR','RBS','VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "stopWords = stopwords.words('english')\n",
    "\n",
    "files_list = []\n",
    "mean_wmd_list = []\n",
    "sd_wmd_list = []\n",
    "median_wmd_list = []\n",
    "\n",
    "for file in os.listdir(path):       #for every file in the path\n",
    "    if file.endswith(\"*.txt\"):       #specify certain files\n",
    "        files_list.append(file)     #save the file's name in the list_files list\n",
    "        file_encoding = 'utf8'\n",
    "        with open(file, encoding=file_encoding, errors = 'ignore') as f:\n",
    "            text = f.read() #reads over each line of the file\n",
    "            sentences = sent_tokenize(text)\n",
    "            new_sent = []\n",
    "            wmds = []\n",
    "            for sentence in sentences:\n",
    "                tokens = word_tokenize(sentence)\n",
    "                tagged = pos_tag(tokens)\n",
    "                content = [(word, tag) for (word, tag) in tagged if tag in tags]\n",
    "                content2 = [(word, tag) for (word, tag) in content if word.isalpha()] #restricts string to alphabetic characters only\n",
    "                content3 = [word for (word, tag) in content2 if word not in stopWords]\n",
    "                lemmas = [getLemma(word) for word in content3]\n",
    "                sent = ' '.join(lemmas)\n",
    "                new_sent.append(sent)\n",
    "                \n",
    "            for v, w in zip(new_sent[:-1],new_sent[1:]):\n",
    "                distance = model.wmdistance(v, w)\n",
    "                wmds.append(distance)\n",
    "                \n",
    "            wmds2 = [x for x in wmds if ~np.isinf(x)]\n",
    "                \n",
    "            mean_wmd = np.mean(wmds2)\n",
    "            sd_wmd = np.std(wmds2)\n",
    "            median_wmd = np.median(wmds2)\n",
    "            \n",
    "            mean_wmd_list.append(mean_wmd)\n",
    "            sd_wmd_list.append(sd_wmd)\n",
    "            median_wmd_list.append(median_wmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#check distribution\n",
    "plt.hist(wmds2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for files with one sentence, splits in half and does wmd between each half\n",
    "\n",
    "tags = ['FW','JJ','JJR','JJS', 'NN', 'NNS', 'NNP', 'NNPS', 'RB', 'RBR','RBS','VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "stopWords = stopwords.words('english')\n",
    "\n",
    "files_list = []\n",
    "mean_wmd_list = []\n",
    "sd_wmd_list = []\n",
    "median_wmd_list = []\n",
    "\n",
    "for file in os.listdir(path):       #for every file in the path\n",
    "    if file.endswith(\"*.txt\"):  # specify certain files\n",
    "        files_list.append(file)\n",
    "        file_encoding = 'utf8'\n",
    "        new_sent = []\n",
    "        new_sent2 = []\n",
    "        wmds = []\n",
    "        with open(file, encoding=file_encoding, errors = 'ignore') as f:\n",
    "            text = f.read() #reads over each line of the file\n",
    "            sentences = sent_tokenize(text)\n",
    "            sentence_length = len(sentences)\n",
    "            if sentence_length <2:\n",
    "                for sentence in sentences:\n",
    "                    firstpart, secondpart = word_tokenize(sentence)[:len(word_tokenize(sentence))//2], word_tokenize(sentence)[len(word_tokenize(sentence))//2:]\n",
    "                    new_sent.append(' '.join(firstpart))\n",
    "                    new_sent.append(' '.join(secondpart))\n",
    "                    \n",
    "            for sentence in new_sent:\n",
    "                tokens = word_tokenize(sentence)\n",
    "                tagged = pos_tag(tokens)\n",
    "                content = [(word, tag) for (word, tag) in tagged if tag in tags]\n",
    "                content2 = [(word, tag) for (word, tag) in content if word.isalpha()] #restricts string to alphabetic characters only\n",
    "                content3 = [word for (word, tag) in content2 if word not in stopWords]\n",
    "                lemmas = [getLemma(word) for word in content3]\n",
    "                sent = ' '.join(lemmas)\n",
    "                new_sent2.append(sent)\n",
    "                \n",
    "            \n",
    "            for v, w in zip(new_sent2[:-1],new_sent2[1:]):\n",
    "                distance = model.wmdistance(v, w)\n",
    "                wmds.append(distance)\n",
    "\n",
    "            wmds2 = [x for x in wmds if ~np.isinf(x)]\n",
    "\n",
    "            mean_wmd = np.mean(wmds2)\n",
    "            sd_wmd = np.std(wmds2)\n",
    "            median_wmd = np.median(wmds2)\n",
    "\n",
    "            mean_wmd_list.append(mean_wmd)\n",
    "            sd_wmd_list.append(sd_wmd)\n",
    "            median_wmd_list.append(median_wmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tokens(text):\n",
    "    '''\n",
    "    takes input of text string, preprocesses and returns list of tokens\n",
    "    '''\n",
    "    \n",
    "    text = text.lower()\n",
    "    tokens = word_tokenize(text)\n",
    "    tokens = [word for word in tokens if word not in stopWords]\n",
    "    tokens = [word for word in tokens if word.isalpha()] #restricts string to alphabetic characters only\n",
    "    #tokens = [getLemma(token) for token in tokens]\n",
    " \n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getLemma(word):\n",
    "    lemma = wn.morphy(word)\n",
    "    if lemma is None:\n",
    "        return word\n",
    "    else:\n",
    "        return lemma"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}