Comment Clustering v4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Imports\n",
    "from pickle import dump, load\n",
    "import nltk\n",
    "from nltk import word_tokenize,FreqDist\n",
    "import re\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.util import ngrams\n",
    "from sklearn.cluster import KMeans\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import paired_distances\n",
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#cluster count\n",
    "def get_document_text(raw_text):\n",
    "    \"\"\" This function takes in raw document text as input which we receive from the API and returns a clean text \n",
    "    of the associated document. It cleans up any HTML code in the text, newline characters, and extracts supplemental\n",
    "    information part of the document.\n",
    "    \n",
    "    INPUT: string\n",
    "    OUTPUT: string\n",
    "    \"\"\"\n",
    "    raw_text = raw_text.replace('\\n',' ')\n",
    "    raw_text = raw_text.replace('*','') # added\n",
    "    raw_text = raw_text.replace('\\r',' ') # added\n",
    "    # Remove any residual HTML tags in text\n",
    "    cleanr = re.compile('<.*?>')\n",
    "    cleantext = re.sub(cleanr, '', raw_text)\n",
    "    return cleantext\n",
    "\n",
    "def tokenize_text(corpus):\n",
    "    pattern = r'''(?x)     # set flag to allow verbose regexps\n",
    "    ((?:[A-Z]\\.)+)         # abbreviations, e.g. B.C.\n",
    "    | (?:(\\w+([-']\\w+))+)  # words with optional internal hyphens e.g. after-ages or author's\n",
    "    | ([a-zA-Z]+)          # capture everything else\n",
    "    '''\n",
    "    tokens = nltk.regexp_tokenize(corpus,pattern)\n",
    "    all_token = [word.lower() for token in tokens for word in token if word != \"\" and word[0] != \"'\" and word[0] != \"-\"]\n",
    "    return all_token\n",
    "\n",
    "def tokenize_text_sent(corpus):\n",
    "    sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "    raw_sents = sent_tokenizer.tokenize(corpus) # Split text into sentences    \n",
    "    return [tokenize_text(sent) for sent in raw_sents]\n",
    "\n",
    "def tag_my_text(sents):\n",
    "    return [nltk.pos_tag(sent) for sent in sents]\n",
    "\n",
    "#Chunk noun phrases in tree \n",
    "def noun_phrase_chunker():\n",
    "    grammar = r\"\"\"\n",
    "    NP: {<DT|PP\\$>?<JJ>*<NN>}   # chunk determiner/possessive, adjectives and noun\n",
    "    \"\"\"\n",
    "    cp = nltk.RegexpParser(grammar)\n",
    "    return cp\n",
    "\n",
    "#Extract only the NP marked phrases from the parse tree, that is the chunk we defined\n",
    "def noun_phrase_extractor(sentences, chunker):\n",
    "    res = []\n",
    "    for sent in sentences:\n",
    "        tree = chunker.parse(sent)\n",
    "        for subtree in tree.subtrees():\n",
    "            if subtree.label() == 'NP' : \n",
    "                res.append(subtree[0:len(subtree)])\n",
    "                #res.append(subtree[0])\n",
    "                #print(subtree)\n",
    "    return res\n",
    "\n",
    "#remove tags and get only the noun phrases , can be adjusted for length\n",
    "def noun_phrase_finder(tagged_text):\n",
    "    all_proper_noun = noun_phrase_extractor(tagged_text,noun_phrase_chunker()) \n",
    "    #does not literally mean proper noun. Chunker only extracts common noun\n",
    "    noun_phrase_list = []                                                      \n",
    "    #noun_phrase_string_list =[]\n",
    "    for noun_phrase in all_proper_noun:\n",
    "        if len(noun_phrase) > 0: #this means where the size of the phrase is greater than 1\n",
    "            small_list =[]\n",
    "            for (word,tag) in noun_phrase:\n",
    "                small_list.append(word)\n",
    "            noun_phrase_list.append(small_list)\n",
    "            #noun_phrase_string_list.append(' '.join(small_list))\n",
    "    return noun_phrase_list\n",
    "\n",
    "#get freq dist obj for noun phrase of different lengths\n",
    "def find_freq(nested_list,nest_len):\n",
    "    #from nltk.probability import FreqDist\n",
    "    fdist_list =[]\n",
    "    for inner_np in nested_list:\n",
    "        if len(inner_np) == nest_len:\n",
    "            fdist_list.append(' '.join(inner_np))\n",
    "    fdist = FreqDist(fdist_list)\n",
    "    return fdist\n",
    "\n",
    "def get_top_unigrams(np):\n",
    "    unigrams = []\n",
    "    for item in np:\n",
    "        if len(item) ==  1:\n",
    "            unigrams.append(item)\n",
    "    fdist_uni = find_freq(np,1)\n",
    "    uni_list = fdist_uni.most_common()\n",
    "    threshold = 0.3 * len(unigrams)\n",
    "    top = []\n",
    "    s = 0\n",
    "    for word,count in uni_list:\n",
    "        top.append(word)\n",
    "        s += count\n",
    "        if s > threshold:\n",
    "            break      \n",
    "    return top\n",
    "\n",
    "# Lesk algorith for disambiguation in case of multiple synsets of a word\n",
    "def compare_overlaps_greedy(context, synsets_signatures, pos=None):\n",
    "    \"\"\"\n",
    "    Calculate overlaps between the context sentence and the synset_signature\n",
    "    and returns the synset with the highest overlap.\n",
    "    \n",
    "    :param context: ``context_sentence`` The context sentence where the ambiguous word occurs.\n",
    "    :param synsets_signatures: ``dictionary`` A list of words that 'signifies' the ambiguous word.\n",
    "    :param pos: ``pos`` A specified Part-of-Speech (POS).\n",
    "    :return: ``lesk_sense`` The Synset() object with the highest signature overlaps.\n",
    "    \"\"\"\n",
    "    # if this returns none that means that there is no overlap\n",
    "    max_overlaps = 0\n",
    "    lesk_sense = None\n",
    "    for ss in synsets_signatures:\n",
    "        if pos and str(ss.pos()) != pos: # Skips different POS.\n",
    "            continue\n",
    "        overlaps = set(synsets_signatures[ss]).intersection(context)\n",
    "        if len(overlaps) > max_overlaps:\n",
    "            lesk_sense = ss\n",
    "            max_overlaps = len(overlaps)  \n",
    "    return lesk_sense\n",
    "\n",
    "def lesk(context_sentence, ambiguous_word, pos=None, dictionary=None):\n",
    "    \"\"\"\n",
    "    This function is the implementation of the original Lesk algorithm (1986).\n",
    "    It requires a dictionary which contains the definition of the different\n",
    "    sense of each word. See http://goo.gl/8TB15w\n",
    "\n",
    "        >>> from nltk import word_tokenize\n",
    "        >>> sent = word_tokenize(\"I went to the bank to deposit money.\")\n",
    "        >>> word = \"bank\"\n",
    "        >>> pos = \"n\"\n",
    "        >>> lesk(sent, word, pos)\n",
    "        Synset('bank.n.07')\n",
    "    \n",
    "    :param context_sentence: The context sentence where the ambiguous word occurs.\n",
    "    :param ambiguous_word: The ambiguous word that requires WSD.\n",
    "    :param pos: A specified Part-of-Speech (POS).\n",
    "    :param dictionary: A list of words that 'signifies' the ambiguous word.\n",
    "    :return: ``lesk_sense`` The Synset() object with the highest signature overlaps.\n",
    "    \"\"\"\n",
    "    if not dictionary:\n",
    "        dictionary = {}\n",
    "        for ss in wn.synsets(ambiguous_word):\n",
    "            dictionary[ss] = ss.definition().split()\n",
    "    best_sense = compare_overlaps_greedy(context_sentence, dictionary, pos)\n",
    "    return best_sense\n",
    "    #return dictionary \n",
    "\n",
    "# this function takes in a word and gets the most relevant synset based on context from the text. \n",
    "# for exact algorith refer the text above (\"what I want to do\" markdown)\n",
    "def get_synset(word,pos_tag_text ,pos):\n",
    "    if len(wn.synsets(word)) == 1:\n",
    "        #print(\"here1\")\n",
    "        return wn.synsets(word)[0]\n",
    "    else:\n",
    "        #get all context sentences\n",
    "        all_sent =[]\n",
    "        for sent in pos_tag_text:\n",
    "            for (w,t) in sent:\n",
    "                if w == word:\n",
    "                    all_sent.append(sent)\n",
    "        #call lesk here\n",
    "        app_syn = lesk(all_sent[len(all_sent)//2], word, pos)\n",
    "        if app_syn != None:\n",
    "            #print(\"here2\")\n",
    "            return app_syn\n",
    "        else:\n",
    "            #second lesk trial with another context sentence\n",
    "            app_syn = lesk(all_sent[len(all_sent)//3], word, pos)\n",
    "            if app_syn != None:\n",
    "                #print(\"here2\")\n",
    "                return app_syn\n",
    "            else:\n",
    "                #give up and choose 1st synset from list with matching pos\n",
    "                #print(\"here3\")\n",
    "                all_syns = wn.synsets(word)\n",
    "                for syn in all_syns:\n",
    "                    #print(syn.pos())\n",
    "                    if syn.pos() == pos:\n",
    "                        return syn\n",
    "    return False\n",
    "\n",
    "# this functions take all the single and double legth phrases form grand_list and gets sysnset for all them. (1 each)\n",
    "def get_singles_synset(uni_list,pos_tag_text):\n",
    "    single_synset =[]\n",
    "    #get synsets of all singletons\n",
    "    for singles in uni_list:\n",
    "        singles_syn = get_synset(singles,pos_tag_text, 'n')\n",
    "        if singles_syn:\n",
    "            single_synset.append(singles_syn)    \n",
    "    return single_synset\n",
    "\n",
    "#get common parents\n",
    "def get_lcs(uni_list,pos_tag_text):\n",
    "    #get all relevant sysnsets\n",
    "    all_synsets = get_singles_synset(uni_list,pos_tag_text)\n",
    "    list_of_all_lcs =[]\n",
    "    for syn in all_synsets:\n",
    "        for syn2 in all_synsets[all_synsets.index(syn)+1:]:\n",
    "            lcs = syn.lowest_common_hypernyms(syn2)\n",
    "            if len(lcs)> 0:\n",
    "                if lcs[0] not in list_of_all_lcs:\n",
    "                    list_of_all_lcs.append(lcs[0])\n",
    "    return list_of_all_lcs\n",
    "\n",
    "# get themes\n",
    "def get_theme(uni_list,pos_tag_text):\n",
    "    # get common parent\n",
    "    parent_sysnset = get_lcs(uni_list,pos_tag_text)\n",
    "    # filter out absolute top level and get lemma_names\n",
    "    lemma_names =[]\n",
    "    for synset in parent_sysnset:\n",
    "        if synset.min_depth() != 0:\n",
    "            #print(synset)\n",
    "            for each_name in synset.lemma_names():\n",
    "                if each_name not in lemma_names:\n",
    "                    lemma_names.append(each_name)\n",
    "                break\n",
    "    return lemma_names\n",
    "\n",
    "def get_cluster_count(document):\n",
    "    text = str(document['text'][0])\n",
    "    cleantext = get_document_text(text)\n",
    "    tagged_tokens = tag_my_text(tokenize_text_sent(cleantext))\n",
    "    np_list = noun_phrase_finder(tagged_tokens)\n",
    "    top_np = get_top_unigrams(np_list)\n",
    "    themes = get_theme(top_np,tagged_tokens)\n",
    "    return len(themes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Comment clustering\n",
    "def process_document(document):\n",
    "    comments = []\n",
    "    for c in document['comment_list']:\n",
    "        c = c.replace('\\n',' ')\n",
    "        if 'attached' in c and len(c) < 500:\n",
    "            pass\n",
    "        elif len(c) <200:\n",
    "            pass\n",
    "        else:\n",
    "            comments.append(str(c))\n",
    "    return comments\n",
    "\n",
    "# Modified from Brandon Rose:\n",
    "def tokenize_text_cluster(text):\n",
    "    tokens = [word.lower() for sent in nltk.sent_tokenize(text) for word in tokenize_text(sent)]\n",
    "    filtered_tokens = []\n",
    "    for token in tokens:\n",
    "        if len(token) > 2:\n",
    "            filtered_tokens.append(token)\n",
    "    return filtered_tokens\n",
    "\n",
    "def vectorize_comments(comments):\n",
    "    tfidf_vec = TfidfVectorizer(tokenizer=tokenize_text_cluster,\n",
    "                                stop_words='english',\n",
    "                                ngram_range=(2,3),\n",
    "                                min_df=0.18, max_df=0.9,\n",
    "                                max_features=200000)\n",
    "    tfidf_matrix = tfidf_vec.fit_transform(comments)\n",
    "    return tfidf_matrix, tfidf_vec\n",
    "\n",
    "def mash_comments(all_comments):\n",
    "    big_comment = []\n",
    "    for cluster in all_comments:\n",
    "        mashed = \"\"\n",
    "        for comment in cluster:\n",
    "            mashed += comment\n",
    "            mashed += \" \"\n",
    "        big_comment.append(mashed)\n",
    "    return big_comment\n",
    "\n",
    "# Adapted from: http://www.markhneedham.com/blog/2015/02/15/pythonscikit-learn-calculating-tfidf-on-how-i-met-your-mother-transcripts/\n",
    "def top_words(all_comments, n_top_words):\n",
    "    mashed_comments = mash_comments(all_comments)\n",
    "    tfidf_matrix, tfidf_vec = vectorize_comments(mashed_comments)\n",
    "    feature_names = tfidf_vec.get_feature_names()\n",
    "    dense = tfidf_matrix.todense()\n",
    "    top_words = []\n",
    "    for i in range(0,len(mashed_comments)):\n",
    "        cluster = dense[i].tolist()[0]\n",
    "        word_scores = [pair for pair in zip(range(0, len(cluster)), cluster) if pair[1] > 0]\n",
    "        sorted_word_scores = sorted(word_scores, key=lambda t: t[1] * -1)\n",
    "        temp_top_words = []\n",
    "        for word, score in [(feature_names[word_id], score) for (word_id, score) in sorted_word_scores][:n_top_words]:\n",
    "            temp_top_words.append(word)\n",
    "        top_words.append(temp_top_words)\n",
    "    return top_words\n",
    "\n",
    "def cluster_comments(document, num_clusters):\n",
    "    cluster_dict = {}\n",
    "\n",
    "    comments = process_document(document)\n",
    "    tfidf_matrix, tfidf_vec = vectorize_comments(comments)\n",
    "    \n",
    "    km = KMeans(n_clusters=num_clusters)\n",
    "    km.fit(tfidf_matrix)\n",
    "    clusters = km.labels_.tolist()\n",
    "    \n",
    "    cluster_center_list = []\n",
    "    for c in clusters:\n",
    "        cluster_center_list.append(km.cluster_centers_[c])\n",
    "    center_distances = paired_distances(tfidf_matrix, cluster_center_list)\n",
    "    \n",
    "    comment_clusters = {'comment': comments, 'cluster': clusters, 'dist': center_distances}\n",
    "    comment_frame = pd.DataFrame(comment_clusters, index = [clusters] , columns = ['comment', 'cluster', 'dist'])\n",
    "    \n",
    "    central_comments = []\n",
    "    all_comments = []\n",
    "    for i in range(num_clusters):\n",
    "        central_comments.append(comment_frame[comment_frame.cluster==i].min().comment)\n",
    "        all_comments.append(list(comment_frame[comment_frame.cluster==i]['comment']))\n",
    "    \n",
    "    tfidf_words = top_words(all_comments, 6)\n",
    "    \n",
    "    cluster_dict['central_comments'] = central_comments\n",
    "    cluster_dict['all_comments'] = all_comments\n",
    "    cluster_dict['top_words'] = tfidf_words\n",
    "    \n",
    "    return cluster_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write to Json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "doc_list1 = load(open(\"data/Master_doc_content\",'rb'))\n",
    "doc_list2 = load(open(\"data/Master2_doc_content\",'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "doc_id1 = [\"FAA-2010-1127-0001\",\"USCBP-2007-0064-1986\",\"FMCSA-2015-0419-0001\",\"NARA-06-0007-0001\",\"APHIS-2006-0041-0001\",\"EBSA-2012-0031-0001\",\"IRS-2010-0009-0001\",\"BOR-2008-0004-0001\",\"OSHA-2013-0023-1443\",\"DOL-2016-0001-0001\",\"NRC-2015-0057-0086\",\"CMS-2010-0259-0001\",\"CMS-2009-0008-0003\",\"CMS-2009-0038-0002\",\"NPS-2014-0005-000\",\"BIS-2015-0011-0001\",\"HUD-2011-0056-0019\",\"HUD-2011-0014-0001\",\"OCC-2011-0002-0001\",\"ACF-2015-0008-0124\",\"ETA-2008-0003-0001\",\"CMS-2012-0152-0004\",\"CFPB-2013-0033-0001\",\"USCIS-2016-0001-0001\",\"FMCSA-2011-0146-0001\",\"USCG-2013-0915-0001\",\"NHTSA-2012-0177-0001\",\"USCBP-2005-0005-0001\"]\n",
    "doc_id2 = [\"HUD-2015-0101-0001\",\"ACF-2010-0003-0001\",\"NPS-2015-0008-0001\",\"FAR-2014-0025-0026\",\"CFPB-2013-0002-0001\",\"DOS-2010-0035-0001\",\"USCG-2013-0915-0001\",\"SBA-2010-0001-0001\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "doc_title1 = [\"Photo Requirements for Pilot Certificates\",\n",
    "             \"Advance Information on Private Aircraft Arriving and Departing the United States\",\n",
    "             \"Evaluation of Safety Sensitive Personnel for Moderate-to-Severe Obstructive Sleep Apnea\",\n",
    "             \"Changes in NARA Research Room and Museum Hours\",\n",
    "             \"Bovine Spongiform Encephalopathy; Minimal-Risk Regions; Importation of Live Bovines and Products Derived From Bovines\",\n",
    "             \"Incentives for Nondiscriminatory Wellness Programs in Group Health Plans\",\n",
    "             \"Furnishing Identifying Number of Tax Return Preparer\",\n",
    "             \"Use of Bureau of Reclamation Land, Facilities, and Waterbodies\",\n",
    "             \"Improve Tracking of Workplace Injuries and Illnesses\",\n",
    "             \"Implementation of the Nondiscrimination and Equal Opportunity Provisions of the Workforce Innovation and Opportunity Act\",\n",
    "             \"Linear No-Threshold Model and Standards for Protection Against Radiation; Extension of Comment Period\",\n",
    "             \"Medicare Program: Accountable Care Organizations and the Medicare Shared Saving Program\",\n",
    "             \"Medicare Program: Changes to the Competitive Acquisition of Certain Durable Medical Equipment, Prosthetics, Orthotics and Supplies (DMEPOS) by Certain Provisions of the Medicare Improvements for Patients and Providers Act of 2008 (MIPPA)\",\n",
    "             \"Medicare Program: Inpatient Rehabilitation Facility Prospective Payment System for Federal Fiscal Year 2010 \",\n",
    "             \"Special Regulations: Areas of the National Park System, Cuyahoga Valley National Park, Bicycling\",\n",
    "             \"Wassenaar Arrangement Plenary Agreements Implementation; Intrusion and Surveillance Items\",\n",
    "             \"Credit Risk Retention 2\",\n",
    "             \"FR 5359–P–01 Equal Access to Housing in HUD Programs Regardless of Sexual Orientation or Gender Identity \",\n",
    "             \"Credit Risk Retention\",\n",
    "             \"Head Start Performance Standards; Extension of Comment Period\",\n",
    "             \"Senior Community Service Employment Program\",\n",
    "             \"Patient Protection and Affordable Care Act: Benefit and Payment Parameters for 2014\",\n",
    "             \"Debt Collection (Regulation F)\",\n",
    "             \"U.S. Citizenship and Immigration Services Fee Schedule\",\n",
    "             \"Applicability of Regulations to Operators of Certain Farm Vehicles and Off-Road Agricultural Equipment\",\n",
    "             \"Carriage of Conditionally Permitted Shale Gas Extraction Waste Water in Bulk\",\n",
    "             \"Federal Motor Vehicle Safety Standards: Event Data Recorders\",\n",
    "             \"Documents Required for Travel Within the Western Hemisphere\"]\n",
    "doc_title2 = [\"FR 5597-P-02 Instituting Smoke- Free Public Housing\",\n",
    "             \"Head Start Program\",\n",
    "             \"Off-Road Vehicle Management: Cape Lookout National Seashore\",\n",
    "             \"Federal Acquisition Regulations: Fair Pay and Safe Workplaces; Second Extension of Time for Comments (FAR Case 2014-025)\",\n",
    "             \"Ability to Repay Standards under Truth in Lending Act (Regulation Z)\",\n",
    "             \"Schedule of Fees for Consular Services, Department of State and Overseas Embassies and Consulates\",\n",
    "             \"Carriage of Conditionally Permitted Shale Gas Extraction Waste Water in Bulk\",\n",
    "             \"Women-Owned Small Business Federal Contract Program\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "1\n",
      "2\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "3\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "4\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "5\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "6\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "7\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "8\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "9\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "10\n",
      "11\n",
      "12\n",
      "Warning: parsing empty text\n",
      "13\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "14\n",
      "Warning: parsing empty text\n",
      "15\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "16\n",
      "17\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "18\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "19\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "20\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "21\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "22\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "23\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "24\n",
      "25\n",
      "26\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "27\n",
      "Warning: parsing empty text\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(doc_list1)):\n",
    "    print(i)\n",
    "    document = doc_list1[i]\n",
    "    cluster_num = get_cluster_count(document)\n",
    "    if cluster_num < 2:\n",
    "        cluster_num = 2\n",
    "    clust_dict = cluster_comments(document, cluster_num)\n",
    "    clust_dict[\"doc_id\"], clust_dict[\"doc_title\"] = doc_id1[i], doc_title1[i]\n",
    "    data.append(clust_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "1\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "2\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "3\n",
      "4\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "5\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "6\n",
      "7\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(doc_list2)):\n",
    "    print(i)\n",
    "    document = doc_list2[i]\n",
    "    cluster_num = get_cluster_count(document)\n",
    "    if cluster_num < 2:\n",
    "        cluster_num = 2\n",
    "    clust_dict = cluster_comments(document, cluster_num)\n",
    "    clust_dict[\"doc_id\"], clust_dict[\"doc_title\"] = doc_id2[i], doc_title2[i]\n",
    "    data.append(clust_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# data[1]['doc_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_obj = {}\n",
    "top_obj[\"data\"] = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('data/comment_data.json', 'w') as outfile:\n",
    "    json.dump(top_obj, outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BEGIN TESTING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "c = process_document(doc_list1[26])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "TEST_cluster_num = get_cluster_count(doc_list1[26])\n",
    "TEST = cluster_comments(document, TEST_cluster_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for comment in c:\n",
    "    if len(tokenize_text_cluster(comment)) < 50:\n",
    "        print(len(tokenize_text_cluster(comment)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cluster_comments(document, num_clusters):\n",
    "    cluster_dict = {}\n",
    "\n",
    "    comments = process_document(document)\n",
    "    tfidf_matrix, tfidf_vec = vectorize_comments(comments)\n",
    "    \n",
    "    km = KMeans(n_clusters=num_clusters)\n",
    "    km.fit(tfidf_matrix)\n",
    "    clusters = km.labels_.tolist()\n",
    "    \n",
    "    cluster_center_list = []\n",
    "    for c in clusters:\n",
    "        cluster_center_list.append(km.cluster_centers_[c])\n",
    "    center_distances = paired_distances(tfidf_matrix, cluster_center_list)\n",
    "    \n",
    "    comment_clusters = {'comment': comments, 'cluster': clusters, 'dist': center_distances}\n",
    "    comment_frame = pd.DataFrame(comment_clusters, index = [clusters] , columns = ['comment', 'cluster', 'dist'])\n",
    "    \n",
    "    central_comments = []\n",
    "    all_comments = []\n",
    "    for i in range(num_clusters):\n",
    "        central_comments.append(comment_frame[comment_frame.cluster==i].min().comment)\n",
    "        all_comments.append(list(comment_frame[comment_frame.cluster==i]['comment']))\n",
    "    \n",
    "    tfidf_words = top_words(all_comments, 6)\n",
    "    \n",
    "    cluster_dict['central_comments'] = central_comments\n",
    "    cluster_dict['all_comments'] = all_comments\n",
    "    cluster_dict['top_words'] = tfidf_words\n",
    "    \n",
    "    return cluster_dict, comment_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cluster_num = get_cluster_count(doc_list1[26])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cluster_dict, comment_frame = cluster_comments(doc_list1[26], cluster_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "comment_frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "comment_frame[comment_frame.cluster==0].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "comment_frame[comment_frame.cluster==4].comment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "comment_frame[comment_frame.cluster==0].min().comment"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}