Reduce Phraser memory usage (drop frequencies)

gensim/models/phrases.py

@@ -208,6 +208,14 @@ def load(cls, *args, **kwargs):
         """
         model = super(PhrasesTransformation, cls).load(*args, **kwargs)
         # update older models
+        # if value in phrasegrams dict is a tuple, load only the scores.
+        try:
+            for components, scores in model.__dict__['phrasegrams'].items():
+                if isinstance(scores, tuple):
+                    model.__dict__['phrasegrams'][components] = scores[1]
+        except KeyError:
+            pass
+
         # if no scoring parameter, use default scoring
         if not hasattr(model, 'scoring'):
             logger.info('older version of %s loaded without scoring function', cls.__name__)
@@ -805,7 +813,7 @@ def __init__(self, phrases_model):
         for bigram, score in phrases_model.export_phrases(corpus, self.delimiter, as_tuples=True):
             if bigram in self.phrasegrams:
                 logger.info('Phraser repeat %s', bigram)
-            self.phrasegrams[bigram] = (phrases_model.vocab[self.delimiter.join(bigram)], score)
+            self.phrasegrams[bigram] = score
             count += 1
             if not count % 50000:
                 logger.info('Phraser added %i phrasegrams', count)
@@ -848,7 +856,7 @@ def score_item(self, worda, wordb, components, scorer):
 
         """
         try:
-            return self.phrasegrams[tuple(components)][1]
+            return self.phrasegrams[tuple(components)]
         except KeyError:
             return -1
 

gensim/test/test_phrases.py

@@ -12,7 +12,6 @@
 import unittest
 
 import six
-
 import numpy as np
 
 from gensim.utils import to_unicode
@@ -646,6 +645,16 @@ def testEncoding(self):
         self.assertTrue(isinstance(transformed, six.text_type))
 
 
+class TestPhraserModelCompatibilty(unittest.TestCase):
+
+    def testCompatibilty(self):
+        bigram_loaded = Phraser.load(datapath("phraser_model_3dot6"))
+        test_sentences = [u'trees', u'graph', u'minors']
+        prev_ver = bigram_loaded[test_sentences]
+        expected_res = ['trees_graph', 'minors']
+        self.assertEqual(prev_ver, expected_res)
+
+
 if __name__ == '__main__':
     logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)
     unittest.main()