Update sklearn API for Gensim models

gensim/sklearn_api/__init__.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011 Radim Rehurek <[email protected]>
+# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html
+"""Scikit learn wrapper for gensim.
+Contains various gensim based implementations which match with scikit-learn standards.
+See [1] for complete set of conventions.
+[1] http://scikit-learn.org/stable/developers/
+"""
+
+
+from .basemodel import BaseTransformer  # noqa: F401
+from .ldamodel import LdaTransformer  # noqa: F401
+from .lsimodel import LsiTransformer  # noqa: F401
+from .rpmodel import RpTransformer  # noqa: F401
+from .ldaseqmodel import LdaSeqTransformer  # noqa: F401
+from .w2vmodel import W2VTransformer  # noqa: F401
+from .atmodel import AuthorTopicTransformer  # noqa: F401

gensim/sklearn_integration/sklearn_wrapper_gensim_atmodel.py → gensim/sklearn_api/atmodel.py

@@ -13,10 +13,10 @@
 from sklearn.exceptions import NotFittedError
 
 from gensim import models
-from gensim.sklearn_integration import BaseSklearnWrapper
+from gensim.sklearn_api import BaseTransformer
 
 
-class SklATModel(BaseSklearnWrapper, TransformerMixin, BaseEstimator):
+class AuthorTopicTransformer(BaseTransformer, TransformerMixin, BaseEstimator):
     """
     Base AuthorTopic module
     """
@@ -27,7 +27,7 @@ def __init__(self, num_topics=100, id2word=None, author2doc=None, doc2author=Non
             gamma_threshold=0.001, serialized=False, serialization_path=None,
             minimum_probability=0.01, random_state=None):
         """
-        Sklearn wrapper for AuthorTopic model. Class derived from gensim.models.AuthorTopicModel
+        Sklearn wrapper for AuthorTopic model. See gensim.models.AuthorTopicModel for parameter details.
         """
         self.gensim_model = None
         self.num_topics = num_topics
@@ -49,25 +49,6 @@ def __init__(self, num_topics=100, id2word=None, author2doc=None, doc2author=Non
         self.minimum_probability = minimum_probability
         self.random_state = random_state
 
-    def get_params(self, deep=True):
-        """
-        Returns all parameters as dictionary.
-        """
-        return {"num_topics": self.num_topics, "id2word": self.id2word,
-                "author2doc": self.author2doc, "doc2author": self.doc2author, "chunksize": self.chunksize,
-                "passes": self.passes, "iterations": self.iterations, "decay": self.decay,
-                "offset": self.offset, "alpha": self.alpha, "eta": self.eta, "update_every": self.update_every,
-                "eval_every": self.eval_every, "gamma_threshold": self.gamma_threshold,
-                "serialized": self.serialized, "serialization_path": self.serialization_path,
-                "minimum_probability": self.minimum_probability, "random_state": self.random_state}
-
-    def set_params(self, **parameters):
-        """
-        Set all parameters.
-        """
-        super(SklATModel, self).set_params(**parameters)
-        return self
-
     def fit(self, X, y=None):
         """
         Fit the model according to the given training data.

gensim/sklearn_integration/base_sklearn_wrapper.py → gensim/sklearn_api/basemodel.py

@@ -11,33 +11,16 @@
 from abc import ABCMeta, abstractmethod
 
 
-class BaseSklearnWrapper(object):
+class BaseTransformer(object):
     """
     Base sklearn wrapper module
     """
     __metaclass__ = ABCMeta
 
-    @abstractmethod
-    def get_params(self, deep=True):
-        pass
-
-    @abstractmethod
-    def set_params(self, **parameters):
-        """
-        Set all parameters.
-        """
-        for parameter, value in parameters.items():
-            setattr(self, parameter, value)
-        return self
-
     @abstractmethod
     def fit(self, X, y=None):
         pass
 
     @abstractmethod
     def transform(self, docs, minimum_probability=None):
         pass
-
-    @abstractmethod
-    def partial_fit(self, X):
-        pass

gensim/sklearn_integration/sklearn_wrapper_gensim_ldamodel.py → gensim/sklearn_api/ldamodel.py

@@ -16,10 +16,10 @@
 
 from gensim import models
 from gensim import matutils
-from gensim.sklearn_integration import BaseSklearnWrapper
+from gensim.sklearn_api import BaseTransformer
 
 
-class SklLdaModel(BaseSklearnWrapper, TransformerMixin, BaseEstimator):
+class LdaTransformer(BaseTransformer, TransformerMixin, BaseEstimator):
     """
     Base LDA module
     """
@@ -31,7 +31,7 @@ def __init__(
             eval_every=10, iterations=50, gamma_threshold=0.001,
             minimum_probability=0.01, random_state=None):
         """
-        Sklearn wrapper for LDA model. derived class for gensim.model.LdaModel .
+        Sklearn wrapper for LDA model. See gensim.model.LdaModel for parameter details.
         """
         self.gensim_model = None
         self.num_topics = num_topics
@@ -49,23 +49,6 @@ def __init__(
         self.minimum_probability = minimum_probability
         self.random_state = random_state
 
-    def get_params(self, deep=True):
-        """
-        Returns all parameters as dictionary.
-        """
-        return {"num_topics": self.num_topics, "id2word": self.id2word, "chunksize": self.chunksize,
-                "passes": self.passes, "update_every": self.update_every, "alpha": self.alpha, "eta": self.eta,
-                "decay": self.decay, "offset": self.offset, "eval_every": self.eval_every, "iterations": self.iterations,
-                "gamma_threshold": self.gamma_threshold, "minimum_probability": self.minimum_probability,
-                "random_state": self.random_state}
-
-    def set_params(self, **parameters):
-        """
-        Set all parameters.
-        """
-        super(SklLdaModel, self).set_params(**parameters)
-        return self
-
     def fit(self, X, y=None):
         """
         Fit the model according to the given training data.
@@ -86,8 +69,8 @@ def fit(self, X, y=None):
 
     def transform(self, docs):
         """
-        Takes as an list of input a documents (documents).
-        Returns matrix of topic distribution for the given document bow, where a_ij
+        Takes a list of documents as input ('docs').
+        Returns a matrix of topic distribution for the given document bow, where a_ij
         indicates (topic_i, topic_probability_j).
         The input `docs` should be in BOW format and can be a list of documents like : [ [(4, 1), (7, 1)], [(9, 1), (13, 1)], [(2, 1), (6, 1)] ]
         or a single document like : [(4, 1), (7, 1)]
@@ -105,7 +88,7 @@ def transform(self, docs):
             probs_docs = list(map(lambda x: x[1], doc_topics))
             # Everything should be equal in length
             if len(probs_docs) != self.num_topics:
-                probs_docs.extend([1e-12]*(self.num_topics - len(probs_docs)))
+                probs_docs.extend([1e-12] * (self.num_topics - len(probs_docs)))
             X[k] = probs_docs
         return np.reshape(np.array(X), (len(docs), self.num_topics))
 

gensim/sklearn_integration/sklearn_wrapper_gensim_ldaseqmodel.py → gensim/sklearn_api/ldaseqmodel.py

@@ -14,10 +14,10 @@
 from sklearn.exceptions import NotFittedError
 
 from gensim import models
-from gensim.sklearn_integration import BaseSklearnWrapper
+from gensim.sklearn_api import BaseTransformer
 
 
-class SklLdaSeqModel(BaseSklearnWrapper, TransformerMixin, BaseEstimator):
+class LdaSeqTransformer(BaseTransformer, TransformerMixin, BaseEstimator):
     """
     Base LdaSeq module
     """
@@ -26,7 +26,7 @@ def __init__(self, time_slice=None, id2word=None, alphas=0.01, num_topics=10,
                 initialize='gensim', sstats=None, lda_model=None, obs_variance=0.5, chain_variance=0.005, passes=10,
                 random_state=None, lda_inference_max_iter=25, em_min_iter=6, em_max_iter=20, chunksize=100):
         """
-        Sklearn wrapper for LdaSeq model. Class derived from gensim.models.LdaSeqModel
+        Sklearn wrapper for LdaSeq model. See gensim.models.LdaSeqModel for parameter details.
         """
         self.gensim_model = None
         self.time_slice = time_slice
@@ -45,24 +45,6 @@ def __init__(self, time_slice=None, id2word=None, alphas=0.01, num_topics=10,
         self.em_max_iter = em_max_iter
         self.chunksize = chunksize
 
-    def get_params(self, deep=True):
-        """
-        Returns all parameters as dictionary.
-        """
-        return {"time_slice": self.time_slice, "id2word": self.id2word,
-                "alphas": self.alphas, "num_topics": self.num_topics, "initialize": self.initialize,
-                "sstats": self.sstats, "lda_model": self.lda_model, "obs_variance": self.obs_variance,
-                "chain_variance": self.chain_variance, "passes": self.passes, "random_state": self.random_state,
-                "lda_inference_max_iter": self.lda_inference_max_iter, "em_min_iter": self.em_min_iter,
-                "em_max_iter": self.em_max_iter, "chunksize": self.chunksize}
-
-    def set_params(self, **parameters):
-        """
-        Set all parameters.
-        """
-        super(SklLdaSeqModel, self).set_params(**parameters)
-        return self
-
     def fit(self, X, y=None):
         """
         Fit the model according to the given training data.
@@ -97,6 +79,3 @@ def transform(self, docs):
             X[k] = transformed_author
 
         return np.reshape(np.array(X), (len(docs), self.num_topics))
-
-    def partial_fit(self, X):
-        raise NotImplementedError("'partial_fit' has not been implemented for SklLdaSeqModel")

gensim/sklearn_integration/sklearn_wrapper_gensim_lsimodel.py → gensim/sklearn_api/lsimodel.py

@@ -16,18 +16,18 @@
 
 from gensim import models
 from gensim import matutils
-from gensim.sklearn_integration import BaseSklearnWrapper
+from gensim.sklearn_api import BaseTransformer
 
 
-class SklLsiModel(BaseSklearnWrapper, TransformerMixin, BaseEstimator):
+class LsiTransformer(BaseTransformer, TransformerMixin, BaseEstimator):
     """
     Base LSI module
     """
 
     def __init__(self, num_topics=200, id2word=None, chunksize=20000,
                  decay=1.0, onepass=True, power_iters=2, extra_samples=100):
         """
-        Sklearn wrapper for LSI model. Class derived from gensim.model.LsiModel.
+        Sklearn wrapper for LSI model. See gensim.model.LsiModel for parameter details.
         """
         self.gensim_model = None
         self.num_topics = num_topics
@@ -38,21 +38,6 @@ def __init__(self, num_topics=200, id2word=None, chunksize=20000,
         self.extra_samples = extra_samples
         self.power_iters = power_iters
 
-    def get_params(self, deep=True):
-        """
-        Returns all parameters as dictionary.
-        """
-        return {"num_topics": self.num_topics, "id2word": self.id2word,
-                "chunksize": self.chunksize, "decay": self.decay, "onepass": self.onepass,
-                "extra_samples": self.extra_samples, "power_iters": self.power_iters}
-
-    def set_params(self, **parameters):
-        """
-        Set all parameters.
-        """
-        super(SklLsiModel, self).set_params(**parameters)
-        return self
-
     def fit(self, X, y=None):
         """
         Fit the model according to the given training data.
@@ -81,13 +66,13 @@ def transform(self, docs):
         # The input as array of array
         check = lambda x: [x] if isinstance(x[0], tuple) else x
         docs = check(docs)
-        X = [[] for i in range(0,len(docs))];
-        for k,v in enumerate(docs):
+        X = [[] for i in range(0, len(docs))]
+        for k, v in enumerate(docs):
             doc_topics = self.gensim_model[v]
             probs_docs = list(map(lambda x: x[1], doc_topics))
             # Everything should be equal in length
             if len(probs_docs) != self.num_topics:
-                probs_docs.extend([1e-12]*(self.num_topics - len(probs_docs)))
+                probs_docs.extend([1e-12] * (self.num_topics - len(probs_docs)))
             X[k] = probs_docs
             probs_docs = []
         return np.reshape(np.array(X), (len(docs), self.num_topics))

gensim/sklearn_integration/sklearn_wrapper_gensim_rpmodel.py → gensim/sklearn_api/rpmodel.py

@@ -14,35 +14,22 @@
 from sklearn.exceptions import NotFittedError
 
 from gensim import models
-from gensim.sklearn_integration import BaseSklearnWrapper
+from gensim.sklearn_api import BaseTransformer
 
 
-class SklRpModel(BaseSklearnWrapper, TransformerMixin, BaseEstimator):
+class RpTransformer(BaseTransformer, TransformerMixin, BaseEstimator):
     """
     Base RP module
     """
 
     def __init__(self, id2word=None, num_topics=300):
         """
-        Sklearn wrapper for RP model. Class derived from gensim.models.RpModel.
+        Sklearn wrapper for RP model. See gensim.models.RpModel for parameter details.
         """
         self.gensim_model = None
         self.id2word = id2word
         self.num_topics = num_topics
 
-    def get_params(self, deep=True):
-        """
-        Returns all parameters as dictionary.
-        """
-        return {"id2word": self.id2word, "num_topics": self.num_topics}
-
-    def set_params(self, **parameters):
-        """
-        Set all parameters.
-        """
-        super(SklRpModel, self).set_params(**parameters)
-        return self
-
     def fit(self, X, y=None):
         """
         Fit the model according to the given training data.
@@ -75,6 +62,3 @@ def transform(self, docs):
             X[k] = probs_docs
 
         return np.reshape(np.array(X), (len(docs), self.num_topics))
-
-    def partial_fit(self, X):
-        raise NotImplementedError("'partial_fit' has not been implemented for SklRpModel")

gensim/sklearn_integration/sklearn_wrapper_gensim_w2vmodel.py → gensim/sklearn_api/w2vmodel.py

@@ -15,10 +15,10 @@
 from sklearn.exceptions import NotFittedError
 
 from gensim import models
-from gensim.sklearn_integration import BaseSklearnWrapper
+from gensim.sklearn_api import BaseTransformer
 
 
-class SklW2VModel(BaseSklearnWrapper, TransformerMixin, BaseEstimator):
+class W2VTransformer(BaseTransformer, TransformerMixin, BaseEstimator):
     """
     Base Word2Vec module
     """
@@ -28,7 +28,7 @@ def __init__(self, size=100, alpha=0.025, window=5, min_count=5,
             sg=0, hs=0, negative=5, cbow_mean=1, hashfxn=hash, iter=5, null_word=0,
             trim_rule=None, sorted_vocab=1, batch_words=10000):
         """
-        Sklearn wrapper for Word2Vec model. Class derived from gensim.models.Word2Vec
+        Sklearn wrapper for Word2Vec model. See gensim.models.Word2Vec for parameter details.
         """
         self.gensim_model = None
         self.size = size
@@ -51,24 +51,6 @@ def __init__(self, size=100, alpha=0.025, window=5, min_count=5,
         self.sorted_vocab = sorted_vocab
         self.batch_words = batch_words
 
-    def get_params(self, deep=True):
-        """
-        Returns all parameters as dictionary.
-        """
-        return {"size": self.size, "alpha": self.alpha, "window": self.window, "min_count": self.min_count,
-        "max_vocab_size": self.max_vocab_size, "sample": self.sample, "seed": self.seed,
-        "workers": self.workers, "min_alpha": self.min_alpha, "sg": self.sg, "hs": self.hs,
-        "negative": self.negative, "cbow_mean": self.cbow_mean, "hashfxn": self.hashfxn,
-        "iter": self.iter, "null_word": self.null_word, "trim_rule": self.trim_rule,
-        "sorted_vocab": self.sorted_vocab, "batch_words": self.batch_words}
-
-    def set_params(self, **parameters):
-        """
-        Set all parameters.
-        """
-        super(SklW2VModel, self).set_params(**parameters)
-        return self
-
     def fit(self, X, y=None):
         """
         Fit the model according to the given training data.
@@ -101,4 +83,5 @@ def transform(self, words):
         return np.reshape(np.array(X), (len(words), self.size))
 
     def partial_fit(self, X):
-        raise NotImplementedError("'partial_fit' has not been implemented for SklW2VModel")
+        raise NotImplementedError("'partial_fit' has not been implemented for W2VTransformer since 'update()' function for Word2Vec class is experimental. "
+            "See usage and documentation of Word2Vec's 'update()' function if you need to train your Word2Vec model incrementally.")