Language Models from Transformers Lib

rasa/nlu/constants.py

@@ -36,6 +36,10 @@
 }
 
 SPACY_DOCS = {TEXT_ATTRIBUTE: "spacy_doc", RESPONSE_ATTRIBUTE: "response_spacy_doc"}
+TRANSFORMERS_DOCS = {
+    TEXT_ATTRIBUTE: "text_transformers_doc",
+    RESPONSE_ATTRIBUTE: "response_transformers_doc",
+}
 
 DENSE_FEATURIZABLE_ATTRIBUTES = [TEXT_ATTRIBUTE, RESPONSE_ATTRIBUTE]
 

rasa/nlu/featurizers/dense_featurizer/lm_featurizer.py

@@ -0,0 +1,64 @@
+import numpy as np
+import typing
+from typing import Any, Optional, Text
+
+from rasa.nlu.config import RasaNLUModelConfig
+from rasa.nlu.featurizers.featurizer import Featurizer
+from rasa.nlu.training_data import Message, TrainingData
+
+if typing.TYPE_CHECKING:
+    from spacy.tokens import Doc
+
+from rasa.nlu.constants import (
+    TEXT_ATTRIBUTE,
+    TRANSFORMERS_DOCS,
+    DENSE_FEATURE_NAMES,
+    DENSE_FEATURIZABLE_ATTRIBUTES,
+    TOKENS_NAMES,
+)
+
+
+class LanguageModelFeaturizer(Featurizer):
+
+    provides = [
+        DENSE_FEATURE_NAMES[attribute] for attribute in DENSE_FEATURIZABLE_ATTRIBUTES
+    ]
+
+    requires = [
+        TRANSFORMERS_DOCS[attribute] for attribute in DENSE_FEATURIZABLE_ATTRIBUTES
+    ] + [TOKENS_NAMES[attribute] for attribute in DENSE_FEATURIZABLE_ATTRIBUTES]
+
+    def train(
+        self,
+        training_data: TrainingData,
+        config: Optional[RasaNLUModelConfig],
+        **kwargs: Any,
+    ) -> None:
+
+        for example in training_data.intent_examples:
+            for attribute in DENSE_FEATURIZABLE_ATTRIBUTES:
+                self._set_lm_features(example, attribute)
+
+    def get_doc(self, message: Message, attribute: Text) -> Any:
+
+        return message.get(TRANSFORMERS_DOCS[attribute])
+
+    def process(self, message: Message, **kwargs: Any) -> None:
+
+        self._set_lm_features(message)
+
+    def _set_lm_features(self, message: Message, attribute: Text = TEXT_ATTRIBUTE):
+        """Adds the precomputed word vectors to the messages features."""
+
+        message_attribute_doc = self.get_doc(message, attribute)
+
+        if message_attribute_doc is not None:
+            sequence_features = message_attribute_doc["sequence_features"]
+            sentence_features = message_attribute_doc["sentence_features"]
+
+            features = np.concatenate([sequence_features, sentence_features])
+
+            features = self._combine_with_existing_dense_features(
+                message, features, DENSE_FEATURE_NAMES[attribute]
+            )
+            message.set(DENSE_FEATURE_NAMES[attribute], features)

rasa/nlu/registry.py

@@ -28,6 +28,7 @@
 from rasa.nlu.featurizers.sparse_featurizer.count_vectors_featurizer import (
     CountVectorsFeaturizer,
 )
+from rasa.nlu.featurizers.dense_featurizer.lm_featurizer import LanguageModelFeaturizer
 from rasa.nlu.featurizers.sparse_featurizer.regex_featurizer import RegexFeaturizer
 from rasa.nlu.model import Metadata
 from rasa.nlu.selectors.embedding_response_selector import ResponseSelector
@@ -36,8 +37,10 @@
 from rasa.nlu.tokenizers.mitie_tokenizer import MitieTokenizer
 from rasa.nlu.tokenizers.spacy_tokenizer import SpacyTokenizer
 from rasa.nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer
+from rasa.nlu.tokenizers.lm_tokenizer import LanguageModelTokenizer
 from rasa.nlu.utils.mitie_utils import MitieNLP
 from rasa.nlu.utils.spacy_utils import SpacyNLP
+from rasa.nlu.utils.hugging_face.hf_transformers import HFTransformersNLP
 from rasa.utils.common import class_from_module_path, raise_warning
 
 if typing.TYPE_CHECKING:
@@ -53,12 +56,14 @@
     # utils
     SpacyNLP,
     MitieNLP,
+    HFTransformersNLP,
     # tokenizers
     MitieTokenizer,
     SpacyTokenizer,
     WhitespaceTokenizer,
     ConveRTTokenizer,
     JiebaTokenizer,
+    LanguageModelTokenizer,
     # extractors
     SpacyEntityExtractor,
     MitieEntityExtractor,
@@ -72,6 +77,7 @@
     LexicalSyntacticFeaturizer,
     CountVectorsFeaturizer,
     ConveRTFeaturizer,
+    LanguageModelFeaturizer,
     # classifiers
     SklearnIntentClassifier,
     MitieIntentClassifier,

rasa/nlu/tokenizers/lm_tokenizer.py

@@ -0,0 +1,35 @@
+import typing
+from typing import Text, List, Any, Dict
+
+from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
+from rasa.nlu.training_data import Message
+
+from rasa.nlu.constants import (
+    TOKENS_NAMES,
+    TRANSFORMERS_DOCS,
+    DENSE_FEATURIZABLE_ATTRIBUTES,
+)
+
+
+class LanguageModelTokenizer(Tokenizer):
+
+    provides = [TOKENS_NAMES[attribute] for attribute in DENSE_FEATURIZABLE_ATTRIBUTES]
+
+    requires = [
+        TRANSFORMERS_DOCS[attribute] for attribute in DENSE_FEATURIZABLE_ATTRIBUTES
+    ]
+
+    defaults = {
+        # Flag to check whether to split intents
+        "intent_tokenization_flag": False,
+        # Symbol on which intent should be split
+        "intent_split_symbol": "_",
+    }
+
+    def get_doc(self, message: Message, attribute: Text) -> Dict[Text, Any]:
+        return message.get(TRANSFORMERS_DOCS[attribute])
+
+    def tokenize(self, message: Message, attribute: Text) -> List[Token]:
+        doc = self.get_doc(message, attribute)
+
+        return doc["tokens"]