GH-873: sync tokenization bug fix for RoBERTa with upstream (see face…

…bookresearch/fairseq#929)
flairNLP · Aug 1, 2019 · b08d6d1 · b08d6d1
1 parent cd814f1
commit b08d6d1
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/flair/embeddings.py b/flair/embeddings.py
@@ -1329,7 +1329,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:
                         layers=self.layers,
                         pooling_operation=self.pooling_operation,
                         subword_start_idx=1,
-                        subword_end_idx=-3,
+                        subword_end_idx=-2,
                     )
 
                     final_subtoken_embedding = torch.cat(subtoken_embeddings)