Update transformers requirement from <3.6,>=3.4 to >=3.4,<4.1

CHANGELOG.md

@@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Renamed module `allennlp.data.tokenizers.token` to `allennlp.data.tokenizers.token_class` to avoid
   [this bug](https://github.com/allenai/allennlp/issues/4819).
+- `transformers` dependency updated to version 4.0.1.
 
 ### Fixed
 

allennlp/nn/util.py

@@ -1767,10 +1767,10 @@ def find_embedding_layer(model: torch.nn.Module) -> torch.nn.Module:
     """
     # We'll look for a few special cases in a first pass, then fall back to just finding a
     # TextFieldEmbedder in a second pass if we didn't find a special case.
-    from transformers.modeling_gpt2 import GPT2Model
-    from transformers.modeling_bert import BertEmbeddings
-    from transformers.modeling_albert import AlbertEmbeddings
-    from transformers.modeling_roberta import RobertaEmbeddings
+    from transformers.models.gpt2.modeling_gpt2 import GPT2Model
+    from transformers.models.bert.modeling_bert import BertEmbeddings
+    from transformers.models.albert.modeling_albert import AlbertEmbeddings
+    from transformers.models.roberta.modeling_roberta import RobertaEmbeddings
     from allennlp.modules.text_field_embedders.text_field_embedder import TextFieldEmbedder
     from allennlp.modules.text_field_embedders.basic_text_field_embedder import (
         BasicTextFieldEmbedder,

setup.py

@@ -64,7 +64,8 @@
         "scikit-learn",
         "scipy",
         "pytest",
-        "transformers>=3.4,<3.6",
+        "transformers>=4.0,<4.1",
+        "sentencepiece",
         "jsonpickle",
         "dataclasses;python_version<'3.7'",
         "filelock>=3.0,<3.1",

tests/data/token_indexers/pretrained_transformer_indexer_test.py

@@ -99,7 +99,7 @@ def test_transformers_vocab_sizes(self, model_name):
 
     def test_transformers_vocabs_added_correctly(self):
         namespace, model_name = "tags", "roberta-base"
-        tokenizer = cached_transformers.get_tokenizer(model_name)
+        tokenizer = cached_transformers.get_tokenizer(model_name, use_fast=False)
         allennlp_tokenizer = PretrainedTransformerTokenizer(model_name)
         indexer = PretrainedTransformerIndexer(model_name=model_name, namespace=namespace)
         allennlp_tokens = allennlp_tokenizer.tokenize("AllenNLP is great!")