Clarify description of the is_split_into_words argument

src/transformers/models/tapas/tokenization_tapas.py

@@ -172,8 +172,9 @@ def whitespace_tokenize(text):
                 length is required by one of the truncation/padding parameters. If the model has no specific maximum
                 input length (like XLNet) truncation/padding to a maximum length will be deactivated.
             is_split_into_words (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Whether or not the input is already pre-tokenized (e.g., split into words), in which case the tokenizer
-                will skip the pre-tokenization step. This is useful for NER or token classification.
+                Wether or not to skip the pre-tokenization step (e.g., splitting input on whitespace). If set to
+                :obj:`True`, the tokenizer assumes the input is already split into words which it will tokenize.
+                This is useful for NER or token classification.
             pad_to_multiple_of (:obj:`int`, `optional`):
                 If set will pad the sequence to a multiple of the provided value. This is especially useful to enable
                 the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).

src/transformers/tokenization_utils.py

@@ -643,7 +643,9 @@ def prepare_for_tokenization(
             text (:obj:`str`):
                 The text to prepare.
             is_split_into_words (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Whether or not the text has been pretokenized.
+                Wether or not to skip the pre-tokenization step (e.g., splitting input on whitespace). If set to
+                :obj:`True`, the tokenizer assumes the input is already split into words which it will tokenize.
+                This is useful for NER or token classification.
             kwargs:
                 Keyword arguments to use for the tokenization.
 

src/transformers/tokenization_utils_base.py

@@ -1286,8 +1286,9 @@ def all_special_ids(self) -> List[int]:
                 returned to provide some overlap between truncated and overflowing sequences. The value of this
                 argument defines the number of overlapping tokens.
             is_split_into_words (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Whether or not the input is already pre-tokenized (e.g., split into words), in which case the tokenizer
-                will skip the pre-tokenization step. This is useful for NER or token classification.
+                Wether or not to skip the pre-tokenization step (e.g., splitting input on whitespace). If set to
+                :obj:`True`, the tokenizer assumes the input is already split into words which it will tokenize.
+                This is useful for NER or token classification.
             pad_to_multiple_of (:obj:`int`, `optional`):
                 If set will pad the sequence to a multiple of the provided value. This is especially useful to enable
                 the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).