From ac93708e7045d9f7c04fa8671e2fc05edcf1f70c Mon Sep 17 00:00:00 2001 From: Gregory Brodsky Date: Tue, 5 Sep 2023 11:05:57 +0300 Subject: [PATCH 1/4] Fix bug #1634 (drop finishing spaces and EOL) --- src/huggingface_hub/utils/_hf_folder.py | 5 +++- tests/test_utils_hf_folder.py | 31 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/huggingface_hub/utils/_hf_folder.py b/src/huggingface_hub/utils/_hf_folder.py index 5c9f07c9ba..77daced7e8 100644 --- a/src/huggingface_hub/utils/_hf_folder.py +++ b/src/huggingface_hub/utils/_hf_folder.py @@ -65,11 +65,14 @@ def get_token(cls) -> Optional[str]: # 1. Is it set by environment variable ? token: Optional[str] = os.environ.get("HUGGING_FACE_HUB_TOKEN") if token is not None: + token = token.replace("\r", "").replace("\n", "").strip() return token # 2. Is it set in token path ? try: - return cls.path_token.read_text() + token = cls.path_token.read_text() + token = token.replace("\r", "").replace("\n", "").strip() + return token except FileNotFoundError: return None diff --git a/tests/test_utils_hf_folder.py b/tests/test_utils_hf_folder.py index 7a1bcf19da..fb9ad3b418 100644 --- a/tests/test_utils_hf_folder.py +++ b/tests/test_utils_hf_folder.py @@ -83,3 +83,34 @@ def test_token_in_old_path(self): # Un-patch new_patcher.stop() old_patcher.stop() + + def test_token_strip(self): + """ + Test the workflow when the token is mistakenly finishing with new-line or space character. + """ + token = _generate_token() + HfFolder.save_token(token + "\n") + self.assertEqual(HfFolder.get_token(), token) + HfFolder.delete_token() + HfFolder.delete_token() + # ^^ not an error, we test that the + # second call does not fail. + self.assertEqual(HfFolder.get_token(), None) + # test TOKEN in env + self.assertEqual(HfFolder.get_token(), None) + with unittest.mock.patch.dict(os.environ, {"HUGGING_FACE_HUB_TOKEN": token}): + self.assertEqual(HfFolder.get_token(), token) + + token = _generate_token() + HfFolder.save_token(token + " ") + self.assertEqual(HfFolder.get_token(), token) + HfFolder.delete_token() + HfFolder.delete_token() + # ^^ not an error, we test that the + # second call does not fail. + self.assertEqual(HfFolder.get_token(), None) + # test TOKEN in env + self.assertEqual(HfFolder.get_token(), None) + with unittest.mock.patch.dict(os.environ, {"HUGGING_FACE_HUB_TOKEN": token}): + self.assertEqual(HfFolder.get_token(), token) + From a81a93c36749d6d58088027cc7f2c9ffb096929f Mon Sep 17 00:00:00 2001 From: Lucain Date: Tue, 5 Sep 2023 10:24:40 +0200 Subject: [PATCH 2/4] Update tests/test_utils_hf_folder.py --- tests/test_utils_hf_folder.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/tests/test_utils_hf_folder.py b/tests/test_utils_hf_folder.py index fb9ad3b418..8b9a729836 100644 --- a/tests/test_utils_hf_folder.py +++ b/tests/test_utils_hf_folder.py @@ -89,28 +89,7 @@ def test_token_strip(self): Test the workflow when the token is mistakenly finishing with new-line or space character. """ token = _generate_token() - HfFolder.save_token(token + "\n") + HfFolder.save_token(" " + token + "\n") self.assertEqual(HfFolder.get_token(), token) HfFolder.delete_token() - HfFolder.delete_token() - # ^^ not an error, we test that the - # second call does not fail. - self.assertEqual(HfFolder.get_token(), None) - # test TOKEN in env - self.assertEqual(HfFolder.get_token(), None) - with unittest.mock.patch.dict(os.environ, {"HUGGING_FACE_HUB_TOKEN": token}): - self.assertEqual(HfFolder.get_token(), token) - - token = _generate_token() - HfFolder.save_token(token + " ") - self.assertEqual(HfFolder.get_token(), token) - HfFolder.delete_token() - HfFolder.delete_token() - # ^^ not an error, we test that the - # second call does not fail. - self.assertEqual(HfFolder.get_token(), None) - # test TOKEN in env - self.assertEqual(HfFolder.get_token(), None) - with unittest.mock.patch.dict(os.environ, {"HUGGING_FACE_HUB_TOKEN": token}): - self.assertEqual(HfFolder.get_token(), token) From 3f67344f8f2be237ae7468b1c80a4e1cce5b0c8e Mon Sep 17 00:00:00 2001 From: Lucain Date: Tue, 5 Sep 2023 10:28:46 +0200 Subject: [PATCH 3/4] Update tests/test_utils_hf_folder.py --- tests/test_utils_hf_folder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_hf_folder.py b/tests/test_utils_hf_folder.py index 8b9a729836..a568624e72 100644 --- a/tests/test_utils_hf_folder.py +++ b/tests/test_utils_hf_folder.py @@ -92,4 +92,4 @@ def test_token_strip(self): HfFolder.save_token(" " + token + "\n") self.assertEqual(HfFolder.get_token(), token) HfFolder.delete_token() - + From b8c49abc5f45fbe02db8912be9bc5a281bfcca83 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Tue, 5 Sep 2023 12:01:59 +0200 Subject: [PATCH 4/4] make style --- tests/test_utils_hf_folder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_utils_hf_folder.py b/tests/test_utils_hf_folder.py index a568624e72..ce0448ec49 100644 --- a/tests/test_utils_hf_folder.py +++ b/tests/test_utils_hf_folder.py @@ -92,4 +92,3 @@ def test_token_strip(self): HfFolder.save_token(" " + token + "\n") self.assertEqual(HfFolder.get_token(), token) HfFolder.delete_token() -