👨‍💻 Configure HF Hub URL with environment variable

setup.py

@@ -13,7 +13,7 @@ def get_version() -> str:
 
 install_requires = [
     "filelock",
-    "requests",
+    "requests>=2.27",
     "tqdm",
     "pyyaml",
     "typing-extensions>=3.7.4.3",  # to be able to import TypeAlias
@@ -27,11 +27,7 @@ def get_version() -> str:
     "torch",
 ]
 
-extras["tensorflow"] = [
-    "tensorflow",
-    "pydot",
-    "graphviz"
-]
+extras["tensorflow"] = ["tensorflow", "pydot", "graphviz"]
 
 extras["testing"] = [
     "pytest",

src/huggingface_hub/constants.py

@@ -1,4 +1,5 @@
 import os
+from urllib.parse import urlparse
 
 
 # Possible values for env variables
@@ -24,10 +25,10 @@
     os.environ.get("HUGGINGFACE_CO_STAGING", "NO").upper() in ENV_VARS_TRUE_VALUES
 )
 
-ENDPOINT = (
+ENDPOINT = os.getenv("HF_HUB_URL") or (
     "https://moon-staging.huggingface.co" if _staging_mode else "https://huggingface.co"
 )
-
+ENDPOINT_DOMAIN = urlparse(ENDPOINT).hostname or "huggingface.co"
 
 HUGGINGFACE_CO_URL_TEMPLATE = ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"
 

src/huggingface_hub/hf_api.py

@@ -25,6 +25,7 @@
 
 from .constants import (
     ENDPOINT,
+    ENDPOINT_DOMAIN,
     REPO_TYPES,
     REPO_TYPES_MAPPING,
     REPO_TYPES_URL_PREFIXES,
@@ -80,7 +81,7 @@ def _validate_repo_id_deprecation(repo_id, name, organization):
     return name, organization
 
 
-def repo_type_and_id_from_hf_id(hf_id: str):
+def repo_type_and_id_from_hf_id(hf_id: str, _hf_co_domain: Optional[str] = None):
     """
     Returns the repo type and ID from a huggingface.co URL linking to a
     repository
@@ -95,15 +96,16 @@ def repo_type_and_id_from_hf_id(hf_id: str):
             - <namespace>/<repo_id>
             - <repo_id>
     """
-    is_hf_url = "huggingface.co" in hf_id and "@" not in hf_id
+    hf_co_domain = _hf_co_domain if _hf_co_domain is not None else ENDPOINT_DOMAIN
+    is_hf_url = hf_co_domain in hf_id and "@" not in hf_id
     url_segments = hf_id.split("/")
     is_hf_id = len(url_segments) <= 3
 
     if is_hf_url:
         namespace, repo_id = url_segments[-2:]
-        if namespace == "huggingface.co":
+        if namespace == hf_co_domain:
             namespace = None
-        if len(url_segments) > 2 and "huggingface.co" not in url_segments[-3]:
+        if len(url_segments) > 2 and hf_co_domain not in url_segments[-3]:
             repo_type = url_segments[-3]
         else:
             repo_type = None

src/huggingface_hub/repository.py

@@ -8,10 +8,15 @@
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Callable, Dict, Iterator, List, Optional, Tuple, Union
+from urllib.parse import urlparse
 
 from tqdm.auto import tqdm
 
-from huggingface_hub.constants import REPO_TYPES_URL_PREFIXES, REPOCARD_NAME
+from huggingface_hub.constants import (
+    ENDPOINT_DOMAIN,
+    REPO_TYPES_URL_PREFIXES,
+    REPOCARD_NAME,
+)
 from huggingface_hub.repocard import metadata_load, metadata_save
 
 from .hf_api import ENDPOINT, HfApi, HfFolder, repo_type_and_id_from_hf_id
@@ -633,7 +638,7 @@ def clone_from(self, repo_url: str, use_auth_token: Union[bool, str, None] = Non
             )
         api = HfApi()
 
-        if "huggingface.co" in repo_url or (
+        if ENDPOINT_DOMAIN in repo_url or (
             "http" not in repo_url and len(repo_url.split("/")) <= 2
         ):
             repo_type, namespace, repo_id = repo_type_and_id_from_hf_id(repo_url)
@@ -655,7 +660,8 @@ def clone_from(self, repo_url: str, use_auth_token: Union[bool, str, None] = Non
                     repo_id = f"{namespace}/{repo_id}"
                 repo_url += repo_id
 
-                repo_url = repo_url.replace("https://", f"https://user:{token}@")
+                scheme = urlparse(repo_url).scheme
+                repo_url = repo_url.replace(f"{scheme}://", f"{scheme}://user:{token}@")
 
                 if namespace == user or namespace in valid_organisations:
                     api.create_repo(
@@ -671,7 +677,7 @@ def clone_from(self, repo_url: str, use_auth_token: Union[bool, str, None] = Non
                 repo_url += repo_id
 
         # For error messages, it's cleaner to show the repo url without the token.
-        clean_repo_url = re.sub(r"https://.*@", "https://", repo_url)
+        clean_repo_url = re.sub(r"(https?)://.*@", r"\1://", repo_url)
         try:
             subprocess.run(
                 "git lfs install".split(),

tests/test_hf_api.py

@@ -1277,4 +1277,7 @@ def test_repo_type_and_id_from_hf_id(self):
         }
 
         for key, value in possible_values.items():
-            self.assertEqual(repo_type_and_id_from_hf_id(key), tuple(value))
+            self.assertEqual(
+                repo_type_and_id_from_hf_id(key, _hf_co_domain="huggingface.co"),
+                tuple(value),
+            )