Skip to content

Commit

Permalink
Add download support for modelscople. (#2032)
Browse files Browse the repository at this point in the history
* Add modelscope support

* Add modelscope support

* renew error message

* format data
  • Loading branch information
pass-lin authored Feb 10, 2025
1 parent abf7432 commit 60d9012
Showing 1 changed file with 33 additions and 1 deletion.
34 changes: 33 additions & 1 deletion keras_hub/src/utils/preset_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,12 @@
KAGGLE_PREFIX = "kaggle://"
GS_PREFIX = "gs://"
HF_PREFIX = "hf://"
MODELSCOPE_PREFIX = "modelscope://"

KAGGLE_SCHEME = "kaggle"
GS_SCHEME = "gs"
HF_SCHEME = "hf"

MODELSCOPE_SCHEME = "modelscope"
ASSET_DIR = "assets"
TOKENIZER_ASSET_DIR = f"{ASSET_DIR}/tokenizer"

Expand Down Expand Up @@ -165,6 +166,37 @@ def get_file(preset, path):
raise ValueError(message)
elif scheme in tf_registered_schemes():
return tf_copy_gfile_to_cache(preset, path)
elif scheme == MODELSCOPE_SCHEME:
try:
from modelscope.hub.snapshot_download import snapshot_download
except ImportError:
raise ImportError(
"To load a preset from ModelScope {preset} using from_preset,"
"install the modelscope package with: pip install modelscope."
)
modelscope_handle = preset.removeprefix(MODELSCOPE_SCHEME + "://")
try:
return_path = snapshot_download(modelscope_handle) + "/" + path
if os.path.exists(return_path):
return return_path
raise FileNotFoundError(
f"`{return_path}` doesn't exist in preset directory `{preset}`."
)
except ValueError as e:
raise ValueError(
"ModelScope handles should follow the format "
f"'modelscope://{{org}}/{{model}}' "
"(e.g., 'modelscope://username/bert_base_en')."
f"Received: preset='{preset}.'"
) from e
except EntryNotFoundError as e:
message = str(e)
if message.find("403 Client Error"):
raise FileNotFoundError(
f"`{path}` not exist in preset directory `{preset}`."
)
else:
raise ValueError(message)
elif scheme == HF_SCHEME:
if huggingface_hub is None:
raise ImportError(
Expand Down

0 comments on commit 60d9012

Please sign in to comment.