Skip to content

Commit

Permalink
use block_size
Browse files Browse the repository at this point in the history
  • Loading branch information
lhoestq committed Jan 11, 2024
1 parent 0d1e704 commit 16ee3b3
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions src/datasets/download/streaming_download_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ def xopen(file: str, mode="r", *args, download_config: Optional[DownloadConfig]
main_hop, *rest_hops = file_str.split("::")
if is_local_path(main_hop):
# ignore fsspec-specific kwargs
kwargs.pop("chunk_size", None)
kwargs.pop("block_size", None)
return open(main_hop, mode, *args, **kwargs)
# add headers and cookies for authentication on the HF Hub and for Google Drive
file, storage_options = _prepare_path_and_storage_options(file_str, download_config=download_config)
Expand Down Expand Up @@ -915,7 +915,7 @@ def _iter_from_urlpath(
compression = _get_extraction_protocol(urlpath, download_config=download_config)
# Set chunk_size=0 to get faster streaming
# (e.g. for hf:// and https:// it uses streaming Requests file-like instances)
with xopen(urlpath, "rb", download_config=download_config, chunk_size=0) as f:
with xopen(urlpath, "rb", download_config=download_config, block_size=0) as f:
if compression == "zip":
yield from cls._iter_zip(f)
else:
Expand Down

0 comments on commit 16ee3b3

Please sign in to comment.