Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix dataset download location #1639

Merged
merged 7 commits into from
Nov 5, 2024
Merged

Fix dataset download location #1639

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
189ca97
do it
dakinggg Nov 5, 2024
235f8f1
pc
dakinggg Nov 5, 2024
5a8c532
fix
dakinggg Nov 5, 2024
9b2e96b
fix ci
dakinggg Nov 5, 2024
7cbeb7a
clean up
dakinggg Nov 5, 2024
c8a843f
try skip
dakinggg Nov 5, 2024
8ad1491
pc
dakinggg Nov 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions llmfoundry/data/finetuning/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ def preprocessing_fn(example: Dict) -> Dict[str, str]:
import importlib
import logging
import os
import tempfile
import warnings
from collections.abc import Mapping
from functools import partial
Expand Down Expand Up @@ -93,6 +92,7 @@ def preprocessing_fn(example: Dict) -> Dict[str, str]:
UnknownExampleTypeError,
)
# yapf: enable
from llmfoundry.utils.file_utils import dist_mkdtemp
from llmfoundry.utils.logging_utils import SpecificWarningFilter

log = logging.getLogger(__name__)
Expand Down Expand Up @@ -888,6 +888,8 @@ def build_from_hf(

signal_file_path = dist.get_node_signal_file_name()

download_folder = dist_mkdtemp()

# Non local rank 0 ranks will wait here for local rank 0 to finish the data processing.
# Once local rank 0 is done, the datasets are all cached on disk, and all other ranks
# can just read them.
Expand All @@ -913,7 +915,7 @@ def build_from_hf(
if not os.path.isdir(dataset_name):
# dataset_name is not a local dir path, download if needed.
local_dataset_dir = os.path.join(
tempfile.mkdtemp(),
download_folder,
dataset_name,
)

Expand Down
5 changes: 3 additions & 2 deletions tests/data/test_dataloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ def test_finetuning_dataloader_safe_load(

tokenizer = build_tokenizer('gpt2', {})

with patch('llmfoundry.data.finetuning.tasks.tempfile.mkdtemp', return_value=str(tmp_path)):
with patch('llmfoundry.utils.file_utils.tempfile.mkdtemp', return_value=str(tmp_path)), patch('os.cpu_count', return_value=1):
with expectation:
_ = build_finetuning_dataloader(
tokenizer=tokenizer,
Expand Down Expand Up @@ -1516,7 +1516,8 @@ def test_ft_dataloader_with_extra_keys():
device_batch_size=device_batch_size,
).dataloader

@pytest.mark.xfail
# TODO: Change this back to xfail after figuring out why it caused CI to hang
@pytest.mark.skip
def test_text_dataloader_with_extra_keys():
max_seq_len = 1024
cfg = {
Expand Down
Loading