From 0e9bbedffd4a810af8b31aac0dfe71ccd337f74e Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" <git@varonathe.org> Date: Mon, 10 Jan 2022 10:21:02 -0500 Subject: [PATCH] Add a wrapper function for the calls to `find_files()` in tests --- dandi/tests/test_dandiapi.py | 21 ++++++++++----------- dandi/tests/test_delete.py | 13 ++++++------- dandi/tests/test_download.py | 19 ++++++------------- dandi/tests/test_upload.py | 7 +++---- dandi/utils.py | 6 +++++- 5 files changed, 30 insertions(+), 36 deletions(-) diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 17fb3cda5..dc3cddd94 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -2,7 +2,6 @@ from datetime import datetime, timezone import logging import os.path -from pathlib import Path import random import re from shutil import rmtree @@ -25,7 +24,7 @@ from ..download import download from ..exceptions import NotFoundError, SchemaVersionError from ..upload import upload -from ..utils import find_files +from ..utils import list_paths def test_upload(local_dandi_api, simple1_nwb, tmp_path): @@ -87,14 +86,11 @@ def test_publish_and_manipulate(local_dandi_api, monkeypatch, tmp_path): download_dir = tmp_path / "download" download_dir.mkdir() - def downloaded_files(): - return list(map(Path, find_files(r".*", paths=[download_dir]))) - dandiset_yaml = download_dir / dandiset_id / dandiset_metadata_file file_in_version = download_dir / dandiset_id / "subdir" / "file.txt" download(dv.version_api_url, download_dir) - assert downloaded_files() == [dandiset_yaml, file_in_version] + assert list_paths(download_dir) == [dandiset_yaml, file_in_version] assert file_in_version.read_text() == "This is test text.\n" (upload_dir / "subdir" / "file.txt").write_text("This is different text.\n") @@ -107,7 +103,7 @@ def downloaded_files(): ) rmtree(download_dir / dandiset_id) download(dv.version_api_url, download_dir) - assert downloaded_files() == [dandiset_yaml, file_in_version] + assert list_paths(download_dir) == [dandiset_yaml, file_in_version] assert file_in_version.read_text() == "This is test text.\n" (upload_dir / "subdir" / "file2.txt").write_text("This is more text.\n") @@ -121,7 +117,7 @@ def downloaded_files(): rmtree(download_dir / dandiset_id) download(d.version_api_url, download_dir) - assert sorted(downloaded_files()) == [ + assert list_paths(download_dir) == [ dandiset_yaml, file_in_version, file_in_version.with_name("file2.txt"), @@ -131,19 +127,22 @@ def downloaded_files(): rmtree(download_dir / dandiset_id) download(dv.version_api_url, download_dir) - assert downloaded_files() == [dandiset_yaml, file_in_version] + assert list_paths(download_dir) == [dandiset_yaml, file_in_version] assert file_in_version.read_text() == "This is test text.\n" d.get_asset_by_path("subdir/file.txt").delete() rmtree(download_dir / dandiset_id) download(d.version_api_url, download_dir) - assert downloaded_files() == [dandiset_yaml, file_in_version.with_name("file2.txt")] + assert list_paths(download_dir) == [ + dandiset_yaml, + file_in_version.with_name("file2.txt"), + ] assert file_in_version.with_name("file2.txt").read_text() == "This is more text.\n" rmtree(download_dir / dandiset_id) download(dv.version_api_url, download_dir) - assert downloaded_files() == [dandiset_yaml, file_in_version] + assert list_paths(download_dir) == [dandiset_yaml, file_in_version] assert file_in_version.read_text() == "This is test text.\n" diff --git a/dandi/tests/test_delete.py b/dandi/tests/test_delete.py index a71cde396..e5fa7128b 100644 --- a/dandi/tests/test_delete.py +++ b/dandi/tests/test_delete.py @@ -7,7 +7,7 @@ from ..delete import delete from ..download import download from ..exceptions import NotFoundError -from ..utils import find_files +from ..utils import list_paths @pytest.mark.parametrize( @@ -70,8 +70,9 @@ def test_delete_paths( ) delete_spy.assert_called() download(text_dandiset["dandiset"].version_api_url, tmp_path) - files = sorted(map(Path, find_files(r".*", paths=[tmp_path]))) - assert files == [tmp_path / dandiset_id / f for f in ["dandiset.yaml"] + remainder] + assert list_paths(tmp_path) == [ + tmp_path / dandiset_id / f for f in ["dandiset.yaml"] + remainder + ] @pytest.mark.parametrize("confirm", [True, False]) @@ -276,8 +277,7 @@ def test_delete_nonexistent_asset_skip_missing( ) delete_spy.assert_called() download(text_dandiset["dandiset"].version_api_url, tmp_path) - files = sorted(map(Path, find_files(r".*", paths=[tmp_path]))) - assert files == [ + assert list_paths(tmp_path) == [ tmp_path / dandiset_id / "dandiset.yaml", tmp_path / dandiset_id / "subdir1" / "apple.txt", tmp_path / dandiset_id / "subdir2" / "banana.txt", @@ -328,8 +328,7 @@ def test_delete_nonexistent_asset_folder_skip_missing( ) delete_spy.assert_called() download(text_dandiset["dandiset"].version_api_url, tmp_path) - files = sorted(map(Path, find_files(r".*", paths=[tmp_path]))) - assert files == [ + assert list_paths(tmp_path) == [ tmp_path / dandiset_id / "dandiset.yaml", tmp_path / dandiset_id / "file.txt", tmp_path / dandiset_id / "subdir2" / "banana.txt", diff --git a/dandi/tests/test_download.py b/dandi/tests/test_download.py index 0ea180ed8..7e76ebf14 100644 --- a/dandi/tests/test_download.py +++ b/dandi/tests/test_download.py @@ -1,7 +1,6 @@ import json import os import os.path as op -from pathlib import Path import re from shutil import rmtree @@ -12,7 +11,7 @@ from ..consts import DRAFT, dandiset_metadata_file from ..dandiarchive import DandisetURL from ..download import download, download_generator -from ..utils import find_files +from ..utils import list_paths # both urls point to 000027 (lean test dataset), and both draft and "released" @@ -137,7 +136,7 @@ def test_download_folder(local_dandi_api, text_dandiset, tmp_path): download( f"dandi://{local_dandi_api['instance_id']}/{dandiset_id}/subdir2/", tmp_path ) - assert sorted(map(Path, find_files(r".*", paths=[tmp_path], dirs=True))) == [ + assert list_paths(tmp_path, dirs=True) == [ tmp_path / "subdir2", tmp_path / "subdir2" / "banana.txt", tmp_path / "subdir2" / "coconut.txt", @@ -152,27 +151,21 @@ def test_download_item(local_dandi_api, text_dandiset, tmp_path): f"dandi://{local_dandi_api['instance_id']}/{dandiset_id}/subdir2/coconut.txt", tmp_path, ) - assert list(map(Path, find_files(r".*", paths=[tmp_path], dirs=True))) == [ - tmp_path / "coconut.txt" - ] + assert list_paths(tmp_path, dirs=True) == [tmp_path / "coconut.txt"] assert (tmp_path / "coconut.txt").read_text() == "Coconut\n" def test_download_asset_id(text_dandiset, tmp_path): asset = text_dandiset["dandiset"].get_asset_by_path("subdir2/coconut.txt") download(asset.download_url, tmp_path) - assert list(map(Path, find_files(r".*", paths=[tmp_path], dirs=True))) == [ - tmp_path / "coconut.txt" - ] + assert list_paths(tmp_path, dirs=True) == [tmp_path / "coconut.txt"] assert (tmp_path / "coconut.txt").read_text() == "Coconut\n" def test_download_asset_id_only(text_dandiset, tmp_path): asset = text_dandiset["dandiset"].get_asset_by_path("subdir2/coconut.txt") download(asset.base_download_url, tmp_path) - assert list(map(Path, find_files(r".*", paths=[tmp_path], dirs=True))) == [ - tmp_path / "coconut.txt" - ] + assert list_paths(tmp_path, dirs=True) == [tmp_path / "coconut.txt"] assert (tmp_path / "coconut.txt").read_text() == "Coconut\n" @@ -266,7 +259,7 @@ def test_download_metadata404(text_dandiset, tmp_path): "message": f"No such asset: {asset}", } ] - assert sorted(map(Path, find_files(r".*", paths=[tmp_path], dirs=True))) == [ + assert list_paths(tmp_path, dirs=True) == [ tmp_path / dandiset_metadata_file, tmp_path / "file.txt", tmp_path / "subdir2", diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py index e4b1cc16f..2881c9de7 100644 --- a/dandi/tests/test_upload.py +++ b/dandi/tests/test_upload.py @@ -10,7 +10,7 @@ from ..exceptions import NotFoundError from ..pynwb_utils import make_nwb_file from ..upload import upload -from ..utils import find_files +from ..utils import list_paths def test_new_upload_download(local_dandi_api, monkeypatch, organized_nwb_dir, tmp_path): @@ -138,12 +138,11 @@ def test_upload_download_small_file(contents, local_dandi_api, monkeypatch, tmp_ download_dir = tmp_path / "download" download_dir.mkdir() download(d.version_api_url, download_dir) - files = sorted(map(Path, find_files(r".*", paths=[download_dir]))) - assert files == [ + assert list_paths(download_dir) == [ download_dir / dandiset_id / dandiset_metadata_file, download_dir / dandiset_id / "file.txt", ] - assert files[1].read_bytes() == contents + assert (download_dir / dandiset_id / "file.txt").read_bytes() == contents @pytest.mark.parametrize("confirm", [True, False]) diff --git a/dandi/utils.py b/dandi/utils.py index 82f9e4674..4a7b59971 100644 --- a/dandi/utils.py +++ b/dandi/utils.py @@ -19,7 +19,7 @@ import subprocess import sys import types -from typing import Optional, Union +from typing import List, Optional, Union import dateutil.parser import requests @@ -324,6 +324,10 @@ def good_file(path): yield path +def list_paths(dirpath: Union[str, Path], dirs: bool = False) -> List[Path]: + return sorted(map(Path, find_files(r".*", [dirpath], dirs=dirs))) + + _cp_supports_reflink = None