From c5ede994da172eefd347ee5725b1df42563e8277 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Mon, 4 Oct 2021 10:38:32 -0400 Subject: [PATCH 1/2] Make the `validate` command validate all files, not just .nwb's --- dandi/cli/cmd_validate.py | 2 +- dandi/validate.py | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/dandi/cli/cmd_validate.py b/dandi/cli/cmd_validate.py index 9bbceff2f..6cd71e9f9 100644 --- a/dandi/cli/cmd_validate.py +++ b/dandi/cli/cmd_validate.py @@ -37,7 +37,7 @@ def validate(paths, schema=None, devel_debug=False): all_files_errors = {} nfiles = 0 for path, errors in validate_( - paths, schema_version=schema, devel_debug=devel_debug + paths, schema_version=schema, devel_debug=devel_debug, allow_any_path=True ): nfiles += 1 if view == "one-at-a-time": diff --git a/dandi/validate.py b/dandi/validate.py index 89229bdd6..16d057442 100644 --- a/dandi/validate.py +++ b/dandi/validate.py @@ -5,7 +5,7 @@ from .metadata import get_metadata from .pynwb_utils import validate as pynwb_validate from .pynwb_utils import validate_cache -from .utils import find_dandi_files, yaml_load +from .utils import find_dandi_files, find_files, yaml_load lgr = get_logger() @@ -15,7 +15,7 @@ # TODO: provide our own "errors" records, which would also include warnings etc -def validate(paths, schema_version=None, devel_debug=False): +def validate(paths, schema_version=None, devel_debug=False, allow_any_path=False): """Validate content Parameters @@ -28,7 +28,8 @@ def validate(paths, schema_version=None, devel_debug=False): path, errors errors for a path """ - for path in find_dandi_files(paths): + filepaths = find_files(".*", paths) if allow_any_path else find_dandi_files(paths) + for path in filepaths: errors = validate_file( path, schema_version=schema_version, devel_debug=devel_debug ) @@ -41,7 +42,7 @@ def validate_file(filepath, schema_version=None, devel_debug=False): filepath, schema_version=None, devel_debug=devel_debug ) else: - return pynwb_validate(filepath, devel_debug=devel_debug) + validate_dandi_nwb( + return pynwb_validate(filepath, devel_debug=devel_debug) + validate_asset_file( filepath, schema_version=schema_version, devel_debug=devel_debug ) @@ -88,13 +89,13 @@ def validate_dandiset_yaml(filepath, schema_version=None, devel_debug=False): @validate_cache.memoize_path -def validate_dandi_nwb(filepath, schema_version=None, devel_debug=False): - """Provide validation of .nwb file regarding requirements we impose""" +def validate_asset_file(filepath, schema_version=None, devel_debug=False): + """Provide validation of asset file regarding requirements we impose""" if schema_version is not None: from dandischema.models import BareAsset, get_schema_version from pydantic import ValidationError - from .metadata import nwb2asset + from .metadata import get_asset_metadata current_version = get_schema_version() if schema_version != current_version: @@ -102,8 +103,12 @@ def validate_dandi_nwb(filepath, schema_version=None, devel_debug=False): f"Unsupported schema version: {schema_version}; expected {current_version}" ) try: - asset = nwb2asset( - filepath, digest=32 * "d" + "-1", digest_type="dandi_etag" + asset = get_asset_metadata( + filepath, + relpath="dummy", + digest=32 * "d" + "-1", + digest_type="dandi_etag", + allow_any_path=True, ) BareAsset(**asset.dict()) except ValidationError as e: From f911a2c5ea4942b8c2967102bf59e5b3cbb45c46 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Mon, 4 Oct 2021 11:10:59 -0400 Subject: [PATCH 2/2] Give `validate` command an `--allow-any-path` option --- dandi/cli/cmd_validate.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dandi/cli/cmd_validate.py b/dandi/cli/cmd_validate.py index 6cd71e9f9..21ba094bb 100644 --- a/dandi/cli/cmd_validate.py +++ b/dandi/cli/cmd_validate.py @@ -8,10 +8,16 @@ @click.command() @devel_option("--schema", help="Validate against new schema version", metavar="VERSION") +@devel_option( + "--allow-any-path", + help="For development: allow DANDI 'unsupported' file types/paths", + default=False, + is_flag=True, +) @click.argument("paths", nargs=-1, type=click.Path(exists=True, dir_okay=True)) @devel_debug_option() @map_to_click_exceptions -def validate(paths, schema=None, devel_debug=False): +def validate(paths, schema=None, devel_debug=False, allow_any_path=False): """Validate files for NWB (and DANDI) compliance. Exits with non-0 exit code if any file is not compliant. @@ -37,7 +43,10 @@ def validate(paths, schema=None, devel_debug=False): all_files_errors = {} nfiles = 0 for path, errors in validate_( - paths, schema_version=schema, devel_debug=devel_debug, allow_any_path=True + paths, + schema_version=schema, + devel_debug=devel_debug, + allow_any_path=allow_any_path, ): nfiles += 1 if view == "one-at-a-time":