Merge pull request #588 from dandi/gh-587

Remove Girder support

.github/workflows/test.yml

@@ -82,10 +82,6 @@ jobs:
       run: |
         pip install git+https://github.com/NeurodataWithoutBorders/pynwb
 
-    - name: Set environment variable when using Python 3.7 to cover more code
-      run: echo DANDI_LOG_GIRDER=1 >> "$GITHUB_ENV"
-      if: matrix.python == '3.7'
-
     - name: Run all tests
       if: "!matrix.dandi_api"
       run: |

DEVELOPMENT.md

@@ -21,65 +21,40 @@ In the source directory
 pre-commit install
 ```
 
-### dandiarchive instance
-
-[dandiarchive](https://github.com/dandi/dandiarchive) repository provides
-docker-compose recipe to establish local instance of the minimally provisioned
-dandiarchive (both with our web frontend, and girder backend).
-See [README.md:Docker](https://github.com/dandi/dandiarchive#docker) for the
-instructions.  In a new instance you would need to generate a new API key to be
-used by `dandi` client for upload etc.
-
-Relevant `dandi` client commands are aware of such an instance (such as `upload`)
-as `local-docker` (as opposed from `local` for a plain girder instance).  See note
-below on `DANDI_DEVEL` environment variable which would be needed to expose
+### dandi-api instance
+
+The [dandi-api](https://github.com/dandi/dandi-api) repository provides a
+docker-compose recipe for establishing a local instance of a fresh dandi-api.
+See
+[README.md:Docker](https://github.com/dandi/dandi-api#develop-with-docker-recommended-quickstart)
+for the instructions.  In a new instance, you would need to generate a new API
+key to be used by the `dandi` client for upload etc.
+
+Relevant `dandi` client commands (such as `upload`) are aware of such an
+instance as `dandi-api-local-docker-tests`.  See the note below on the
+`DANDI_DEVEL` environment variable, which is needed in order to expose the
 development command line options.
 
-## Operating against current "beta" instance of the dandiarchive.org
-
-`dandi-cli` has all functionality for interaction with the beta instance deployed at
-https://gui-beta-dandiarchive-org.netlify.app . Some operations (like `upload`) would
-need that instance to be explicitly specified, and for that `DANDI_DEVEL=1` env variable
-should be set.  Here is an example for the upload invocation
-
-   DANDI_DEVEL=1 dandi upload -i dandi-api
-
-in the simplest scenario.  Additional options might come handy, such as
-
-- `--jobs 10:2` - upload 10 files in parallel with up to 2 threads per file
-- `--allow-any-path --validation ignore` - if you need to upload non-nwb files
-
 ## Environment variables
 
-- `DANDI_DEVEL` -- enables otherwise hidden command line options,
-  such as explicit specification of the girder instance, collection, etc.
-  All those options would otherwise be hidden from the user visible (`--help`)
-  interface, unless this env variable is set to non-empty value
+- `DANDI_DEVEL` -- enables otherwise hidden command line options, such as
+  explicit specification of the dandi-api instance.  All those options would
+  otherwise be hidden from the user-visible (`--help`) interface, unless this
+  env variable is set to a non-empty value
 
 - `DANDI_API_KEY` -- avoids using keyrings, thus making it possible to
   "temporarily" use another account etc for the "API" version of the server.
 
-- `DANDI_GIRDER_API_KEY` -- avoids using keyrings, thus making it possible to
-  "temporarily" use another account etc for the Girder version of the server.
-
 - `DANDI_LOG_LEVEL` -- set log level. By default `INFO`, should be an int (`10` - `DEBUG`).
 
-- `DANDI_LOG_GIRDER` -- log REST requests.
-
 - `DANDI_CACHE` -- clear persistent cache handling. Known values
   are `clear` - would clear the cache, `ignore` - would ignore it. Note that for
   metadata cache we use only released portion of `dandi.__version__` as a token.
   If handling of metadata has changed while developing, set this env var to
   `clear` to have cache `clear()`ed before use.
 
 - `DANDI_INSTANCEHOST` -- defaults to `localhost`. Point to host/IP which hosts
-  a local instance of dandiarchive. Typically,
-  `DANDI_REUSE_LOCAL_DOCKER_TESTS_API_KEY`
-  should also be set.
-
-- `DANDI_REUSE_LOCAL_DOCKER_TESTS_API_KEY` -- make the
- `local_docker_compose*` fixtures use the given API key for
- `DANDI_INSTANCEHOST` instead of spinning up a new environment with a new key.
+  a local instance of dandiarchive.
 
 - `DANDI_TESTS_PERSIST_DOCKER_COMPOSE` -- When set, the tests will reuse the
   same Docker containers across test runs instead of creating & destroying a

dandi/cli/base.py

@@ -101,10 +101,6 @@ def map_to_click_exceptions(f):
     @click.pass_obj
     @wraps(f)
     def wrapper(obj, *args, **kwargs):
-        import girder_client as gcl
-
-        from ..girder import get_HttpError_response
-
         try:
             return f(*args, **kwargs)
         # Prints global Usage: useless in majority of cases.
@@ -113,14 +109,7 @@ def wrapper(obj, *args, **kwargs):
         # except ValueError as e:
         #     raise click.UsageError(str(e))
         except Exception as e:
-            if isinstance(e, gcl.HttpError):
-                resp = get_HttpError_response(e)
-                if resp is None:
-                    e_str = str(e)
-                else:
-                    e_str = resp.get("message", str(e))
-            else:
-                e_str = str(e)
+            e_str = str(e)
             lgr.debug("Caught exception %s", e_str)
             if not map_to_click_exceptions._do_map:
                 raise

dandi/cli/cmd_ls.py

@@ -68,7 +68,6 @@ def ls(paths, schema, metadata, fields=None, format="auto", recursive=False, job
     # TODO: more logical ordering in case of fields = None
     from .formatter import JSONFormatter, PYOUTFormatter, YAMLFormatter
     from ..consts import metadata_all_fields
-    from ..dandiapi import DandiAPIClient
 
     # TODO: avoid
     from ..support.pyout import PYOUT_SHORT_NAMES_rev
@@ -128,10 +127,7 @@ def assets_gen():
                     else:
                         dandiset_id = None
                     if recursive and assets:
-                        if isinstance(client, DandiAPIClient) and metadata in (
-                            "all",
-                            "assets",
-                        ):
+                        if metadata in ("all", "assets"):
                             for a in assets:
                                 if "metadata" not in a:
                                     a["metadata"] = client.get_asset(

dandi/cli/cmd_upload.py

@@ -6,7 +6,6 @@
     instance_option,
     map_to_click_exceptions,
 )
-from ..consts import collection_drafts
 
 
 class IntColonInt(click.ParamType):
@@ -64,13 +63,6 @@ def get_metavar(self, param):
 #
 # TODO: should always go to dandi for now
 @instance_option()
-# TODO: should always go into 'drafts' (consts.collection_drafts)
-@devel_option(
-    "-c", "--girder-collection", help="For development: Girder collection to upload to"
-)
-# TODO: figure out folder for the dandiset
-@devel_option("--girder-top-folder", help="For development: Girder top folder")
-#
 @devel_option(
     "--fake-data",
     help="For development: fake file content (filename will be stored instead of actual load)",
@@ -98,8 +90,6 @@ def upload(
     validation="require",
     dandiset_path=None,
     # Development options should come as kwargs
-    girder_collection=collection_drafts,
-    girder_top_folder=None,
     dandi_instance="dandi",
     fake_data=False,  # TODO: not implemented, prune?
     allow_any_path=False,
@@ -132,8 +122,6 @@ def upload(
         existing=existing,
         validation=validation,
         dandiset_path=dandiset_path,
-        girder_collection=girder_collection,
-        girder_top_folder=girder_top_folder,
         dandi_instance=dandi_instance,
         fake_data=fake_data,
         allow_any_path=allow_any_path,

dandi/cli/command.py

@@ -144,11 +144,10 @@ def main(ctx, log_level, pdb=False):
 from .cmd_download import download  # noqa: E402
 from .cmd_ls import ls  # noqa: E402
 from .cmd_organize import organize  # noqa: E402
-from .cmd_register import register  # noqa: E402
 from .cmd_upload import upload  # noqa: E402
 from .cmd_validate import validate  # noqa: E402
 
-__all_commands__ = (ls, organize, upload, download, validate, register, digest, delete)
+__all_commands__ = (ls, organize, upload, download, validate, digest, delete)
 
 for cmd in __all_commands__:
     main.add_command(cmd)

dandi/consts.py

@@ -1,11 +1,9 @@
-# name: url
 from collections import namedtuple
+import os
 
 # A list of metadata fields which dandi extracts from .nwb files.
 # Additional fields (such as `number_of_*`) might be added by the
 # get_metadata`
-import os
-
 metadata_nwb_file_fields = (
     "experiment_description",
     "experimenter",
@@ -66,67 +64,33 @@
 
 metadata_all_fields = metadata_nwb_fields + metadata_dandiset_fields
 
-# checksums and other digests to compute on the files to upload
-# Order matters - observed compute time from shorter to longer
-# Those are not to be included in metadata reported for a local file,
-# but will be available for files in the archive
-metadata_digests = ("sha1", "md5", "sha512", "sha256")
-
 dandiset_metadata_file = "dandiset.yaml"
 dandiset_identifier_regex = "^[0-9]{6}$"
 
-dandi_instance = namedtuple(
-    "dandi_instance", ("metadata_version", "girder", "gui", "redirector", "api")
-)
+dandi_instance = namedtuple("dandi_instance", ("gui", "redirector", "api"))
 
 # So it could be easily mapped to external IP (e.g. from within VM)
 # to test against instance running outside of current environment
 instancehost = os.environ.get("DANDI_INSTANCEHOST", "localhost")
 
 known_instances = {
-    "local-girder-only": dandi_instance(
-        0, f"http://{instancehost}:8080", None, None, None
-    ),  # just pure girder
-    # Redirector: TODO https://github.com/dandi/dandiarchive/issues/139
-    "local-docker": dandi_instance(
-        0,
-        f"http://{instancehost}:8080",
-        f"http://{instancehost}:8085",
-        None,
-        f"http://{instancehost}:9000",  # ATM it is minio, not sure where /api etc
-        # may be https://github.com/dandi/dandi-publish/pull/71 would help
-    ),
-    "local-docker-tests": dandi_instance(
-        0,
-        f"http://{instancehost}:8081",
-        f"http://{instancehost}:8086",
-        f"http://{instancehost}:8079",
-        None,
-    ),
     "dandi": dandi_instance(
-        0,
-        "https://girder.dandiarchive.org",
         "https://gui.dandiarchive.org",
         "https://dandiarchive.org",
-        None,  # publish. is gone, superseded by API which did not yet fully superseded the rest
+        "https://api.dandiarchive.org/api",
     ),
-    "dandi-api": dandi_instance(
-        1,
-        None,
+    "dandi-devel": dandi_instance(
         "https://gui-beta-dandiarchive-org.netlify.app",
         None,
-        "https://api.dandiarchive.org/api",
+        None,
     ),
     "dandi-api-local-docker-tests": dandi_instance(
-        1, None, None, None, f"http://{instancehost}:8000/api"
+        None, None, f"http://{instancehost}:8000/api"
     ),
 }
 # to map back url: name
 known_instances_rev = {vv: k for k, v in known_instances.items() for vv in v if vv}
 
-collection_drafts = "drafts"
-collection_releases = "releases"
-
 file_operation_modes = [
     "dry",
     "simulate",

dandi/dandiapi.py

@@ -10,8 +10,8 @@
 import tenacity
 
 from .consts import MAX_CHUNK_SIZE, known_instances_rev
-from .girder import keyring_lookup
 from . import get_logger
+from .keyring import keyring_lookup
 from .utils import USER_AGENT, try_multiple
 
 lgr = get_logger()