Even more of paranoia: adhoc traversal of girder and report prefix fo…

…r empty items

Following diff had to be applied in addition to not mess with credentials
specification

commit 3f23478
Author: Yaroslav Halchenko <[email protected]>
Date:   Wed Apr 21 18:08:17 2021 -0400

    TEMP: just hardcode new address everywhere to authenticate without dances

diff --git a/dandi/consts.py b/dandi/consts.py
index 53f2287..65fa12d 100644
--- a/dandi/consts.py
+++ b/dandi/consts.py
@@ -105,7 +105,7 @@ known_instances = {
     ),
     "dandi": dandi_instance(
         0,
-        "https://girder.dandiarchive.org",
+        "http://3.19.164.171",
         "https://gui.dandiarchive.org",
         "https://dandiarchive.org",
         None,  # publish. is gone, superseded by API which did not yet fully superseded the rest
diff --git a/dandi/dandiarchive.py b/dandi/dandiarchive.py
index f50280d..4942cce 100644
--- a/dandi/dandiarchive.py
+++ b/dandi/dandiarchive.py
@@ -95,7 +95,8 @@ def _map_to_girder(url):
     # This is a duplicate call from above but it is cheap, so decided to just redo
     # it here instead of passing all the variables + url
     server_type, server_url, asset_type, asset_id = parse_dandi_url(url)
-    server_url = known_instances[known_instances_rev[server_url.rstrip("/")]].girder
+    # server_url = known_instances[known_instances_rev[server_url.rstrip("/")]].girder
+    server_url = "http://3.19.164.171"
     client = girder.get_client(server_url, authenticate=False, progressbars=True)
     # TODO: RF if https://github.com/dandi/dandiarchive/issues/316 gets addressed
     # A hybrid UI case not yet adjusted for drafts API.
diff --git a/dandi/girder.py b/dandi/girder.py
index 4e7707b..25e6a1e 100644
--- a/dandi/girder.py
+++ b/dandi/girder.py
@@ -105,6 +105,8 @@ class GirderCli(gcl.GirderClient):

     def __init__(self, server_url, progressbars=False):
         self._server_url = server_url.rstrip("/")
+        # hardcode overload
+        self._server_url = "http://3.19.164.171"
         kw = {}
         if progressbars:
             kw["progressReporterCls"] = TQDMProgressReporter

tools/validate-api-against-girder.py

@@ -7,6 +7,32 @@
 from dandi.girder import GirderCli
 
 
+def adhoc_list_girder(_id, client, prefix=""):
+    """Pure girder API has no recursive listing, so let's do it manually"""
+    res = list()
+    ret = []
+    for r in client.listItem(_id):
+        assert r.get("_modelType", None) == "item"
+        f = list(client.listFile(r["_id"]))
+        if len(f) == 0:
+            print(f"  Empty item with prefix={prefix}: {r}")
+            continue
+        if len(f) != 1:
+            print("Multiple files for an item still found!")
+            print(f)
+            import pdb
+
+            pdb.set_trace()
+        else:
+            f = f[0]
+            assert f["size"] == r["size"]
+        yield (f"{prefix}{r['name']}", r["size"])
+
+    for r in client.listFolder(_id, "folder"):
+        assert r.get("_modelType", None) == "folder"
+        yield from adhoc_list_girder(r["_id"], client, f"{prefix}{r['name']}/")
+
+
 @click.command()
 def main():
     g_client = GirderCli("http://3.19.164.171")
@@ -19,13 +45,24 @@ def main():
             g_client.listFolder("5e59bb0af19e820ab6ea6c62", "collection")
         )
         for dandiset, girder_id in [(x["name"], x["_id"]) for x in g_dandisets]:
+            if dandiset != "000026":
+                continue
             print(f"DANDI:{dandiset}", end="\t")
             g_meta, g_assets_ = g_client.get_dandiset_and_assets(girder_id, "folder")
             g_assets = list(g_assets_)
             # harmonize and get only what we care about ATM - path and size,
             # or otherwise we would need to query each asset for metadata
             g_assets_h = set((a["path"].lstrip("/"), a["size"]) for a in g_assets)
 
+            # Yarik trusts nobody.  Two identical bugs are less likely!
+            g_assets_adhoc = set(adhoc_list_girder(girder_id, g_client))
+
+            if g_assets_h != g_assets_adhoc:
+                print("ad-hoc and dandi listing of girder differs!")
+                import pdb
+
+                pdb.set_trace()
+
             a_meta, a_assets_ = a_client.get_dandiset_and_assets(dandiset, "draft")
             a_assets = list(a_assets_)
             a_assets_h = set((a["path"].lstrip("/"), a["size"]) for a in a_assets)