debugging - why is test_list_private_datasets failing?

pinecone_datasets/catalog.py

@@ -72,7 +72,8 @@ def load(**kwargs) -> "Catalog":
             )
         collected_datasets = []
         try:
-            metadata_files = fs.cat(public_datasets_base_path + "/*/metadata.json")
+            prefix = "gs" if isinstance(fs, gcsfs.GCSFileSystem) else "s3"
+            metadata_files = fs.cat(prefix + "://" + public_datasets_base_path + "/*/metadata.json")
             for path, value in metadata_files.items():
                 name = path.split("/")[1]
                 try:

pinecone_datasets/fs.py

@@ -21,6 +21,7 @@ def get_cloud_fs(
         fs: Union[gcsfs.GCSFileSystem, s3fs.S3FileSystem] - the filesystem object
     """
     is_anon = path == cfg.Storage.endpoint
+    print(f"get_cloud_fs(path:{path}, kwargs:{kwargs}, is_anon:{is_anon}")
     if path.startswith("gs://") or "storage.googleapis.com" in path:
         fs = gcsfs.GCSFileSystem(token="anon" if is_anon else None, **kwargs)
     elif path.startswith("s3://") or "s3.amazonaws.com" in path:

tests/unit/test_private_datasets.py

@@ -6,6 +6,8 @@
 
 def test_list_private_datasets():
     os.environ["DATASETS_CATALOG_BASEPATH"] = "s3://ram-datasets"
+    os.environ["S3FS_LOGGING_LEVEL"] = "DEBUG"
+    os.environ["GCSFS_DEBUG"] = "DEBUG"
     lst = list_datasets(endpoint_url="https://storage.googleapis.com")
     print(lst)
     del os.environ["DATASETS_CATALOG_BASEPATH"]