Merge pull request #8 from usnistgov/feature/dbio-exp-search

DBIO: tweak search API

docker/midasserver/mongo/docker-compose.mongo.yml

@@ -26,5 +26,5 @@ services:
       - mongodb
     environment:
       ME_CONFIG_MONGODB_ADMINUSERNAME: ${OAR_MONGODB_ADMIN_USER}
-      ME_CONFIG_MONGODB_ADMINUSERNAME: ${OAR_MONGODB_ADMIN_PASS}
+      ME_CONFIG_MONGODB_ADMINUSERPASS: ${OAR_MONGODB_ADMIN_PASS}
       ME_CONFIG_MONGODB_URL: mongodb://${OAR_MONGODB_ADMIN_USER}:${OAR_MONGODB_ADMIN_PASS}@mongodb:27017/

python/nistoar/midas/dap/service/mds3.py

@@ -2372,13 +2372,13 @@ def __init__(self, service: ProjectService, subapp: SubApp, wsgienv: dict, start
         if hasattr(service, '_fmcli'):
             self._fmcli = service._fmcli
 
-    def _select_records(self, perms) -> Iterator[ProjectRecord]:
+    def _select_records(self, perms, **constraints) -> Iterator[ProjectRecord]:
         """
         submit a search query in a project specific way.  This implementation ensures that 
         DAPProjectRecords are returned.
         :return:  an iterator for the matched records
         """
-        for rec in self._dbcli.select_records(perms):
+        for rec in self._dbcli.select_records(perms, **constraints):
             yield to_DAPRec(rec, self._fmcli)
 
     def _adv_selected_records(self, filter, perms) -> Iterator[ProjectRecord]:

python/nistoar/midas/dbio/base.py

@@ -636,8 +636,7 @@ def name_exists(self, name: str, owner: str = None) -> bool:
         """
         if not owner:
             owner = self._cli.user_id
-        it = self._cli._select_from_coll(
-            GROUPS_COLL, incl_deact=True, name=name, owner=owner)
+        it = self._cli._select_from_coll(GROUPS_COLL, incl_deact=True, name=name, owner=owner)
         try:
             return bool(next(it))
         except StopIteration:
@@ -668,8 +667,7 @@ def get_by_name(self, name: str, owner: str = None) -> Group:
         """
         if not owner:
             owner = self._cli.user_id
-        matches = self._cli._select_from_coll(
-            GROUPS_COLL, incl_deact=True, name=name, owner=owner)
+        matches = self._cli._select_from_coll(GROUPS_COLL, incl_deact=True, name=name, owner=owner)
         for m in matches:
             m = Group(m, self._cli)
             if m.authorized(ACLs.READ):
@@ -683,17 +681,15 @@ def select_ids_for_user(self, id: str) -> MutableSet:
         member of another group.  Deactivated groups are not included.
         """
         checked = set()
-        out = set(g['id'] for g in self._cli._select_prop_contains(
-            GROUPS_COLL, 'members', id))
+        out = set(g['id'] for g in self._cli._select_prop_contains(GROUPS_COLL, 'members', id))
 
         follow = list(out)
         while len(follow) > 0:
-            g = follow.pop(0)
-            if g not in checked:
-                add = set(g['id'] for g in self._cli._select_prop_contains(
-                    GROUPS_COLL, 'members', g))
+            gg = follow.pop(0)
+            if gg not in checked:
+                add = set(g['id'] for g in self._cli._select_prop_contains(GROUPS_COLL, 'members', gg))
                 out |= add
-                checked.add(g)
+                checked.add(gg)
                 follow.extend(add.difference(checked))
 
         out.add(PUBLIC_GROUP)
@@ -981,8 +977,7 @@ def name_exists(self, name: str, owner: str = None) -> bool:
         """
         if not owner:
             owner = self.user_id
-        it = self._select_from_coll(
-            self._projcoll, incl_deact=True, name=name, owner=owner)
+        it = self._select_from_coll(self._projcoll, incl_deact=True, name=name, owner=owner)
         try:
             return bool(next(it))
         except StopIteration:
@@ -995,8 +990,7 @@ def get_record_by_name(self, name: str, owner: str = None) -> Group:
         """
         if not owner:
             owner = self.user_id
-        matches = self._select_from_coll(
-            self._projcoll, incl_deact=True, name=name, owner=owner)
+        matches = self._select_from_coll(self._projcoll, incl_deact=True, name=name, owner=owner)
         for m in matches:
             m = ProjectRecord(self._projcoll, m, self)
             if m.authorized(ACLs.READ):
@@ -1023,13 +1017,15 @@ def get_record_for(self, id: str, perm: str = ACLs.READ) -> ProjectRecord:
         if not out.authorized(perm):
             raise NotAuthorized(self.user_id, perm)
         return out
-
-    def check_query_structure(query):
+
+    @classmethod
+    def check_query_structure(cls, query):
         if not isinstance(query, dict):
             return False
 
-        valid_operators = ['$and', '$or', '$not', '$nor', '$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin', '$exists', '$type', '$expr', '$jsonSchema', '$mod', '$regex', '$text',
-                            '$where', '$geoIntersects', '$geoWithin', '$near', '$nearSphere', '$all', '$elemMatch', '$size', '$bitsAllClear', '$bitsAllSet', '$bitsAnyClear', '$bitsAnySet', '$comment', '$meta']
+        valid_operators = ['$and', '$or', '$not', '$nor', '$eq', '$ne', '$gt', '$gte', '$lt',
+                           '$lte', '$in', '$nin', '$exists', '$type', '$mod', '$regex', '$text',
+                           '$all', '$elemMatch', '$size']
 
         for key in query.keys():
             if key not in valid_operators:
@@ -1056,7 +1052,7 @@ def select_records(self, perm: Permissions = ACLs.OWN, **constraints) -> Iterato
         raise NotImplementedError()
 
     @abstractmethod
-    def select_constraint_records(self,filter:dict, perm: Permissions = ACLs.OWN) -> Iterator[ProjectRecord]:
+    def adv_select_records(self, filter: Mapping, perm: Permissions = ACLs.OWN) -> Iterator[ProjectRecord]:
         """
         return an iterator of project records for which the given user has at least one of the
         permissions and the records meet all the constraints given
@@ -1069,7 +1065,6 @@ def select_constraint_records(self,filter:dict, perm: Permissions = ACLs.OWN) ->
         :param str       **cst: a json that describes all the constraints the records should meet. 
                                 the schema of this json is the query structure used by mongodb.
         """
-        print("BASE PYY")
         raise NotImplementedError()
 
     def is_connected(self) -> bool:

python/nistoar/midas/dbio/fsbased.py

@@ -128,7 +128,7 @@ def _upsert(self, coll: str, recdata: Mapping) -> bool:
         except KeyError:
             raise base.DBIOException("_upsert(): record is missing 'id' property")
 
-    def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.ProjectRecord]:
+    def select_records(self, perm: base.Permissions=base.ACLs.OWN, **cnsts) -> Iterator[base.ProjectRecord]:
         if isinstance(perm, str):
             perm = [perm]
         if isinstance(perm, (list, tuple)):
@@ -152,8 +152,8 @@ def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.
                         yield rec
                         break
 
-    def select_constraint_records(self, perm: base.Permissions = base.ACLs.OWN,
-                                  **cst) -> Iterator[base.ProjectRecord]:
+    def adv_select_records(self, perm: base.Permissions = base.ACLs.OWN,
+                           **cst) -> Iterator[base.ProjectRecord]:
         raise NotImplementedError()
 
     def _save_action_data(self, actdata: Mapping):

python/nistoar/midas/dbio/inmem.py

@@ -69,7 +69,7 @@ def _upsert(self, coll: str, recdata: Mapping) -> bool:
         self._db[coll][recdata['id']] = deepcopy(recdata)
         return not exists
 
-    def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.ProjectRecord]:
+    def select_records(self, perm: base.Permissions=base.ACLs.OWN, **cnsts) -> Iterator[base.ProjectRecord]:
         if isinstance(perm, str):
             perm = [perm]
         if isinstance(perm, (list, tuple)):
@@ -81,7 +81,8 @@ def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.
                     yield deepcopy(rec)
                     break
 
-    def select_constraint_records(self,filter:dict,perm: base.Permissions=base.ACLs.OWN,) -> Iterator[base.ProjectRecord]:
+    def adv_select_records(self, filter:dict,
+                           perm: base.Permissions=base.ACLs.OWN,) -> Iterator[base.ProjectRecord]:
         if(base.DBClient.check_query_structure(filter) == True):
             try:
                 if isinstance(perm, str):
@@ -164,4 +165,4 @@ def create_client(self, servicetype: str, config: Mapping={}, foruser: str = bas
         if servicetype not in self._db:
             self._db[servicetype] = {}
         return InMemoryDBClient(self._db, cfg, servicetype, foruser)
-
+

python/nistoar/midas/dbio/mongo.py

@@ -182,7 +182,7 @@ def _delete_from(self, collname, id):
             raise base.DBIOException("Failed while deleting record with id=%s: %s" % (id, str(ex)))
 
 
-    def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.ProjectRecord]:
+    def select_records(self, perm: base.Permissions=base.ACLs.OWN, **cnsts) -> Iterator[base.ProjectRecord]:
         if isinstance(perm, str):
             perm = [perm]
         if isinstance(perm, (list, tuple)):
@@ -199,14 +199,15 @@ def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.
             coll = self.native[self._projcoll]
 
             for rec in coll.find(constraints, {'_id': False}):
-                yield base.ProjectRecord(self._projcoll, rec)
+                yield base.ProjectRecord(self._projcoll, rec, self)
 
         except Exception as ex:
             raise base.DBIOException("Failed while selecting records: " + str(ex), cause=ex)
 
 
-    def select_constraint_records(self,filter:dict, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.ProjectRecord]:
-        if(base.DBClient.check_query_structure(filter) == True):
+    def adv_select_records(self, filter: dict,
+                           perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.ProjectRecord]:
+        if base.DBClient.check_query_structure(filter):
             if isinstance(perm, str):
                 perm = [perm]
             if isinstance(perm, (list, tuple)):
@@ -226,7 +227,7 @@ def select_constraint_records(self,filter:dict, perm: base.Permissions=base.ACLs
 
                 coll = self.native[self._projcoll]
                 for rec in coll.find(filter):
-                    yield base.ProjectRecord(self._projcoll, rec)
+                    yield base.ProjectRecord(self._projcoll, rec, self)
 
             except Exception as ex:
                 raise base.DBIOException(

python/nistoar/midas/dbio/wsgi/project.py

@@ -430,14 +430,14 @@ def __init__(self, service: ProjectService, subapp: SubApp, wsgienv: dict, start
     def do_OPTIONS(self, path):
         return self.send_options(["GET", "POST"])
 
-    def _select_records(self, perms) -> Iterator[ProjectRecord]:
+    def _select_records(self, perms, **constraints) -> Iterator[ProjectRecord]:
         """
         submit a search query in a project specific way.  This method is provided as a 
         hook to subclasses that may need to specialize the search strategy or manipulate the results.  
         This implementation passes the query directly to the generic DBClient instance.
         :return:  a generator that iterates through the matched records
         """
-        return self._dbcli.select_records(perms)
+        return self._dbcli.select_records(perms, **constraints)
 
     def do_GET(self, path, ashead=False):
         """
@@ -516,7 +516,7 @@ def _adv_select_records(self, filter, perms) -> Iterator[ProjectRecord]:
         This base implementation passes the query directly to the generic DBClient instance.
         :return:  a generator that iterates through the matched records
         """
-        return self._dbcli.select_constraint_records(filter, perms)
+        return self._dbcli.adv_select_records(filter, perms)
 
     def create_record(self, newdata: Mapping):
         """

python/tests/nistoar/midas/dbio/test_inmem.py

@@ -257,7 +257,7 @@ def test_upsert(self):
         rec2 = self.cli._get_from_coll(base.GROUPS_COLL, "p:bob")
         self.assertEqual(rec2, {"id": "p:bob", "members": ["p:bob", "alice"]})
 
-    def test_select_constraint_records(self):
+    def test_adv_select_records(self):
         # inject some data into the database
         id = "pdr0:0002"
         rec = base.ProjectRecord(
@@ -313,17 +313,17 @@ def test_select_constraint_records(self):
         constraint_wrong = {'$a,nkd': [
             {'$okn,r': [{'name': 'test 2'}, {'name': 'test3'}]}]}
         with self.assertRaises(SyntaxError) as context:
-            recs = list(self.cli.select_constraint_records(constraint_wrong,base.ACLs.READ))
+            recs = list(self.cli.adv_select_records(constraint_wrong,base.ACLs.READ))
         self.assertEqual(str(context.exception), "Wrong query format")
-        recs = list(self.cli.select_constraint_records(constraint_or))
+        recs = list(self.cli.adv_select_records(constraint_or))
         self.assertEqual(len(recs), 2)
         self.assertEqual(recs[0].id, "pdr0:0006")
         self.assertEqual(recs[1].id, "pdr0:0003")
 
-        recs = list(self.cli.select_constraint_records(constraint_and))
+        recs = list(self.cli.adv_select_records(constraint_and))
         self.assertEqual(len(recs), 1)
         self.assertEqual(recs[0].id, "pdr0:0003")
-        recs = list(self.cli.select_constraint_records(constraint_andor,base.ACLs.READ))
+        recs = list(self.cli.adv_select_records(constraint_andor,base.ACLs.READ))
         self.assertEqual(len(recs), 2)
         self.assertEqual(recs[0].id, "pdr0:0006")
         self.assertEqual(recs[1].id, "pdr0:0003")

python/tests/nistoar/midas/dbio/test_mongo.py

@@ -313,7 +313,7 @@ def test_select_records(self):
         self.assertTrue(isinstance(recs[0], base.ProjectRecord))
         self.assertEqual(recs[1].id, id)
 
-    def test_select_constraint_records(self):
+    def test_adv_select_records(self):
 
         # inject some data into the database
 
@@ -369,17 +369,17 @@ def test_select_constraint_records(self):
         constraint_wrong = {'$a,nkd': [
             {'$okn,r': [{'name': 'test 2'}, {'name': 'test3'}]}]}
         with self.assertRaises(SyntaxError) as context:
-            recs = list(self.cli.select_constraint_records(**constraint_wrong))
+            recs = list(self.cli.adv_select_records(constraint_wrong))
         self.assertEqual(str(context.exception), "Wrong query format")
-        recs = list(self.cli.select_constraint_records(base.ACLs.READ,**constraint_or))
+        recs = list(self.cli.adv_select_records(constraint_or, base.ACLs.READ))
         self.assertEqual(len(recs), 2)
         self.assertEqual(recs[0].id, "pdr0:0006")
         self.assertEqual(recs[1].id, "pdr0:0003")
 
-        recs = list(self.cli.select_constraint_records(**constraint_and))
+        recs = list(self.cli.adv_select_records(constraint_and))
         self.assertEqual(len(recs), 1)
         self.assertEqual(recs[0].id, "pdr0:0003")
-        recs = list(self.cli.select_constraint_records(base.ACLs.READ,**constraint_andor))
+        recs = list(self.cli.adv_select_records(constraint_andor, base.ACLs.READ))
         self.assertEqual(len(recs), 2)
         self.assertEqual(recs[0].id, "pdr0:0006")
         self.assertEqual(recs[1].id, "pdr0:0003")