diff --git a/docker/midasserver/mongo/docker-compose.mongo.yml b/docker/midasserver/mongo/docker-compose.mongo.yml index f0f58980..7085aa31 100644 --- a/docker/midasserver/mongo/docker-compose.mongo.yml +++ b/docker/midasserver/mongo/docker-compose.mongo.yml @@ -26,5 +26,5 @@ services: - mongodb environment: ME_CONFIG_MONGODB_ADMINUSERNAME: ${OAR_MONGODB_ADMIN_USER} - ME_CONFIG_MONGODB_ADMINUSERNAME: ${OAR_MONGODB_ADMIN_PASS} + ME_CONFIG_MONGODB_ADMINUSERPASS: ${OAR_MONGODB_ADMIN_PASS} ME_CONFIG_MONGODB_URL: mongodb://${OAR_MONGODB_ADMIN_USER}:${OAR_MONGODB_ADMIN_PASS}@mongodb:27017/ diff --git a/python/nistoar/midas/dap/service/mds3.py b/python/nistoar/midas/dap/service/mds3.py index e7664af0..350ab917 100644 --- a/python/nistoar/midas/dap/service/mds3.py +++ b/python/nistoar/midas/dap/service/mds3.py @@ -2372,13 +2372,13 @@ def __init__(self, service: ProjectService, subapp: SubApp, wsgienv: dict, start if hasattr(service, '_fmcli'): self._fmcli = service._fmcli - def _select_records(self, perms) -> Iterator[ProjectRecord]: + def _select_records(self, perms, **constraints) -> Iterator[ProjectRecord]: """ submit a search query in a project specific way. This implementation ensures that DAPProjectRecords are returned. :return: an iterator for the matched records """ - for rec in self._dbcli.select_records(perms): + for rec in self._dbcli.select_records(perms, **constraints): yield to_DAPRec(rec, self._fmcli) def _adv_selected_records(self, filter, perms) -> Iterator[ProjectRecord]: diff --git a/python/nistoar/midas/dbio/base.py b/python/nistoar/midas/dbio/base.py index a4f8f854..47fcf0a7 100644 --- a/python/nistoar/midas/dbio/base.py +++ b/python/nistoar/midas/dbio/base.py @@ -636,8 +636,7 @@ def name_exists(self, name: str, owner: str = None) -> bool: """ if not owner: owner = self._cli.user_id - it = self._cli._select_from_coll( - GROUPS_COLL, incl_deact=True, name=name, owner=owner) + it = self._cli._select_from_coll(GROUPS_COLL, incl_deact=True, name=name, owner=owner) try: return bool(next(it)) except StopIteration: @@ -668,8 +667,7 @@ def get_by_name(self, name: str, owner: str = None) -> Group: """ if not owner: owner = self._cli.user_id - matches = self._cli._select_from_coll( - GROUPS_COLL, incl_deact=True, name=name, owner=owner) + matches = self._cli._select_from_coll(GROUPS_COLL, incl_deact=True, name=name, owner=owner) for m in matches: m = Group(m, self._cli) if m.authorized(ACLs.READ): @@ -683,17 +681,15 @@ def select_ids_for_user(self, id: str) -> MutableSet: member of another group. Deactivated groups are not included. """ checked = set() - out = set(g['id'] for g in self._cli._select_prop_contains( - GROUPS_COLL, 'members', id)) + out = set(g['id'] for g in self._cli._select_prop_contains(GROUPS_COLL, 'members', id)) follow = list(out) while len(follow) > 0: - g = follow.pop(0) - if g not in checked: - add = set(g['id'] for g in self._cli._select_prop_contains( - GROUPS_COLL, 'members', g)) + gg = follow.pop(0) + if gg not in checked: + add = set(g['id'] for g in self._cli._select_prop_contains(GROUPS_COLL, 'members', gg)) out |= add - checked.add(g) + checked.add(gg) follow.extend(add.difference(checked)) out.add(PUBLIC_GROUP) @@ -981,8 +977,7 @@ def name_exists(self, name: str, owner: str = None) -> bool: """ if not owner: owner = self.user_id - it = self._select_from_coll( - self._projcoll, incl_deact=True, name=name, owner=owner) + it = self._select_from_coll(self._projcoll, incl_deact=True, name=name, owner=owner) try: return bool(next(it)) except StopIteration: @@ -995,8 +990,7 @@ def get_record_by_name(self, name: str, owner: str = None) -> Group: """ if not owner: owner = self.user_id - matches = self._select_from_coll( - self._projcoll, incl_deact=True, name=name, owner=owner) + matches = self._select_from_coll(self._projcoll, incl_deact=True, name=name, owner=owner) for m in matches: m = ProjectRecord(self._projcoll, m, self) if m.authorized(ACLs.READ): @@ -1023,13 +1017,15 @@ def get_record_for(self, id: str, perm: str = ACLs.READ) -> ProjectRecord: if not out.authorized(perm): raise NotAuthorized(self.user_id, perm) return out - - def check_query_structure(query): + + @classmethod + def check_query_structure(cls, query): if not isinstance(query, dict): return False - valid_operators = ['$and', '$or', '$not', '$nor', '$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin', '$exists', '$type', '$expr', '$jsonSchema', '$mod', '$regex', '$text', - '$where', '$geoIntersects', '$geoWithin', '$near', '$nearSphere', '$all', '$elemMatch', '$size', '$bitsAllClear', '$bitsAllSet', '$bitsAnyClear', '$bitsAnySet', '$comment', '$meta'] + valid_operators = ['$and', '$or', '$not', '$nor', '$eq', '$ne', '$gt', '$gte', '$lt', + '$lte', '$in', '$nin', '$exists', '$type', '$mod', '$regex', '$text', + '$all', '$elemMatch', '$size'] for key in query.keys(): if key not in valid_operators: @@ -1056,7 +1052,7 @@ def select_records(self, perm: Permissions = ACLs.OWN, **constraints) -> Iterato raise NotImplementedError() @abstractmethod - def select_constraint_records(self,filter:dict, perm: Permissions = ACLs.OWN) -> Iterator[ProjectRecord]: + def adv_select_records(self, filter: Mapping, perm: Permissions = ACLs.OWN) -> Iterator[ProjectRecord]: """ return an iterator of project records for which the given user has at least one of the permissions and the records meet all the constraints given @@ -1069,7 +1065,6 @@ def select_constraint_records(self,filter:dict, perm: Permissions = ACLs.OWN) -> :param str **cst: a json that describes all the constraints the records should meet. the schema of this json is the query structure used by mongodb. """ - print("BASE PYY") raise NotImplementedError() def is_connected(self) -> bool: diff --git a/python/nistoar/midas/dbio/fsbased.py b/python/nistoar/midas/dbio/fsbased.py index 2561b782..b3657c88 100644 --- a/python/nistoar/midas/dbio/fsbased.py +++ b/python/nistoar/midas/dbio/fsbased.py @@ -128,7 +128,7 @@ def _upsert(self, coll: str, recdata: Mapping) -> bool: except KeyError: raise base.DBIOException("_upsert(): record is missing 'id' property") - def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.ProjectRecord]: + def select_records(self, perm: base.Permissions=base.ACLs.OWN, **cnsts) -> Iterator[base.ProjectRecord]: if isinstance(perm, str): perm = [perm] if isinstance(perm, (list, tuple)): @@ -152,8 +152,8 @@ def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base. yield rec break - def select_constraint_records(self, perm: base.Permissions = base.ACLs.OWN, - **cst) -> Iterator[base.ProjectRecord]: + def adv_select_records(self, perm: base.Permissions = base.ACLs.OWN, + **cst) -> Iterator[base.ProjectRecord]: raise NotImplementedError() def _save_action_data(self, actdata: Mapping): diff --git a/python/nistoar/midas/dbio/inmem.py b/python/nistoar/midas/dbio/inmem.py index 5a6e5e6f..717238f9 100644 --- a/python/nistoar/midas/dbio/inmem.py +++ b/python/nistoar/midas/dbio/inmem.py @@ -69,7 +69,7 @@ def _upsert(self, coll: str, recdata: Mapping) -> bool: self._db[coll][recdata['id']] = deepcopy(recdata) return not exists - def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.ProjectRecord]: + def select_records(self, perm: base.Permissions=base.ACLs.OWN, **cnsts) -> Iterator[base.ProjectRecord]: if isinstance(perm, str): perm = [perm] if isinstance(perm, (list, tuple)): @@ -81,7 +81,8 @@ def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base. yield deepcopy(rec) break - def select_constraint_records(self,filter:dict,perm: base.Permissions=base.ACLs.OWN,) -> Iterator[base.ProjectRecord]: + def adv_select_records(self, filter:dict, + perm: base.Permissions=base.ACLs.OWN,) -> Iterator[base.ProjectRecord]: if(base.DBClient.check_query_structure(filter) == True): try: if isinstance(perm, str): @@ -164,4 +165,4 @@ def create_client(self, servicetype: str, config: Mapping={}, foruser: str = bas if servicetype not in self._db: self._db[servicetype] = {} return InMemoryDBClient(self._db, cfg, servicetype, foruser) - \ No newline at end of file + diff --git a/python/nistoar/midas/dbio/mongo.py b/python/nistoar/midas/dbio/mongo.py index 890b48bb..d6cc7c28 100644 --- a/python/nistoar/midas/dbio/mongo.py +++ b/python/nistoar/midas/dbio/mongo.py @@ -182,7 +182,7 @@ def _delete_from(self, collname, id): raise base.DBIOException("Failed while deleting record with id=%s: %s" % (id, str(ex))) - def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.ProjectRecord]: + def select_records(self, perm: base.Permissions=base.ACLs.OWN, **cnsts) -> Iterator[base.ProjectRecord]: if isinstance(perm, str): perm = [perm] if isinstance(perm, (list, tuple)): @@ -199,14 +199,15 @@ def select_records(self, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base. coll = self.native[self._projcoll] for rec in coll.find(constraints, {'_id': False}): - yield base.ProjectRecord(self._projcoll, rec) + yield base.ProjectRecord(self._projcoll, rec, self) except Exception as ex: raise base.DBIOException("Failed while selecting records: " + str(ex), cause=ex) - def select_constraint_records(self,filter:dict, perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.ProjectRecord]: - if(base.DBClient.check_query_structure(filter) == True): + def adv_select_records(self, filter: dict, + perm: base.Permissions=base.ACLs.OWN) -> Iterator[base.ProjectRecord]: + if base.DBClient.check_query_structure(filter): if isinstance(perm, str): perm = [perm] if isinstance(perm, (list, tuple)): @@ -226,7 +227,7 @@ def select_constraint_records(self,filter:dict, perm: base.Permissions=base.ACLs coll = self.native[self._projcoll] for rec in coll.find(filter): - yield base.ProjectRecord(self._projcoll, rec) + yield base.ProjectRecord(self._projcoll, rec, self) except Exception as ex: raise base.DBIOException( diff --git a/python/nistoar/midas/dbio/wsgi/project.py b/python/nistoar/midas/dbio/wsgi/project.py index c144b029..6aba08bd 100644 --- a/python/nistoar/midas/dbio/wsgi/project.py +++ b/python/nistoar/midas/dbio/wsgi/project.py @@ -430,14 +430,14 @@ def __init__(self, service: ProjectService, subapp: SubApp, wsgienv: dict, start def do_OPTIONS(self, path): return self.send_options(["GET", "POST"]) - def _select_records(self, perms) -> Iterator[ProjectRecord]: + def _select_records(self, perms, **constraints) -> Iterator[ProjectRecord]: """ submit a search query in a project specific way. This method is provided as a hook to subclasses that may need to specialize the search strategy or manipulate the results. This implementation passes the query directly to the generic DBClient instance. :return: a generator that iterates through the matched records """ - return self._dbcli.select_records(perms) + return self._dbcli.select_records(perms, **constraints) def do_GET(self, path, ashead=False): """ @@ -516,7 +516,7 @@ def _adv_select_records(self, filter, perms) -> Iterator[ProjectRecord]: This base implementation passes the query directly to the generic DBClient instance. :return: a generator that iterates through the matched records """ - return self._dbcli.select_constraint_records(filter, perms) + return self._dbcli.adv_select_records(filter, perms) def create_record(self, newdata: Mapping): """ diff --git a/python/tests/nistoar/midas/dbio/test_inmem.py b/python/tests/nistoar/midas/dbio/test_inmem.py index 068299b1..f01a13e3 100644 --- a/python/tests/nistoar/midas/dbio/test_inmem.py +++ b/python/tests/nistoar/midas/dbio/test_inmem.py @@ -257,7 +257,7 @@ def test_upsert(self): rec2 = self.cli._get_from_coll(base.GROUPS_COLL, "p:bob") self.assertEqual(rec2, {"id": "p:bob", "members": ["p:bob", "alice"]}) - def test_select_constraint_records(self): + def test_adv_select_records(self): # inject some data into the database id = "pdr0:0002" rec = base.ProjectRecord( @@ -313,17 +313,17 @@ def test_select_constraint_records(self): constraint_wrong = {'$a,nkd': [ {'$okn,r': [{'name': 'test 2'}, {'name': 'test3'}]}]} with self.assertRaises(SyntaxError) as context: - recs = list(self.cli.select_constraint_records(constraint_wrong,base.ACLs.READ)) + recs = list(self.cli.adv_select_records(constraint_wrong,base.ACLs.READ)) self.assertEqual(str(context.exception), "Wrong query format") - recs = list(self.cli.select_constraint_records(constraint_or)) + recs = list(self.cli.adv_select_records(constraint_or)) self.assertEqual(len(recs), 2) self.assertEqual(recs[0].id, "pdr0:0006") self.assertEqual(recs[1].id, "pdr0:0003") - recs = list(self.cli.select_constraint_records(constraint_and)) + recs = list(self.cli.adv_select_records(constraint_and)) self.assertEqual(len(recs), 1) self.assertEqual(recs[0].id, "pdr0:0003") - recs = list(self.cli.select_constraint_records(constraint_andor,base.ACLs.READ)) + recs = list(self.cli.adv_select_records(constraint_andor,base.ACLs.READ)) self.assertEqual(len(recs), 2) self.assertEqual(recs[0].id, "pdr0:0006") self.assertEqual(recs[1].id, "pdr0:0003") diff --git a/python/tests/nistoar/midas/dbio/test_mongo.py b/python/tests/nistoar/midas/dbio/test_mongo.py index 6dcfd723..294b12e1 100644 --- a/python/tests/nistoar/midas/dbio/test_mongo.py +++ b/python/tests/nistoar/midas/dbio/test_mongo.py @@ -313,7 +313,7 @@ def test_select_records(self): self.assertTrue(isinstance(recs[0], base.ProjectRecord)) self.assertEqual(recs[1].id, id) - def test_select_constraint_records(self): + def test_adv_select_records(self): # inject some data into the database @@ -369,17 +369,17 @@ def test_select_constraint_records(self): constraint_wrong = {'$a,nkd': [ {'$okn,r': [{'name': 'test 2'}, {'name': 'test3'}]}]} with self.assertRaises(SyntaxError) as context: - recs = list(self.cli.select_constraint_records(**constraint_wrong)) + recs = list(self.cli.adv_select_records(constraint_wrong)) self.assertEqual(str(context.exception), "Wrong query format") - recs = list(self.cli.select_constraint_records(base.ACLs.READ,**constraint_or)) + recs = list(self.cli.adv_select_records(constraint_or, base.ACLs.READ)) self.assertEqual(len(recs), 2) self.assertEqual(recs[0].id, "pdr0:0006") self.assertEqual(recs[1].id, "pdr0:0003") - recs = list(self.cli.select_constraint_records(**constraint_and)) + recs = list(self.cli.adv_select_records(constraint_and)) self.assertEqual(len(recs), 1) self.assertEqual(recs[0].id, "pdr0:0003") - recs = list(self.cli.select_constraint_records(base.ACLs.READ,**constraint_andor)) + recs = list(self.cli.adv_select_records(constraint_andor, base.ACLs.READ)) self.assertEqual(len(recs), 2) self.assertEqual(recs[0].id, "pdr0:0006") self.assertEqual(recs[1].id, "pdr0:0003")