From 9775e3b13892489f362e6e1967a024a200492d25 Mon Sep 17 00:00:00 2001 From: arilieb Date: Wed, 29 Jan 2025 12:39:06 -0800 Subject: [PATCH] uwes Refactor (#931) * Unverefied event indexed escrowed couples (uwes) database refactor to use B64OnIoDupSuber. Working, needs cleanup and refactor of two more tests. * Fixed tests that were breaking with invalid type errors. Cleaned up commented code and made slight revisions to documentation to reflect changes. --- src/keri/core/eventing.py | 132 ++++++++++++++++++-------------------- src/keri/db/basing.py | 106 +----------------------------- tests/db/test_basing.py | 83 ++++++++++++------------ 3 files changed, 107 insertions(+), 214 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 24bd4708..3bb45e9b 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5156,8 +5156,8 @@ def escrowUWReceipt(self, serder, wigers, said): # don't know witness pre yet without witness list so no verfer in wiger # if wiger.verfer.transferable: # skip transferable verfers # continue # skip invalid triplets - couple = said.encode("utf-8") + wiger.qb64b - self.db.addUwe(key=snKey(serder.preb, serder.sn), val=couple) + self.db.uwes.addOn(keys=serder.preb, on=serder.sn, val=(said, wiger.qb64)) + # log escrowed logger.debug("Kevery process: escrowed unverified witness indexed receipt" " of pre= %s sn=%x dig=%s", serder.pre, serder.sn, said) @@ -6017,8 +6017,7 @@ def processEscrowUnverWitness(self): This allows FIFO processing of escrows for events with same prefix and sn but different digest. - Uses .uwes reads .db.getUwe - was put there by.db.addUwe(self, key, val) which is IOVal with dups. + Uses .uwes reads .db.uwes.get() which is B64OnIoDupSuber Value is couple @@ -6044,78 +6043,69 @@ def processEscrowUnverWitness(self): If successful then remove from escrow table """ - ims = bytearray() - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, ecouple in self.db.getUweItemIter(key=key): - try: - pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key - - # get escrowed receipt's rdiger of receipted event and - # wiger indexed signature of receipted event - rdiger, wiger = deWitnessCouple(ecouple) - - # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(rdiger.qb64b))) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", rdiger.qb64b) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(rdiger.qb64b)) - - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutUWE): - # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", rdiger.qb64b) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(rdiger.qb64b)) - - # lookup database dig of the receipted event in pwes escrow - # using pre and sn lastEvt - found = self._processEscrowFindUnver(pre=pre, - sn=sn, - rsaider=rdiger, - wiger=wiger) - - if not found: # no partial witness escrow of event found - # so keep in escrow by raising UnverifiedWitnessReceiptError - logger.debug("Kevery unescrow error: Missing witness " - "receipted evt at pre=%s sn=%x", (pre, sn)) + for (pre, snh), (rdiger, wiger) in self.db.uwes.getItemIter(): + try: + rdigerBytes = rdiger.encode('utf-8') + # check date if expired then remove escrow. + dtb = self.db.getDts(dgKey(pre, bytes(rdigerBytes))) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", rdigerBytes) - raise UnverifiedWitnessReceiptError("Missing witness " - "receipted evt at pre={} sn={:x}".format(pre, sn)) + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(rdigerBytes)) - except UnverifiedWitnessReceiptError as ex: - # still waiting on missing prior event to validate - # only happens if we process above - if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutUWE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", rdiger.qb64b) - except Exception as ex: # log diagnostics errors etc - # error other than out of order so remove from OO escrow - self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val - if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + raise ValidationError("Stale event escrow " + "at dig = {}.".format(rdiger.qb64b)) + + # lookup database dig of the receipted event in pwes escrow + # using pre and sn lastEvt + sn = int(snh, 16) + rdigerClassed = Diger(qb64=rdiger) + wigerClassed = Siger(qb64=wiger) + found = self._processEscrowFindUnver(pre=pre, + sn=sn, + rsaider=rdigerClassed, + wiger=wigerClassed) + + if not found: # no partial witness escrow of event found + # so keep in escrow by raising UnverifiedWitnessReceiptError + logger.debug("Kevery unescrow error: Missing witness " + "receipted evt at pre=%s sn=%x", (pre, sn)) + + raise UnverifiedWitnessReceiptError("Missing witness " + "receipted evt at pre={} sn={:x}".format(pre, sn)) + + except UnverifiedWitnessReceiptError as ex: + # still waiting on missing prior event to validate + # only happens if we process above + if logger.isEnabledFor(logging.DEBUG): # adds exception data + logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val - logger.info("Kevery unescrow succeeded for event pre=%s " - "sn=%s", pre, sn) + except Exception as ex: # log diagnostics errors etc + # error other than out of order so remove from OO escrow + self.db.uwes.rem(keys=(pre, snh), val=(rdiger, wiger)) + if logger.isEnabledFor(logging.DEBUG): # adds exception data + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.uwes.rem(keys=(pre, snh), val=(rdiger, wiger)) + logger.info("Kevery unescrow succeeded for event pre=%s " + "sn=%s", pre, sn) def processEscrowUnverNonTrans(self): """ diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 958a544e..b5f9d977 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -813,7 +813,6 @@ class Baser(dbing.LMDBer): witness nontrans identifier prefix from witness list and index is offset into witness list of latest establishment event for receipted event - snKey DB is keyed by receipted event controller prefix plus sn of serialized event More than one value per DB key is allowed @@ -1021,10 +1020,7 @@ def reopen(self, **kwa): self.pdes = subing.OnIoDupSuber(db=self, subkey='pdes.') self.udes = subing.CatCesrSuber(db=self, subkey='udes.', klas=(coring.Seqner, coring.Saider)) - self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) - #self.uwes = subing.CatCesrIoSetSuber(db=self, subkey='uwes.', - #klas=(coring.Saider, indexing.Siger)) - + self.uwes = subing.B64OnIoDupSuber(db=self, subkey='uwes.') self.ooes = self.env.open_db(key=b'ooes.', dupsort=True) self.dels = self.env.open_db(key=b'dels.', dupsort=True) self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) @@ -1395,15 +1391,14 @@ def clearEscrows(self): self.delPses(key=k) for (k, _) in self.getPweItemIter(): self.delPwes(key=k) - for (k, _) in self.getUweItemIter(): - self.delUwes(key=k) for (k, _) in self.getOoeItemIter(): self.delOoes(key=k) for (k, _) in self.getLdeItemIter(): self.delLdes(key=k) for (pre, said), edig in self.qnfs.getItemIter(): self.qnfs.rem(keys=(pre, said)) - + for (pre, snh), rdigerWigerTuple in self.uwes.getItemIter(): + self.uwes.rem(keys=(pre, snh)) for escrow in [self.qnfs, self.misfits, self.delegables, self.pdes, self.udes, self.rpes, self.epsd, self.eoobi, self.dpub, self.gpwe, self.gdee, self.dpwe, self.gpse, self.epse, self.dune]: @@ -2894,101 +2889,6 @@ def delPwe(self, key, val): """ return self.delIoDupVal(self.pwes, key, val) - def putUwes(self, key, vals): - """ - Use snKey() - Write each entry from list of bytes witness receipt couples vals to key - Witness couple is edig+wig - Adds to existing receipts at key if any - Returns True If at least one of vals is added as dup, False otherwise - Duplicates are inserted in insertion order. - """ - return self.putIoDupVals(self.uwes, key, vals) - - def addUwe(self, key, val): - """ - Use snKey() - Add receipt couple val bytes as dup to key in db - Witness couple is edig+wig - Adds to existing values at key if any - Returns True If at least one of vals is added as dup, False otherwise - Duplicates are inserted in insertion order. - """ - return self.addIoDupVal(self.uwes, key, val) - - def getUwes(self, key): - """ - Use snKey() - Return list of receipt couples at key - Witness couple is edig+wig - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupVals(self.uwes, key) - - def getUwesIter(self, key): - """ - Use snKey() - Return iterator of receipt couples at key - Witness couple is edig+wig - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoDupValsIter(self.uwes, key) - - def getUweLast(self, key): - """ - Use snKey() - Return last inserted dup partial signed escrowed receipt couple val at key - Witness couple is edig+wig - Returns None if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupValLast(self.uwes, key) - - def getUweItemIter(self, key=b''): - """ - Use sgKey() - Return iterator of partial signed escrowed receipt couple items at next - key after key. - Items is (key, val) where proem has already been stripped from val - val is couple edig+wig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getTopIoDupItemIter(self.uwes, key) - #return self.getIoDupItemsNextIter(self.uwes, key, skip) - - def cntUwes(self, key): - """ - Use snKey() - Return count of receipt couples at key - Returns zero if no entry at key - """ - return self.cntIoDupVals(self.uwes, key) - - def delUwes(self, key): - """ - Use snKey() - Deletes all values at key in db. - Returns True If key exists in database Else False - """ - return self.delIoDupVals(self.uwes, key) - - def delUwe(self, key, val): - """ - Use snKey() - Deletes dup val at key in db. - Returns True If dup at exists in db Else False - - Parameters: - key is bytes of key within sub db's keyspace - val is dup val (does not include insertion ordering proem) - """ - return self.delIoDupVal(self.uwes, key, val) - def putOoes(self, key, vals): """ Use snKey() diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index ad4a17ee..cd88511f 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -899,52 +899,52 @@ def test_baser(): # Unverified Witness Receipt Escrows # test .uwes insertion order dup methods. dup vals are insertion order key = b'A' - vals = [b"z", b"m", b"x", b"a"] - - assert db.getUwes(key) == [] - assert db.getUweLast(key) == None - assert db.cntUwes(key) == 0 - assert db.delUwes(key) == False - assert db.putUwes(key, vals) == True - assert db.getUwes(key) == vals # preserved insertion order - assert db.cntUwes(key) == len(vals) == 4 - assert db.getUweLast(key) == vals[-1] - assert db.putUwes(key, vals=[b'a']) == False # duplicate - assert db.getUwes(key) == vals # no change - assert db.addUwe(key, b'a') == False # duplicate - assert db.addUwe(key, b'b') == True - assert db.getUwes(key) == [b"z", b"m", b"x", b"a", b"b"] - assert [val for val in db.getUwesIter(key)] == [b"z", b"m", b"x", b"a", b"b"] - assert db.delUwes(key) == True - assert db.getUwes(key) == [] + vals = [('z',), ('m',), ('x',), ('a',)] + + assert db.uwes.get(key) == [] + assert db.uwes.getLast(key) == None + assert db.uwes.cnt(key) == 0 + assert db.uwes.rem(key) == False + assert db.uwes.put(key, vals) == True + assert db.uwes.get(key) == vals # preserved insertion order + assert db.uwes.cnt(key) == len(vals) == 4 + assert db.uwes.getLast(key) == vals[-1] + assert db.uwes.put(key, vals=[b'a']) == False # duplicate + assert db.uwes.get(key) == vals # no change + assert db.uwes.add(key, b'a') == False # duplicate + assert db.uwes.add(key, b'b') == True + assert db.uwes.get(key) == [('z',), ('m',), ('x',), ('a',), ('b',)] + assert [val for key, val in db.uwes.getItemIter(key)] == [('z',), ('m',), ('x',), ('a',), ('b',)] + assert db.uwes.rem(key) == True + assert db.uwes.get(key) == [] # Setup Tests for getUweItemsNext and getUweItemsNextIter - aKey = snKey(pre=b'A', sn=1) - aVals = [b"z", b"m", b"x"] - bKey = snKey(pre=b'A', sn=2) - bVals = [b"o", b"r", b"z"] - cKey = snKey(pre=b'A', sn=4) - cVals = [b"h", b"n"] - dKey = snKey(pre=b'A', sn=7) - dVals = [b"k", b"b"] + aKey = ('A', '00000000000000000000000000000001') + aVals = [('z',), ('m',), ('x',)] + bKey = ('A', '00000000000000000000000000000002') + bVals = [('o',), ('r',), ('z',)] + cKey = ('A', '00000000000000000000000000000004') + cVals = [('h',), ('n',)] + dKey = ('A', '00000000000000000000000000000007') + dVals = [('k',), ('b',)] - assert db.putUwes(key=aKey, vals=aVals) - assert db.putUwes(key=bKey, vals=bVals) - assert db.putUwes(key=cKey, vals=cVals) - assert db.putUwes(key=dKey, vals=dVals) + assert db.uwes.put(keys=aKey, vals=aVals) + assert db.uwes.put(keys=bKey, vals=bVals) + assert db.uwes.put(keys=cKey, vals=cVals) + assert db.uwes.put(keys=dKey, vals=dVals) # Test getUweItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getUweItemIter()] + items = [item for item in db.uwes.getItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getUweItemIter(key=aKey)] + items = [item for item in db.uwes.getItemIter(keys=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey @@ -952,34 +952,34 @@ def test_baser(): assert vals == aVals # bVals - items = [item for item in db.getUweItemIter(key=bKey)] + items = [item for item in db.uwes.getItemIter(keys=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey vals = [val for key, val in items] assert vals == bVals for key, val in items: - assert db.delUwe(ikey, val) == True + assert db.uwes.rem(ikey, val) == True # cVals - items = [item for item in db.getUweItemIter(key=cKey)] + items = [item for item in db.uwes.getItemIter(keys=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey vals = [val for key, val in items] assert vals == cVals for key, val in items: - assert db.delUwe(ikey, val) == True + assert db.uwes.rem(ikey, val) == True # dVals - items = [item for item in db.getUweItemIter(key=dKey)] + items = [item for item in db.uwes.getItemIter(keys=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey vals = [val for key, val in items] assert vals == dVals for key, val in items: - assert db.delUwe(ikey, val) == True + assert db.uwes.rem(ikey, val) == True @@ -1825,13 +1825,16 @@ def test_clear_escrows(): db.putVres(key, vals) db.putPses(key, vals) db.putPwes(key, vals) - db.putUwes(key, vals) db.putOoes(key, vals) db.putLdes(key, vals) pre = b'k' snh = b'snh' saidb = b'saidb' + + db.uwes.add(keys=(pre, snh), val=saidb) + assert db.uwes.cnt(keys=(pre, snh)) == 1 + db.qnfs.add(keys=(pre, saidb), val=b"z") assert db.qnfs.cnt(keys=(pre, saidb)) == 1 @@ -1888,7 +1891,7 @@ def test_clear_escrows(): assert db.getVres(key) == [] assert db.getPses(key) == [] assert db.getPwes(key) == [] - assert db.getUwes(key) == [] + assert db.uwes.get(key) == [] assert db.getOoes(key) == [] assert db.getLdes(key) == [] assert db.qnfs.cnt(keys=(pre, saidb)) == 0