From 78d921b87348c3a610f39cadce4d729eac6bb962 Mon Sep 17 00:00:00 2001 From: Tudor Voicu Date: Thu, 26 Aug 2021 16:50:36 +0200 Subject: [PATCH 1/5] [Divulgence pruning] Prune disclosed contracts * Adapt CommonStorageBackend.pruneEvents to prune all immediately divulged events * Adapt ParticipantPruningIT to assert immediate divulgence pruning * Adapt ParticipantPruningIT tests for divulgence pruning to assert the ACS before and after pruning CHANGELOG_BEGIN CHANGELOG_END --- .../suites/ParticipantPruningIT.scala | 56 +--- .../backend/common/CommonStorageBackend.scala | 32 +++ .../backend/StorageBackendTestValues.scala | 3 +- .../backend/StorageBackendTestsPruning.scala | 261 +++++++++++++++++- 4 files changed, 296 insertions(+), 56 deletions(-) diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ParticipantPruningIT.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ParticipantPruningIT.scala index 8684cd5686e9..ede45213a7c3 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ParticipantPruningIT.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ParticipantPruningIT.scala @@ -595,15 +595,7 @@ class ParticipantPruningIT extends LedgerTestSuite { divulgence.exerciseDivulge(_, contract), ) - _ <- divulgencePruneAndCheck( - alice, - bob, - alpha, - beta, - contract, - divulgence, - disclosureVisibility = false, - ) + _ <- divulgencePruneAndCheck(alice, bob, alpha, beta, contract, divulgence) } yield () }) @@ -624,15 +616,7 @@ class ParticipantPruningIT extends LedgerTestSuite { divulgeNotDiscloseTemplate.exerciseDivulgeNoDisclose(_, divulgence), ) - _ <- divulgencePruneAndCheck( - alice, - bob, - alpha, - beta, - contract, - divulgence, - disclosureVisibility = false, - ) + _ <- divulgencePruneAndCheck(alice, bob, alpha, beta, contract, divulgence) } yield () }) @@ -649,15 +633,8 @@ class ParticipantPruningIT extends LedgerTestSuite { alice, divulgence.exerciseCreateAndDisclose, ) - _ <- divulgencePruneAndCheck( - alice, - bob, - alpha, - beta, - contract, - divulgence, - disclosureVisibility = true, - ) + + _ <- divulgencePruneAndCheck(alice, bob, alpha, beta, contract, divulgence) } yield () }) @@ -679,8 +656,6 @@ class ParticipantPruningIT extends LedgerTestSuite { beta: ParticipantTestContext, contract: Primitive.ContractId[Contract], divulgence: binding.Primitive.ContractId[Divulgence], - // TODO Divulgence pruning: Remove when immediate divulgence pruning is implemented - disclosureVisibility: Boolean, )(implicit ec: ExecutionContext) = for { // Check that Bob can fetch the contract @@ -716,23 +691,12 @@ class ParticipantPruningIT extends LedgerTestSuite { _ <- pruneAtCurrentOffset(beta, bob, pruneAllDivulgedContracts = true) // TODO Divulgence pruning: Check ACS equality before and after pruning - // TODO Divulgence pruning: Remove the true-clause of the if-statement below - // when disclosure pruning is implemented. - _ <- - if (disclosureVisibility) { - beta - .exerciseAndGetContract[Dummy]( - bob, - divulgence.exerciseCanFetch(_, contract), - ) - } else { - beta - .exerciseAndGetContract[Dummy]( - bob, - divulgence.exerciseCanFetch(_, contract), - ) - .mustFail("Bob cannot access the divulged contract after the second pruning") - } + _ <- beta + .exerciseAndGetContract[Dummy]( + bob, + divulgence.exerciseCanFetch(_, contract), + ) + .mustFail("Bob cannot access the divulged contract after the second pruning") } yield () private def populateLedgerAndGetOffsets(participant: ParticipantTestContext, submitter: Party)( diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala index 1920797b5ba1..879ee4013f18 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala @@ -479,6 +479,9 @@ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_ // Events + protected def arrayContains(arrayColumnName: String, element: String): String = + s"$element = any($arrayColumnName)" + def pruneEvents( pruneUpToInclusive: Offset, pruneAllDivulgedContracts: Boolean, @@ -539,6 +542,35 @@ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_ where delete_events.event_offset <= $pruneUpToInclusive""" }(connection, loggingContext) + + if (pruneAllDivulgedContracts) { + val prunedAllDivulgedContractsUpToInclusive = + SQL""" + select participant_all_divulged_contracts_pruned_up_to_inclusive + from parameters + """ + .as(offset("participant_all_divulged_contracts_pruned_up_to_inclusive").?.single)( + connection + ) + .getOrElse(Offset.beforeBegin) + + pruneWithLogging(queryDescription = "Immediate divulgence events pruning") { + SQL""" + -- Immediate divulgence pruning + delete + from participant_events_create c + where event_offset > $prunedAllDivulgedContractsUpToInclusive + and event_offset <= $pruneUpToInclusive + and not exists ( + select 1 + from party_entries as p + where p.ledger_offset <= c.event_offset + and p.is_local + and #${arrayContains("c.flat_event_witnesses", "p.party")} + ) + """ + }(connection, loggingContext) + } } private def pruneWithLogging(queryDescription: String)(query: SimpleSql[Row])( diff --git a/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestValues.scala b/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestValues.scala index 981e126972a1..96bd0937e10d 100644 --- a/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestValues.scala +++ b/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestValues.scala @@ -70,6 +70,7 @@ private[backend] object StorageBackendTestValues { def dtoPartyEntry( offset: Offset, party: String = someParty, + isLocal: Boolean = true, ): DbDto.PartyEntry = DbDto.PartyEntry( ledger_offset = offset.toHexString, recorded_at = someTime, @@ -78,7 +79,7 @@ private[backend] object StorageBackendTestValues { display_name = Some(party), typ = JdbcLedgerDao.acceptType, rejection_reason = None, - is_local = Some(true), + is_local = Some(isLocal), ) def dtoPackage(offset: Offset): DbDto.Package = DbDto.Package( diff --git a/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala b/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala index ad622c1ca007..6a2d88a32132 100644 --- a/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala +++ b/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala @@ -4,6 +4,7 @@ package com.daml.platform.store.backend import com.daml.lf.data.Ref +import com.daml.platform.store.appendonlydao.events.ContractId import com.daml.platform.store.backend.EventStorageBackend.{FilterParams, RangeParams} import org.scalatest.flatspec.AsyncFlatSpec import org.scalatest.matchers.should.Matchers @@ -61,7 +62,7 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB // Prune // TODO Divulgence pruning: Adapt the tests for all divulged contracts pruning and set the flag to `true` _ <- executeSql( - backend.pruneEvents(offset(2), pruneAllDivulgedContracts = false)(_, loggingContext) + backend.pruneEvents(offset(2), pruneAllDivulgedContracts = true)(_, loggingContext) ) _ <- executeSql(backend.updatePrunedUptoInclusive(offset(2))) // Make sure the events are not visible anymore @@ -89,9 +90,11 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB } it should "not prune an active contract" in { - val someParty = Ref.Party.assertFromString("party") + val partyName = "party" + val someParty = Ref.Party.assertFromString(partyName) + val partyEntry = dtoPartyEntry(offset(1), "party") val create = dtoCreate( - offset = offset(1), + offset = offset(2), eventSequentialId = 1L, contractId = "#1", signatory = someParty, @@ -102,11 +105,11 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB for { _ <- executeSql(backend.initializeParameters(someIdentityParams)) // Ingest a create and archive event - _ <- executeSql(ingest(Vector(create), _)) - _ <- executeSql(backend.updateLedgerEnd(ParameterStorageBackend.LedgerEnd(offset(1), 1L))) + _ <- executeSql(ingest(Vector(partyEntry, create), _)) + _ <- executeSql(backend.updateLedgerEnd(ParameterStorageBackend.LedgerEnd(offset(2), 1L))) // Make sure the events are visible before1 <- executeSql(backend.transactionEvents(range, filter)) - before2 <- executeSql(backend.activeContractEvents(range, filter, offset(1))) + before2 <- executeSql(backend.activeContractEvents(range, filter, offset(2))) before3 <- executeSql(backend.flatTransaction(createTransactionId, filter)) before4 <- executeSql(backend.transactionTreeEvents(range, filter)) before5 <- executeSql(backend.transactionTree(createTransactionId, filter)) @@ -114,12 +117,12 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB // Prune // TODO Divulgence pruning: Adapt the tests for all divulged contracts pruning and set the flag to `true` _ <- executeSql( - backend.pruneEvents(offset(1), pruneAllDivulgedContracts = false)(_, loggingContext) + backend.pruneEvents(offset(2), pruneAllDivulgedContracts = true)(_, loggingContext) ) - _ <- executeSql(backend.updatePrunedUptoInclusive(offset(1))) + _ <- executeSql(backend.updatePrunedUptoInclusive(offset(2))) // Make sure the events are still visible - active contracts should not be pruned after1 <- executeSql(backend.transactionEvents(range, filter)) - after2 <- executeSql(backend.activeContractEvents(range, filter, offset(1))) + after2 <- executeSql(backend.activeContractEvents(range, filter, offset(2))) after3 <- executeSql(backend.flatTransaction(createTransactionId, filter)) after4 <- executeSql(backend.transactionTreeEvents(range, filter)) after5 <- executeSql(backend.transactionTree(createTransactionId, filter)) @@ -142,6 +145,246 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB } } + it should "prune all retroactively and immediately divulged contracts (if pruneAllDivulgedContracts is set)" in { + val partyName = "party" + val divulgee = Ref.Party.assertFromString(partyName) + val contract1_id = "#1" + val contract2_id = "#2" + + val contract1_immediateDivulgence = dtoCreate( + offset = offset(1), + eventSequentialId = 1L, + contractId = contract1_id, + signatory = divulgee, + ) + val partyEntry = dtoPartyEntry(offset(2), partyName) + val contract2_createWithLocalStakeholders = dtoCreate( + offset = offset(3), + eventSequentialId = 2L, + contractId = contract2_id, + signatory = divulgee, + ) + val contract1_retroactiveDivulgence = + dtoDivulgence( + offset = offset(3), + eventSequentialId = 3L, + contractId = contract1_id, + divulgee = partyName, + ) + + for { + _ <- executeSql(backend.initializeParameters(someIdentityParams)) + // Ingest + _ <- executeSql( + ingest( + Vector( + contract1_immediateDivulgence, + partyEntry, + contract1_retroactiveDivulgence, + contract2_createWithLocalStakeholders, + ), + _, + ) + ) + _ <- executeSql(backend.updateLedgerEnd(ParameterStorageBackend.LedgerEnd(offset(4), 4L))) + contract1_beforePruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract1_id), + ) + ) + contract2_beforePruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract2_id), + ) + ) + _ <- executeSql( + backend.pruneEvents(offset(3), pruneAllDivulgedContracts = true)(_, loggingContext) + ) + contract1_afterPruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract1_id), + ) + ) + contract2_afterPruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract2_id), + ) + ) + } yield { + contract1_beforePruning should not be empty + contract2_beforePruning should not be empty + + contract1_afterPruning shouldBe empty + // Immediate divulgence for contract 2 occurred after the associated party became locally hosted + // It is not pruned + contract2_afterPruning should not be empty + } + } + + it should "only prune retroactively divulged contracts if there exists an associated consuming exercise (if pruneAllDivulgedContracts is not set)" in { + val signatory = "signatory" + val divulgee = Ref.Party.assertFromString("party") + val contract1_id = "#1" + val contract2_id = "#2" + + val contract1_create = dtoCreate( + offset = offset(1), + eventSequentialId = 1L, + contractId = contract1_id, + signatory = signatory, + ) + val contract1_divulgence = dtoDivulgence( + offset = offset(2), + eventSequentialId = 2L, + contractId = contract1_id, + divulgee = "party", + ) + val contract1_consumingExercise = dtoExercise( + offset = offset(3), + eventSequentialId = 3L, + consuming = true, + contractId = contract1_id, + ) + val contract2_divulgence = dtoDivulgence( + offset = offset(4), + eventSequentialId = 4L, + contractId = contract2_id, + divulgee = divulgee, + ) + + for { + _ <- executeSql(backend.initializeParameters(someIdentityParams)) + // Ingest + _ <- executeSql( + ingest( + Vector( + contract1_create, + contract1_divulgence, + contract1_consumingExercise, + contract2_divulgence, + ), + _, + ) + ) + _ <- executeSql(backend.updateLedgerEnd(ParameterStorageBackend.LedgerEnd(offset(5), 5L))) + contract1_beforePruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract1_id), + ) + ) + contract2_beforePruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract2_id), + ) + ) + _ <- executeSql( + backend.pruneEvents(offset(4), pruneAllDivulgedContracts = false)(_, loggingContext) + ) + contract1_afterPruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract1_id), + ) + ) + contract2_afterPruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract2_id), + ) + ) + } yield { + contract1_beforePruning should not be empty + contract2_beforePruning should not be empty + + contract1_afterPruning shouldBe empty + contract2_afterPruning should not be empty + } + } + + it should "prune all retroactively and immediately divulged contracts if pruneAllDivulgedContracts is set" in { + val partyName = "party" + val divulgee = Ref.Party.assertFromString(partyName) + val contract1_id = "#1" + val contract2_id = "#2" + + val contract1_immediateDivulgence = dtoCreate( + offset = offset(1), + eventSequentialId = 1L, + contractId = contract1_id, + signatory = divulgee, + ) + val partyEntry = dtoPartyEntry(offset(2), partyName) + val contract2_createWithLocalStakeholders = dtoCreate( + offset = offset(3), + eventSequentialId = 2L, + contractId = contract2_id, + signatory = divulgee, + ) + val contract1_retroactiveDivulgence = + dtoDivulgence( + offset = offset(3), + eventSequentialId = 3L, + contractId = contract1_id, + divulgee = partyName, + ) + + for { + _ <- executeSql(backend.initializeParameters(someIdentityParams)) + // Ingest + _ <- executeSql( + ingest( + Vector( + contract1_immediateDivulgence, + partyEntry, + contract1_retroactiveDivulgence, + contract2_createWithLocalStakeholders, + ), + _, + ) + ) + _ <- executeSql(backend.updateLedgerEnd(ParameterStorageBackend.LedgerEnd(offset(4), 4L))) + contract1_beforePruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract1_id), + ) + ) + contract2_beforePruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract2_id), + ) + ) + _ <- executeSql( + backend.pruneEvents(offset(3), pruneAllDivulgedContracts = true)(_, loggingContext) + ) + contract1_afterPruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract1_id), + ) + ) + contract2_afterPruning <- executeSql( + backend.activeContractWithoutArgument( + Set(divulgee), + ContractId.assertFromString(contract2_id), + ) + ) + } yield { + contract1_beforePruning should not be empty + contract2_beforePruning should not be empty + + contract1_afterPruning shouldBe empty + contract2_afterPruning should not be empty + } + } + it should "prune completions" in { val someParty = Ref.Party.assertFromString("party") val completion = dtoCompletion( From d9aae2c01c47f2c4cefcdb9ca9127d34747a35c6 Mon Sep 17 00:00:00 2001 From: Tudor Voicu Date: Sat, 28 Aug 2021 07:59:24 +0200 Subject: [PATCH 2/5] Adapt conformance tests --- .../BUILD.bazel | 2 + .../suites/ParticipantPruningIT.scala | 28 ++++-- ledger/ledger-on-memory/BUILD.bazel | 2 +- ledger/ledger-on-sql/BUILD.bazel | 10 +- .../backend/common/CommonStorageBackend.scala | 3 +- .../backend/StorageBackendTestsPruning.scala | 94 ++----------------- ledger/sandbox-on-x/BUILD.bazel | 2 +- 7 files changed, 45 insertions(+), 96 deletions(-) diff --git a/ledger/ledger-api-test-tool-on-canton/BUILD.bazel b/ledger/ledger-api-test-tool-on-canton/BUILD.bazel index 329692cef6e3..16524c3498f8 100644 --- a/ledger/ledger-api-test-tool-on-canton/BUILD.bazel +++ b/ledger/ledger-api-test-tool-on-canton/BUILD.bazel @@ -188,5 +188,7 @@ conformance_test( "--verbose", "--concurrent-test-runs=1", # lowered from default #procs to reduce flakes - details in https://github.com/digital-asset/daml/issues/7316 "--include=ParticipantPruningIT", + # Disable tests targeting only multi-participant setups + "--exclude=ParticipantPruningIT:PRDisclosureAndRetroactiveDivulgence", ], ) if not is_windows else None diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ParticipantPruningIT.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ParticipantPruningIT.scala index ede45213a7c3..884e6e9365de 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ParticipantPruningIT.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ParticipantPruningIT.scala @@ -551,7 +551,11 @@ class ParticipantPruningIT extends LedgerTestSuite { divulgence.exerciseCanFetch(_, contract), ) - _ <- pruneAtCurrentOffset(participant, bob, pruneAllDivulgedContracts = false) + _ <- pruneAtCurrentOffset( + participant, + bob, + pruneAllDivulgedContracts = false, + ) // Bob can still see the divulged contract _ <- participant.exerciseAndGetContract[Dummy]( @@ -562,7 +566,11 @@ class ParticipantPruningIT extends LedgerTestSuite { // Archive the divulged contract _ <- participant.exercise(alice, contract.exerciseArchive) - _ <- pruneAtCurrentOffset(participant, bob, pruneAllDivulgedContracts = false) + _ <- pruneAtCurrentOffset( + participant, + bob, + pruneAllDivulgedContracts = false, + ) _ <- participant .exerciseAndGetContract[Dummy]( @@ -690,7 +698,6 @@ class ParticipantPruningIT extends LedgerTestSuite { _ <- pruneAtCurrentOffset(beta, bob, pruneAllDivulgedContracts = true) - // TODO Divulgence pruning: Check ACS equality before and after pruning _ <- beta .exerciseAndGetContract[Dummy]( bob, @@ -747,13 +754,22 @@ class ParticipantPruningIT extends LedgerTestSuite { private def pruneAtCurrentOffset( participant: ParticipantTestContext, - party: Party, + localParty: Party, pruneAllDivulgedContracts: Boolean, )(implicit ec: ExecutionContext): Future[Unit] = for { offset <- participant.currentEnd() // Dummy needed to prune at this offset - _ <- participant.create(party, Dummy(party)) + _ <- participant.create(localParty, Dummy(localParty)) + + acsBeforePruning <- participant.activeContracts(localParty) _ <- participant.prune(offset, pruneAllDivulgedContracts = pruneAllDivulgedContracts) - } yield () + acsAfterPruning <- participant.activeContracts(localParty) + + } yield { + assert( + acsBeforePruning == acsAfterPruning, + s"Active contract set comparison before and after pruning failed: $acsBeforePruning vs $acsAfterPruning", + ) + } } diff --git a/ledger/ledger-on-memory/BUILD.bazel b/ledger/ledger-on-memory/BUILD.bazel index 472b3a08e2f0..efcb56c2148d 100644 --- a/ledger/ledger-on-memory/BUILD.bazel +++ b/ledger/ledger-on-memory/BUILD.bazel @@ -226,7 +226,7 @@ conformance_test( "--verbose", "--include=ParticipantPruningIT", # Disable tests targeting only append-only schema functionality - "--exclude=ParticipantPruningIT:PRLocalAndNonLocalRetroactiveDivulgences,ParticipantPruningIT:PRRetroactiveDivulgences", + "--exclude=ParticipantPruningIT:PRLocalAndNonLocalRetroactiveDivulgences,ParticipantPruningIT:PRRetroactiveDivulgences,ParticipantPruningIT:PRDisclosureAndRetroactiveDivulgence", ], ) diff --git a/ledger/ledger-on-sql/BUILD.bazel b/ledger/ledger-on-sql/BUILD.bazel index b98f34a54eee..431285d05da5 100644 --- a/ledger/ledger-on-sql/BUILD.bazel +++ b/ledger/ledger-on-sql/BUILD.bazel @@ -280,7 +280,7 @@ da_scala_test_suite( "--verbose", "--include=ParticipantPruningIT", # Disable tests targeting only append-only schema functionality - "--exclude=ParticipantPruningIT:PRLocalAndNonLocalRetroactiveDivulgences,ParticipantPruningIT:PRRetroactiveDivulgences", + "--exclude=ParticipantPruningIT:PRLocalAndNonLocalRetroactiveDivulgences,ParticipantPruningIT:PRRetroactiveDivulgences,ParticipantPruningIT:PRDisclosureAndRetroactiveDivulgence", ], ), conformance_test( @@ -441,6 +441,8 @@ conformance_test( test_tool_args = [ "--verbose", "--include=ParticipantPruningIT", + # Disable tests targeting only multi-participant setups + "--exclude=ParticipantPruningIT:PRDisclosureAndRetroactiveDivulgence", ], ) @@ -459,6 +461,8 @@ conformance_test( test_tool_args = [ "--verbose", "--include=ParticipantPruningIT", + # Disable tests targeting only multi-participant setups + "--exclude=ParticipantPruningIT:PRDisclosureAndRetroactiveDivulgence", ], ) @@ -477,6 +481,8 @@ conformance_test( test_tool_args = [ "--verbose", "--include=ParticipantPruningIT", + # Disable tests targeting only multi-participant setups + "--exclude=ParticipantPruningIT:PRDisclosureAndRetroactiveDivulgence", ], ) @@ -493,7 +499,7 @@ conformance_test( "--verbose", "--include=ParticipantPruningIT", # Disable tests targeting only append-only schema functionality - "--exclude=ParticipantPruningIT:PRLocalAndNonLocalRetroactiveDivulgences,ParticipantPruningIT:PRRetroactiveDivulgences", + "--exclude=ParticipantPruningIT:PRLocalAndNonLocalRetroactiveDivulgences,ParticipantPruningIT:PRRetroactiveDivulgences,ParticipantPruningIT:PRDisclosureAndRetroactiveDivulgence", ], ) diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala index 879ee4013f18..7d4217c1945d 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala @@ -557,8 +557,7 @@ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_ pruneWithLogging(queryDescription = "Immediate divulgence events pruning") { SQL""" -- Immediate divulgence pruning - delete - from participant_events_create c + delete from participant_events_create c where event_offset > $prunedAllDivulgedContractsUpToInclusive and event_offset <= $pruneUpToInclusive and not exists ( diff --git a/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala b/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala index 6a2d88a32132..8b26b33b7112 100644 --- a/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala +++ b/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala @@ -60,7 +60,6 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB before5 <- executeSql(backend.transactionTree(createTransactionId, filter)) before6 <- executeSql(backend.rawEvents(0, 2L)) // Prune - // TODO Divulgence pruning: Adapt the tests for all divulged contracts pruning and set the flag to `true` _ <- executeSql( backend.pruneEvents(offset(2), pruneAllDivulgedContracts = true)(_, loggingContext) ) @@ -115,7 +114,6 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB before5 <- executeSql(backend.transactionTree(createTransactionId, filter)) before6 <- executeSql(backend.rawEvents(0, 1L)) // Prune - // TODO Divulgence pruning: Adapt the tests for all divulged contracts pruning and set the flag to `true` _ <- executeSql( backend.pruneEvents(offset(2), pruneAllDivulgedContracts = true)(_, loggingContext) ) @@ -158,7 +156,7 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB signatory = divulgee, ) val partyEntry = dtoPartyEntry(offset(2), partyName) - val contract2_createWithLocalStakeholders = dtoCreate( + val contract2_createWithLocalStakeholder = dtoCreate( offset = offset(3), eventSequentialId = 2L, contractId = contract2_id, @@ -181,7 +179,7 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB contract1_immediateDivulgence, partyEntry, contract1_retroactiveDivulgence, - contract2_createWithLocalStakeholders, + contract2_createWithLocalStakeholder, ), _, ) @@ -219,8 +217,8 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB contract2_beforePruning should not be empty contract1_afterPruning shouldBe empty - // Immediate divulgence for contract 2 occurred after the associated party became locally hosted - // It is not pruned + // Immediate divulgence for contract2 occurred after the associated party became locally hosted + // so it is not pruned contract2_afterPruning should not be empty } } @@ -270,6 +268,7 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB _, ) ) + // Set the ledger end past the last ingested event so we can prune up to it inclusively _ <- executeSql(backend.updateLedgerEnd(ParameterStorageBackend.LedgerEnd(offset(5), 5L))) contract1_beforePruning <- executeSql( backend.activeContractWithoutArgument( @@ -299,88 +298,15 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB ) ) } yield { + // Contract 1 appears as active tu `divulgee` before pruning contract1_beforePruning should not be empty + // Contract 2 appears as active tu `divulgee` before pruning contract2_beforePruning should not be empty + // Contract 1 should not be visible anymore to `divulgee` after pruning contract1_afterPruning shouldBe empty - contract2_afterPruning should not be empty - } - } - - it should "prune all retroactively and immediately divulged contracts if pruneAllDivulgedContracts is set" in { - val partyName = "party" - val divulgee = Ref.Party.assertFromString(partyName) - val contract1_id = "#1" - val contract2_id = "#2" - - val contract1_immediateDivulgence = dtoCreate( - offset = offset(1), - eventSequentialId = 1L, - contractId = contract1_id, - signatory = divulgee, - ) - val partyEntry = dtoPartyEntry(offset(2), partyName) - val contract2_createWithLocalStakeholders = dtoCreate( - offset = offset(3), - eventSequentialId = 2L, - contractId = contract2_id, - signatory = divulgee, - ) - val contract1_retroactiveDivulgence = - dtoDivulgence( - offset = offset(3), - eventSequentialId = 3L, - contractId = contract1_id, - divulgee = partyName, - ) - - for { - _ <- executeSql(backend.initializeParameters(someIdentityParams)) - // Ingest - _ <- executeSql( - ingest( - Vector( - contract1_immediateDivulgence, - partyEntry, - contract1_retroactiveDivulgence, - contract2_createWithLocalStakeholders, - ), - _, - ) - ) - _ <- executeSql(backend.updateLedgerEnd(ParameterStorageBackend.LedgerEnd(offset(4), 4L))) - contract1_beforePruning <- executeSql( - backend.activeContractWithoutArgument( - Set(divulgee), - ContractId.assertFromString(contract1_id), - ) - ) - contract2_beforePruning <- executeSql( - backend.activeContractWithoutArgument( - Set(divulgee), - ContractId.assertFromString(contract2_id), - ) - ) - _ <- executeSql( - backend.pruneEvents(offset(3), pruneAllDivulgedContracts = true)(_, loggingContext) - ) - contract1_afterPruning <- executeSql( - backend.activeContractWithoutArgument( - Set(divulgee), - ContractId.assertFromString(contract1_id), - ) - ) - contract2_afterPruning <- executeSql( - backend.activeContractWithoutArgument( - Set(divulgee), - ContractId.assertFromString(contract2_id), - ) - ) - } yield { - contract1_beforePruning should not be empty - contract2_beforePruning should not be empty - - contract1_afterPruning shouldBe empty + // Contract 2 did not have a locally stored exercise event + // hence its divulgence is not pruned - appears active to `divulgee` contract2_afterPruning should not be empty } } diff --git a/ledger/sandbox-on-x/BUILD.bazel b/ledger/sandbox-on-x/BUILD.bazel index 4872b8ec98f5..bb6301329c1a 100644 --- a/ledger/sandbox-on-x/BUILD.bazel +++ b/ledger/sandbox-on-x/BUILD.bazel @@ -164,7 +164,7 @@ conformance_test( "--verbose", "--include=ParticipantPruningIT", # Disable tests targeting only append-only schema functionality - "--exclude=ParticipantPruningIT:PRLocalAndNonLocalRetroactiveDivulgences,ParticipantPruningIT:PRRetroactiveDivulgences", + "--exclude=ParticipantPruningIT:PRLocalAndNonLocalRetroactiveDivulgences,ParticipantPruningIT:PRRetroactiveDivulgences,ParticipantPruningIT:PRDisclosureAndRetroactiveDivulgence", ], ) From 47f73f8802408fdde6716116895abf746ffd79c7 Mon Sep 17 00:00:00 2001 From: Tudor Voicu Date: Mon, 30 Aug 2021 17:26:18 +0200 Subject: [PATCH 3/5] Adapt disclosure query for Oracle --- .../backend/common/CommonStorageBackend.scala | 42 ++++++++++--------- .../store/backend/common/QueryStrategy.scala | 8 ++++ .../store/backend/h2/H2StorageBackend.scala | 4 ++ .../backend/oracle/OracleStorageBackend.scala | 5 +++ .../postgresql/PostgresStorageBackend.scala | 4 ++ 5 files changed, 44 insertions(+), 19 deletions(-) diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala index 7d4217c1945d..9995a1ebfa9d 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala @@ -37,7 +37,6 @@ import scalaz.syntax.tag._ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_BATCH] { private val logger: ContextualizedLogger = ContextualizedLogger.get(this.getClass) - // Ingestion private val SQL_DELETE_OVERSPILL_ENTRIES: List[SqlQuery] = @@ -55,6 +54,8 @@ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_ SQL("DELETE FROM party_entries WHERE ledger_offset > {ledger_offset}"), ) + def queryStrategy: QueryStrategy + override def initializeIngestion( connection: Connection ): Option[ParameterStorageBackend.LedgerEnd] = { @@ -479,9 +480,6 @@ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_ // Events - protected def arrayContains(arrayColumnName: String, element: String): String = - s"$element = any($arrayColumnName)" - def pruneEvents( pruneUpToInclusive: Offset, pruneAllDivulgedContracts: Boolean, @@ -544,34 +542,40 @@ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_ }(connection, loggingContext) if (pruneAllDivulgedContracts) { - val prunedAllDivulgedContractsUpToInclusive = - SQL""" - select participant_all_divulged_contracts_pruned_up_to_inclusive - from parameters - """ - .as(offset("participant_all_divulged_contracts_pruned_up_to_inclusive").?.single)( - connection - ) - .getOrElse(Offset.beforeBegin) + val pruneAfterClause = + participantAllDivulgedContractsPrunedUpToInclusive(connection) match { + case Some(pruneAfter) => cSQL"and event_offset > $pruneAfter" + case None => cSQL"" + } pruneWithLogging(queryDescription = "Immediate divulgence events pruning") { SQL""" -- Immediate divulgence pruning delete from participant_events_create c - where event_offset > $prunedAllDivulgedContractsUpToInclusive - and event_offset <= $pruneUpToInclusive + where event_offset <= $pruneUpToInclusive + -- Only prune create events which did not have a locally hosted party before their creation offset and not exists ( select 1 - from party_entries as p - where p.ledger_offset <= c.event_offset - and p.is_local - and #${arrayContains("c.flat_event_witnesses", "p.party")} + from party_entries p + where p.typ = 'accept' + and p.ledger_offset <= c.event_offset + and #${queryStrategy.isTrue("p.is_local")} + and #${queryStrategy.arrayContains("c.flat_event_witnesses", "p.party")} ) + $pruneAfterClause """ }(connection, loggingContext) } } + private def participantAllDivulgedContractsPrunedUpToInclusive( + connection: Connection + ): Option[Offset] = + SQL"select participant_all_divulged_contracts_pruned_up_to_inclusive from parameters" + .as(offset("participant_all_divulged_contracts_pruned_up_to_inclusive").?.single)( + connection + ) + private def pruneWithLogging(queryDescription: String)(query: SimpleSql[Row])( connection: Connection, loggingContext: LoggingContext, diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/QueryStrategy.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/QueryStrategy.scala index 8aa0a84aed6e..6aed295945b3 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/QueryStrategy.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/QueryStrategy.scala @@ -47,4 +47,12 @@ trait QueryStrategy { * @return the function name */ def booleanOrAggregationFunction: String = "bool_or" + + /** Predicate which tests if the element referenced by the `elementColumnName` + * is in the array from column `arrayColumnName` + */ + def arrayContains(arrayColumnName: String, elementColumnName: String): String + + /** Boolean predicate */ + def isTrue(booleanColumnName: String): String } diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2StorageBackend.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2StorageBackend.scala index c795403775f6..32da0421411b 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2StorageBackend.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2StorageBackend.scala @@ -127,6 +127,10 @@ private[backend] object H2StorageBackend .map(p => cSQL"array_contains(#$columnName, '#${p.toString}')") .mkComposite("(", " or ", ")") + override def arrayContains(arrayColumnName: String, elementColumnName: String): String = + s"array_contains($arrayColumnName, $elementColumnName)" + + override def isTrue(booleanColumnName: String): String = booleanColumnName } override def queryStrategy: QueryStrategy = H2QueryStrategy diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleStorageBackend.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleStorageBackend.scala index 123d717a9843..8ef590efe691 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleStorageBackend.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleStorageBackend.scala @@ -113,6 +113,11 @@ private[backend] object OracleStorageBackend s"""case when ($column = $value) then 1 else 0 end""" override def booleanOrAggregationFunction: String = "max" + + override def arrayContains(arrayColumnName: String, elementColumnName: String): String = + s"EXISTS (SELECT 1 FROM JSON_TABLE($arrayColumnName, '$$[*]' columns (value PATH '$$')) WHERE value = $elementColumnName)" + + override def isTrue(booleanColumnName: String): String = s"$booleanColumnName = 1" } override def queryStrategy: QueryStrategy = OracleQueryStrategy diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresStorageBackend.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresStorageBackend.scala index 1dd9bf924d67..5111db8cc18b 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresStorageBackend.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresStorageBackend.scala @@ -188,6 +188,10 @@ private[backend] object PostgresStorageBackend cSQL"#$columnName::text[] && $partiesArray::text[]" } + override def arrayContains(arrayColumnName: String, elementColumnName: String): String = + s"$elementColumnName = any($arrayColumnName)" + + override def isTrue(booleanColumnName: String): String = booleanColumnName } override def queryStrategy: QueryStrategy = PostgresQueryStrategy From aec2a5cfeaffab81124870976e637dfeffbb09ec Mon Sep 17 00:00:00 2001 From: Tudor Voicu Date: Tue, 31 Aug 2021 15:33:09 +0200 Subject: [PATCH 4/5] Addressed review comments --- .../BUILD.bazel | 2 -- .../backend/common/CommonStorageBackend.scala | 32 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/ledger/ledger-api-test-tool-on-canton/BUILD.bazel b/ledger/ledger-api-test-tool-on-canton/BUILD.bazel index 16524c3498f8..329692cef6e3 100644 --- a/ledger/ledger-api-test-tool-on-canton/BUILD.bazel +++ b/ledger/ledger-api-test-tool-on-canton/BUILD.bazel @@ -188,7 +188,5 @@ conformance_test( "--verbose", "--concurrent-test-runs=1", # lowered from default #procs to reduce flakes - details in https://github.com/digital-asset/daml/issues/7316 "--include=ParticipantPruningIT", - # Disable tests targeting only multi-participant setups - "--exclude=ParticipantPruningIT:PRDisclosureAndRetroactiveDivulgence", ], ) if not is_windows else None diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala index 9995a1ebfa9d..f15ab3632f20 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala @@ -525,22 +525,6 @@ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_ )""" }(connection, loggingContext) - pruneWithLogging(queryDescription = "Exercise (consuming) events pruning") { - SQL""" - -- Exercise events (consuming) - delete from participant_events_consuming_exercise delete_events - where - delete_events.event_offset <= $pruneUpToInclusive""" - }(connection, loggingContext) - - pruneWithLogging(queryDescription = "Exercise (non-consuming) events pruning") { - SQL""" - -- Exercise events (non-consuming) - delete from participant_events_non_consuming_exercise delete_events - where - delete_events.event_offset <= $pruneUpToInclusive""" - }(connection, loggingContext) - if (pruneAllDivulgedContracts) { val pruneAfterClause = participantAllDivulgedContractsPrunedUpToInclusive(connection) match { @@ -566,6 +550,22 @@ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_ """ }(connection, loggingContext) } + + pruneWithLogging(queryDescription = "Exercise (consuming) events pruning") { + SQL""" + -- Exercise events (consuming) + delete from participant_events_consuming_exercise delete_events + where + delete_events.event_offset <= $pruneUpToInclusive""" + }(connection, loggingContext) + + pruneWithLogging(queryDescription = "Exercise (non-consuming) events pruning") { + SQL""" + -- Exercise events (non-consuming) + delete from participant_events_non_consuming_exercise delete_events + where + delete_events.event_offset <= $pruneUpToInclusive""" + }(connection, loggingContext) } private def participantAllDivulgedContractsPrunedUpToInclusive( From 1a981c2b7d130a892151c552c70f414d0318461c Mon Sep 17 00:00:00 2001 From: Tudor Voicu Date: Tue, 31 Aug 2021 17:57:04 +0200 Subject: [PATCH 5/5] Rebased --- .../store/backend/common/CommonStorageBackend.scala | 6 +++++- .../platform/store/backend/StorageBackendTestsPruning.scala | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala index f15ab3632f20..3c8017a3c7df 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CommonStorageBackend.scala @@ -480,6 +480,7 @@ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_ // Events + // TODO Cleanup: Extract to [[EventStorageBackendTemplate]] def pruneEvents( pruneUpToInclusive: Offset, pruneAllDivulgedContracts: Boolean, @@ -526,11 +527,14 @@ private[backend] trait CommonStorageBackend[DB_BATCH] extends StorageBackend[DB_ }(connection, loggingContext) if (pruneAllDivulgedContracts) { - val pruneAfterClause = + val pruneAfterClause = { + // We need to distinguish between the two cases since lexicographical comparison + // in Oracle doesn't work with '' (empty strings are treated as NULLs) as one of the operands participantAllDivulgedContractsPrunedUpToInclusive(connection) match { case Some(pruneAfter) => cSQL"and event_offset > $pruneAfter" case None => cSQL"" } + } pruneWithLogging(queryDescription = "Immediate divulgence events pruning") { SQL""" diff --git a/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala b/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala index 8b26b33b7112..733ea474682f 100644 --- a/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala +++ b/ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsPruning.scala @@ -164,7 +164,7 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB ) val contract1_retroactiveDivulgence = dtoDivulgence( - offset = offset(3), + offset = Some(offset(3)), eventSequentialId = 3L, contractId = contract1_id, divulgee = partyName, @@ -236,7 +236,7 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB signatory = signatory, ) val contract1_divulgence = dtoDivulgence( - offset = offset(2), + offset = Some(offset(2)), eventSequentialId = 2L, contractId = contract1_id, divulgee = "party", @@ -248,7 +248,7 @@ private[backend] trait StorageBackendTestsPruning extends Matchers with StorageB contractId = contract1_id, ) val contract2_divulgence = dtoDivulgence( - offset = offset(4), + offset = Some(offset(4)), eventSequentialId = 4L, contractId = contract2_id, divulgee = divulgee,