From de40ae77935d2b949f05e07a527a24684774f34e Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Mon, 22 Apr 2024 11:47:24 -0700 Subject: [PATCH 1/2] Upgrade baselet for fixes to dumpData New baselet upgrade removes the requirement for a partition argument for `dumpData` on each baselet type. If a partition arg is not provided then all data in the baselet instance is returned. --- CHANGELOG.md | 2 ++ package.json | 4 ++-- src/common/utxobased/db/Processor.ts | 15 ++++++++++----- yarn.lock | 8 ++++---- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87a14762..fc6dc437 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- fixed: Missing `scriptPubkeyByPath` processor data from wallet data dump returned by `dumpData` + ## 2.6.0 (2024-04-09) - added: New fallback server engine info, starting with NOWNodes blockbook servers diff --git a/package.json b/package.json index 0f62faa2..65cc1698 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "@bitcoin-js/tiny-secp256k1-asmjs": "^2.2.3", "altcoin-js": "^1.0.0", "async-mutex": "^0.2.6", - "baselet": "^0.2.4", + "baselet": "^0.3.0", "biggystring": "^4.1.3", "bip32": "^2.0.5", "bip32grs": "^2.0.5", @@ -107,4 +107,4 @@ "typescript": "^4.1.2", "wif": "^2.0.6" } -} \ No newline at end of file +} diff --git a/src/common/utxobased/db/Processor.ts b/src/common/utxobased/db/Processor.ts index 78c651f1..177fe0d5 100644 --- a/src/common/utxobased/db/Processor.ts +++ b/src/common/utxobased/db/Processor.ts @@ -157,12 +157,17 @@ export async function makeProcessor( }, async dumpData(): Promise { - const allBases = Object.values(baselets.all) + type AllBases = typeof baselets.all + const allBases: Array = Object.values( + baselets.all + ) return await Promise.all( - allBases.map(async base => ({ - databaseName: base.databaseName, - data: (await base.dumpData('')) as unknown - })) + allBases.map(async base => { + return { + databaseName: base.databaseName, + data: (await base.dumpData()) as unknown + } + }) ) }, diff --git a/yarn.lock b/yarn.lock index 8b475d8f..eeb9503e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1637,10 +1637,10 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" -baselet@^0.2.4: - version "0.2.4" - resolved "https://registry.yarnpkg.com/baselet/-/baselet-0.2.4.tgz#11450ead06353d9c9971a983790655b93211c097" - integrity sha512-42EmWggEbg1XlOjUIuDkepKo5WQUF8b5r7R6avxCw+T8fW2YEl3YNRLKTDguJC+CXJuu478HkdZ/jiLzinxf7Q== +baselet@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/baselet/-/baselet-0.3.0.tgz#2cebf5ae5833711ce40dcbc0cf06a8732592a94e" + integrity sha512-GNwU6O9vkg1TOLzYepsPE7mP7+hXOJhxUHWE4MARXEiiTLG/RxCL7xpbbGAdnh9JvAlJsa5OSkATMloCWQOpMw== dependencies: disklet "^0.4.5" memlet "^0.1.6" From 83c14124b892f3f8e1ab01f04c1d198dfc9c5a1b Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Fri, 19 Apr 2024 17:42:55 -0700 Subject: [PATCH 2/2] Write to scriptPubkeyByPath index table after addressByScriptPubkey If scriptPubkeyByPath is written to before addressByScriptPubkey, there is a chance that the write to addressByScriptPubkey fails or is interrupted, causing the processor data to be incorrect (an index on data that was never written). This appears to the user as an infamous "Missing processor address...during initialization" error message. We hopefully fix the issue and error for good with this change. The change includes moving the write to scriptPubkeyByPath at the end of the `saveAddress` processor routine. We do this with a update variable which we declare at the start of the routine and define it only when the update to the scriptPubkeyByPath table is necessary. In addition to this, we reworked some of the control-flow of this routine and modified the conditions in the control-flow for better readability. It's important this code is maintainable and easy to reason about. --- CHANGELOG.md | 1 + src/common/utxobased/db/Processor.ts | 71 ++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc6dc437..b2b3ea11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - fixed: Missing `scriptPubkeyByPath` processor data from wallet data dump returned by `dumpData` +- fixed: Prevent wallet processor data corruption ("Missing processor address" error) ## 2.6.0 (2024-04-09) diff --git a/src/common/utxobased/db/Processor.ts b/src/common/utxobased/db/Processor.ts index 177fe0d5..90170e20 100644 --- a/src/common/utxobased/db/Processor.ts +++ b/src/common/utxobased/db/Processor.ts @@ -404,6 +404,15 @@ export async function makeProcessor( async saveAddress(address: IAddress): Promise { await baselets.address(async tables => { + // This variable is used to update the scriptPubkeyByPath table. + // The path table must be written after the address table because the + // path table is an index of the address table. + // This variable acts as a catch for that update to be done after the + // address table is written. + let indexTableUpdate: + | { path: AddressPath; scriptPubkey: string } + | undefined + const [existingAddress] = await tables.addressByScriptPubkey.query('', [ address.scriptPubkey ]) @@ -420,14 +429,14 @@ export async function makeProcessor( 'Attempted to save address with an existing path, but different script pubkey' ) - await tables.scriptPubkeyByPath.insert( - addressPathToPrefix(address.path), - address.path.addressIndex, - address.scriptPubkey - ) + indexTableUpdate = { + path: address.path, + scriptPubkey: address.scriptPubkey + } - // check if this address is used and if so, whether it has a higher last used index - if (address.used || (existingAddress?.used ?? false)) { + // check if this address is used and if so, whether it has a higher + // last used index + if (address.used || existingAddress?.used === true) { let [lastUsed] = await tables.lastUsedByFormatPath.query('', [ addressPathToPrefix(address.path) ]) @@ -443,6 +452,7 @@ export async function makeProcessor( } } + // Update routine: if (existingAddress != null) { // Only update the lastQueriedBlockHeight on the address if one was given and is greater than the existing value if ( @@ -468,16 +478,21 @@ export async function makeProcessor( existingAddress.balance = address.balance } - // Only update the path field if one was given and currently does not have one - // NOTE: Addresses can be stored in the db without a path due to the `EdgeCurrencyEngine.addGapLimitAddresses` function - // Once an address path is known, it should never be updated + /* + Only update the path field if one was given and the existing address + currently does not have one. We never update paths for addresses, only + insert paths when they're not present. + + NOTE: Addresses can be stored in the db without a path due to the + `EdgeCurrencyEngine.addGapLimitAddresses` function. Once an address + path is known, it should never be updated + */ if (address.path != null && existingAddress.path == null) { existingAddress.path = address.path - await tables.scriptPubkeyByPath.insert( - addressPathToPrefix(address.path), - address.path.addressIndex, - address.scriptPubkey - ) + indexTableUpdate = { + path: existingAddress.path, + scriptPubkey: existingAddress.scriptPubkey + } } // Only update the used flag if one was given and is true @@ -500,18 +515,32 @@ export async function makeProcessor( ) } } + + // Update the address table: await tables.addressByScriptPubkey.insert( '', existingAddress.scriptPubkey, existingAddress ) - return } - await tables.addressByScriptPubkey.insert( - '', - address.scriptPubkey, - address - ) + + // Insert routine: + if (existingAddress == null) { + await tables.addressByScriptPubkey.insert( + '', + address.scriptPubkey, + address + ) + } + + // Insert the path into the index table: + if (indexTableUpdate != null) { + await tables.scriptPubkeyByPath.insert( + addressPathToPrefix(indexTableUpdate.path), + indexTableUpdate.path.addressIndex, + indexTableUpdate.scriptPubkey + ) + } }) },