From fa29d3f1b4cee76232ef5ae5780ba3647949dd09 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Tue, 7 Dec 2021 12:29:18 -0500 Subject: [PATCH 01/23] chain: more comments in syncTree, assert tree and chain are in sync --- lib/blockchain/chain.js | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index aa44ce566..48f42c6fc 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -126,11 +126,35 @@ class Chain extends AsyncEmitter { */ async syncTree() { + this.logger.info('Synchronizing Tree with block history...'); + const {treeInterval} = this.network.names; - const last = this.height - (this.height % treeInterval); - this.logger.info('Synchronizing Tree with block history...'); + // Current state of the tree, freshly loaded from disk. + // It will be in the most recently-committed state, + // which was at the last tree interval. There might have been + // new blocks added to the chain since then. + const currentRoot = this.db.treeRoot(); + + // Get the block that corresponds to beginning of the last tree interval. + // The tree root in this block header will be the first block that + // commits to the new tree root hash, but the transactions it contains + // have not yet been added to the tree. + const last = this.height - (this.height % treeInterval); + const entry = await this.db.getEntryByHeight(last + 1); + + // Require that tree is less than one tree interval behind the chain. + // We can skip this check only if the blockchain is less than + // one tree interval old, OR the chain is already at the tree interval, + // meaning the tree is already in sync with the chain and there is no delta. + if (entry) { + assert(entry.treeRoot.equals(currentRoot)); + } else { + assert(last === 0 || last === this.height); + } + // Replay all blocks since the last tree interval to rebuild + // the `txn` which is the in-memory delta between tree interval commitments. for (let height = last + 1; height <= this.height; height++) { const entry = await this.db.getEntryByHeight(height); assert(entry); From 042da6dbfca5ae19d096dad7a5b907af7f9b96d3 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Wed, 8 Dec 2021 14:06:02 -0500 Subject: [PATCH 02/23] chain: wrap saveNames() in a DB batch and use _saveNames() internally --- lib/blockchain/chaindb.js | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 1cc60ee97..9157ec3f0 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -1930,6 +1930,25 @@ class ChainDB { */ async saveNames(view, entry, revert) { + this.start(); + try { + await this._saveNames(view, entry, revert); + } catch (e) { + this.drop(); + throw e; + } + await this.commit(); + } + + /** + * Commit names to tree, assuming batch is started. + * @private + * @param {CoinView} view + * @param {ChainEntry} entry + * @param {Boolean} revert + */ + + async _saveNames(view, entry, revert) { for (const ns of view.names.values()) { const {nameHash} = ns; @@ -1980,7 +1999,7 @@ class ChainDB { else this.put(layout.w.encode(entry.height), undo.encode()); - return this.saveNames(view, entry, false); + return this._saveNames(view, entry, false); } /** @@ -2005,7 +2024,7 @@ class ChainDB { this.del(layout.w.encode(entry.height)); } - return this.saveNames(view, entry, true); + return this._saveNames(view, entry, true); } /** From 2492ea82c27f38ebdf57396ae342b32b8357378d Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Wed, 8 Dec 2021 14:11:23 -0500 Subject: [PATCH 03/23] chain: enable syncTree() to process multiple tree intervals --- lib/blockchain/chain.js | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 48f42c6fc..00ae2d6df 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -140,17 +140,28 @@ class Chain extends AsyncEmitter { // The tree root in this block header will be the first block that // commits to the new tree root hash, but the transactions it contains // have not yet been added to the tree. - const last = this.height - (this.height % treeInterval); - const entry = await this.db.getEntryByHeight(last + 1); - - // Require that tree is less than one tree interval behind the chain. - // We can skip this check only if the blockchain is less than + let last = this.height - (this.height % treeInterval); + let entry = await this.db.getEntryByHeight(last + 1); + + // Using the current tree root hash, rewind the blockchain to the beginning + // of the corresponding tree interval. If the tree has been compacted, + // this may be SEVERAL tree intervals behind the current chain tip + // (this is required to support chain reorgs to a limited depth, + // similar to pruning nodes keeping the last 288 blocks on disk). + // We can skip this only if the blockchain is less than // one tree interval old, OR the chain is already at the tree interval, // meaning the tree is already in sync with the chain and there is no delta. - if (entry) { - assert(entry.treeRoot.equals(currentRoot)); - } else { + if (!entry) { assert(last === 0 || last === this.height); + } else { + for (;;) { + if (entry.treeRoot.equals(currentRoot)) + break; + + last -= treeInterval; + entry = await this.db.getEntryByHeight(last + 1); + assert(entry); + } } // Replay all blocks since the last tree interval to rebuild @@ -171,8 +182,8 @@ class Chain extends AsyncEmitter { for (const tx of block.txs) await this.verifyCovenants(tx, view, height, hardened); - assert((height % this.network.names.treeInterval) !== 0); - + // If the chain replay crosses a tree interval, it will commit + // and write to disk in saveNames(), resetting the `txn` like usual. await this.db.saveNames(view, entry, false); } From 6c9253501bb8985995d5d4fd27b3602d95104b3d Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Wed, 8 Dec 2021 14:12:05 -0500 Subject: [PATCH 04/23] chain: implement urkel tree compaction to historical root --- lib/blockchain/chain.js | 46 +++++++++++++++++++++++++++++++++++++++ lib/blockchain/chaindb.js | 19 ++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 00ae2d6df..0972616b1 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2079,6 +2079,52 @@ class Chain extends AsyncEmitter { } } + /** + * Compact the Urkel Tree. + * Removes all historical state and all data not + * linked directly to the provided root node hash. + * @param {Hash} root + * @returns {Promise} + */ + + async compactTree() { + const unlock = await this.locker.lock(); + this.logger.info('Compacting Urkel Tree...'); + + // To support chain reorgs of limited depth we compact the tree + // to some commitment point in recent history, then rebuild it from there + // back up to the current chain tip. In order to support pruning nodes, + // all blocks above this depth must be available on disk. + // This actually further reduces the ability for a pruning node to recover + // from a deep reorg. On mainnet, `keepBlocks` is 288. A normal pruning + // node can recover from a reorg up to that depth. Compacting the tree + // potentially reduces that depth to 288 - 36 = 252. A reorg deeper than + // that will result in a `MissingNodeError` thrown by Urkel inside + // chain.saveNames() as it tries to restore a deleted state. + + // Oldest block available to a pruning node. + const oldestBlock = this.height - this.network.block.keepBlocks; + + // Distance from that block to the start of the oldest tree interval. + const toNextInterval = + this.network.names.treeInterval - + (oldestBlock % this.network.names.treeInterval); + + // Get the oldest Urkel Tree root state a pruning node can recover from. + const oldestTreeIntervalStart = oldestBlock + toNextInterval + 1; + const entry = await this.db.getEntryByHeight(oldestTreeIntervalStart); + + try { + // Rewind Urkel Tree and delete all historical state. + await this.db.compactTree(entry.treeRoot); + + // Replay the blockchain back up to the tip to rebuild tree. + return await this.syncTree(); + } finally { + unlock(); + } + } + /** * Scan the blockchain for transactions containing specified address hashes. * @param {Hash} start - Block hash to start at. diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 9157ec3f0..37f13bf1d 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -947,6 +947,25 @@ class ChainDB { return true; } + /** + * Compact the Urkel Tree. + * Removes all historical state and all data not + * linked directly to the provided root node hash. + * @param {Hash} root + * @returns {Promise} + */ + + async compactTree(root) { + // Rewind tree to historical commitment + await this.tree.inject(root); + + // Delete historical data + await this.tree.compact(); + + // Reset in-memory tree delta + this.txn = this.tree.txn(); + } + /** * Get the _next_ block hash (does not work by height). * @param {Hash} hash From 50eed57b1719a488b5a14460efb406a5daa44d35 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Fri, 10 Dec 2021 15:04:30 -0500 Subject: [PATCH 05/23] test: compacting urkel tree --- test/chain-tree-compaction-test.js | 325 +++++++++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 test/chain-tree-compaction-test.js diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js new file mode 100644 index 000000000..780630c81 --- /dev/null +++ b/test/chain-tree-compaction-test.js @@ -0,0 +1,325 @@ +'use strict'; + +const os = require('os'); +const path = require('path'); +const fs = require('bfile'); +const assert = require('bsert'); +const Network = require('../lib/protocol/network'); +const Miner = require('../lib/mining/miner'); +const Chain = require('../lib/blockchain/chain'); +const blockstore = require('../lib/blockstore'); +const MemWallet = require('./util/memwallet'); +const rules = require('../lib/covenants/rules'); +const NameState = require('../lib/covenants/namestate'); + +const network = Network.get('regtest'); +const { + treeInterval, + biddingPeriod, + revealPeriod +} = network.names; + +describe('Tree Compacting', function() { + for (const prune of [true, false]) { + describe(`Chain: ${prune ? 'Pruning' : 'Archival'}`, function() { + const prefix = path.join( + os.tmpdir(), + `hsd-tree-compacting-test-${Date.now()}` + ); + const treePath = path.join(prefix, 'tree', '0000000001'); + + const blocks = blockstore.create({ + prefix, + network + }); + + const chain = new Chain({ + memory: false, + prefix, + blocks, + network, + prune + }); + + const miner = new Miner({chain}); + const cpu = miner.cpu; + + const wallet = new MemWallet({network}); + wallet.getNameStatus = async (nameHash) => { + assert(Buffer.isBuffer(nameHash)); + const height = chain.height + 1; + return chain.db.getNameStatus(nameHash, height); + }; + + const mempool = []; + + function send(mtx, mempool) { + mempool.push(mtx.toTX()); + } + + async function mineBlocks(num, mempool = []) { + const job = await cpu.createJob(); + while (mempool.length) { + job.pushTX(mempool.pop()); + } + job.refresh(); + const block = await job.mineAsync(); + const entry = await chain.add(block); + wallet.addBlock(entry, block.txs); + + for (num--; num > 0; num--) { + const job = await cpu.createJob(); + const block = await job.mineAsync(); + const entry = await chain.add(block); + wallet.addBlock(entry, block.txs); + } + } + + const oldKeepBlocks = network.block.keepBlocks; + const oldpruneAfterHeight = network.block.pruneAfterHeight; + + let name, nameHash; + const treeRoots = []; + + before(async () => { + // Copy the 1:8 ratio from mainnet + network.block.keepBlocks = treeInterval * 8; + + // Ensure old blocks are pruned right away + network.block.pruneAfterHeight = 1; + + await blocks.ensure(); + await blocks.open(); + await chain.open(); + await miner.open(); + }); + + after(async () => { + await miner.close(); + await chain.close(); + await blocks.close(); + await fs.rimraf(prefix); + + network.block.keepBlocks = oldKeepBlocks; + network.block.pruneAfterHeight = oldpruneAfterHeight; + }); + + it('should fund wallet', async () => { + miner.addresses.length = 0; + miner.addAddress(wallet.getReceive()); + await mineBlocks(100); + }); + + it('should win an auction and register', async () => { + name = rules.grindName(3, chain.height, network); + nameHash = rules.hashName(name); + send(await wallet.sendOpen(name), mempool); + await mineBlocks(treeInterval + 1, mempool); + send(await wallet.sendBid(name, 10000, 10000), mempool); + await mineBlocks(biddingPeriod, mempool); + send(await wallet.sendReveal(name), mempool); + await mineBlocks(revealPeriod, mempool); + + // Instead of using a version 0 serialized `Resource` with DNS data, + // just register a single byte as a counter. + send(await wallet.sendRegister(name, Buffer.from([0x00])), mempool); + await mineBlocks(treeInterval, mempool); + }); + + it('should update Urkel Tree 20 times', async () => { + let count = 0; + chain.on('tree commit', (rootHash, entry, block) => { + count++; + // Keep track of all new tree root hashes. + treeRoots.push(rootHash); + }); + + for (let i = 1; i <= 20; i++) { + // Every namestate update, increment the name's 1-byte data resource. + send(await wallet.sendUpdate(name, Buffer.from([i])), mempool); + await mineBlocks(treeInterval, mempool); + } + + assert.strictEqual(count, 20); + + const ns = await chain.db.getNameStateByName(name); + assert.bufferEqual(ns.data, Buffer.from([20])); + }); + + it('should restore tree state from any historical root', async () => { + for (let i = 0; i < treeRoots.length; i++) { + // Restore old tree state using historical root hash. + await chain.db.tree.inject(treeRoots[i]); + + // Get old namestate from old tree state. + const raw = await chain.db.tree.get(nameHash); + const ns = NameState.decode(raw); + + // Counter in the name's data resource should match. + assert.bufferEqual(ns.data, Buffer.from([i + 1])); + } + }); + + it('should compact tree', async () => { + const before = await fs.stat(treePath); + await chain.compactTree(); + const after = await fs.stat(treePath); + + // Urkel Tree should be smaller now. + // Urkel Tree files are padded to ensure that Meta nodes are written + // at predictable offsets so they can be quickly discovered + // without indexing (Meta nodes point to the current tree root + // node as well as the PREVIOUS Meta node). This makes it + // really complicated to estimate exactly what the size of the file + // should be before and after compacting. + // The compacting process definitely should have deleted 12 of + // the 20 recent tree namestate updates so we can make + // sure that the data savings is at least that size. It should + // also have deleted the OPEN, REVEAL and original REGISTER, + // as well as tree updates written during the initial wallet funding. + // One advantage we have is there is only one name in the tree, + // so there are no internal nodes. In fact, the tree root node + // is just our name's leaf node! + const META_NODE_SIZE = 38; // See urkel/radix/store.js + const LEAF_NODE_SIZE = 40; // See urkel/radix/nodes.js + const NAMESTATE_DATA_SIZE = 53; // NameState.getSize(); + + // First 100 blocks wrote a Meta node every tree interval + const fundingWallet = (100 / treeInterval) * META_NODE_SIZE; + + const eachUpdate = ( + META_NODE_SIZE + + LEAF_NODE_SIZE + + NAMESTATE_DATA_SIZE + ); + + // Auction wrote a Meta node and namestate update every tree interval + const auction = ( + (treeInterval + 1 + biddingPeriod + revealPeriod + treeInterval) / + treeInterval + ) * eachUpdate; + + const minReduction = fundingWallet + auction + (eachUpdate * 12); + + // The margin of error here is the padding. + assert(before.size - after.size >= minReduction); + + // We also expect the compacted tree to be at least big enough + // for the last 8 namestate updates. Padding makes precision difficult. + assert(after.size >= (eachUpdate * 8)); + }); + + it('should ONLY restore tree state from most recent roots', async () => { + for (let i = 0; i < treeRoots.length; i++) { + if (i < (treeRoots.length - 8)) { + // Old root node has been deleted, tree state can not be restored. + await assert.rejects( + chain.db.tree.inject(treeRoots[i]), + {message: `Missing node: ${treeRoots[i].toString('hex')}.`} + ); + continue; + } + + // Last 8 tree roots are recovered successfully like before compaction. + await chain.db.tree.inject(treeRoots[i]); + const raw = await chain.db.tree.get(nameHash); + const ns = NameState.decode(raw); + assert.bufferEqual(ns.data, Buffer.from([i + 1])); + } + }); + + it('should compact tree a second time with no new data', async () => { + // If user executes rpc compacttree repeatedly, + // it shouldn't break anything. + const before = await fs.stat(treePath); + await chain.compactTree(); + const after = await fs.stat(treePath); + + // Should be no change + assert.strictEqual(before.size, after.size); + }); + + it('should ONLY restore tree state from most recent roots', async () => { + // Data recovery conditions are the same after second compacttree. + for (let i = 0; i < treeRoots.length; i++) { + if (i < (treeRoots.length - 8)) { + // Old root node has been deleted, tree state can not be restored. + await assert.rejects( + chain.db.tree.inject(treeRoots[i]), + {message: `Missing node: ${treeRoots[i].toString('hex')}.`} + ); + continue; + } + + // Last 8 tree roots are recovered successfully like before compaction. + await chain.db.tree.inject(treeRoots[i]); + const raw = await chain.db.tree.get(nameHash); + const ns = NameState.decode(raw); + assert.bufferEqual(ns.data, Buffer.from([i + 1])); + } + }); + + it('should recover txn between tree intervals', async () => { + // Get current counter value. + let raw = await chain.db.tree.get(nameHash); + let ns = NameState.decode(raw); + let counter = ns.data[0]; + + // Increment counter and commit one tree interval. + send(await wallet.sendUpdate(name, Buffer.from([++counter])), mempool); + await mineBlocks(treeInterval, mempool); + + // Tree and txn are synced due to tree commitment. + assert.bufferEqual(chain.db.tree.rootHash(), chain.db.txn.rootHash()); + + // Increment counter and confirm, but do not advance to tree interval. + send(await wallet.sendUpdate(name, Buffer.from([++counter])), mempool); + await mineBlocks(1, mempool); + + // The txn is updated, but the tree is still in last-committed state + assert.notBufferEqual(chain.db.tree.rootHash(), chain.db.txn.rootHash()); + raw = await chain.db.txn.get(nameHash); + ns = NameState.decode(raw); + assert.bufferEqual(ns.data, Buffer.from([counter])); + raw = await chain.db.tree.get(nameHash); + ns = NameState.decode(raw); + assert.bufferEqual(ns.data, Buffer.from([counter - 1])); + + // Save + const txnRootBefore = chain.db.txn.rootHash(); + const treeRootBefore = chain.db.tree.rootHash(); + + // Compact + const before = await fs.stat(treePath); + await chain.compactTree(); + const after = await fs.stat(treePath); + assert(before.size > after.size); + + // Check + assert.bufferEqual(txnRootBefore, chain.db.txn.rootHash()); + assert.bufferEqual(treeRootBefore, chain.db.tree.rootHash()); + assert.notBufferEqual(chain.db.tree.rootHash(), chain.db.txn.rootHash()); + raw = await chain.db.txn.get(nameHash); + ns = NameState.decode(raw); + assert.bufferEqual(ns.data, Buffer.from([counter])); + raw = await chain.db.tree.get(nameHash); + ns = NameState.decode(raw); + assert.bufferEqual(ns.data, Buffer.from([counter - 1])); + }); + + it(`should ${prune ? '' : 'not '}have pruned chain`, async () => { + // Sanity check. Everything worked on a chain that is indeed pruning. + // Start at height 2 because pruneAfterHeight == 1 + for (let i = 2; i <= chain.height; i++) { + const entry = await chain.getEntry(i); + const block = await chain.getBlock(entry.hash); + + if (prune && i <= chain.height - network.block.keepBlocks) + assert.strictEqual(block, null); + else + assert(block); + } + }); + }); + } +}); From eb0e8960a86f8afbcb63a500e53938fa3f32b413 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Tue, 14 Dec 2021 15:29:02 -0500 Subject: [PATCH 06/23] chain: save tree root to DB before compacting for failure recovery --- lib/blockchain/chain.js | 8 +++-- lib/blockchain/chaindb.js | 8 +++++ test/chain-tree-compaction-test.js | 47 ++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 0972616b1..1d4e5eb06 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -130,9 +130,11 @@ class Chain extends AsyncEmitter { const {treeInterval} = this.network.names; - // Current state of the tree, freshly loaded from disk. - // It will be in the most recently-committed state, - // which was at the last tree interval. There might have been + // Current state of the tree, loaded from chain database and + // injected in chainDB.open(). It should be in the most + // recently-committed state, which should have been at the last + // tree interval. We might also need to recover from a + // failed compactTree() operation. Either way, there might have been // new blocks added to the chain since then. const currentRoot = this.db.treeRoot(); diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 37f13bf1d..72bb19b31 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -956,6 +956,14 @@ class ChainDB { */ async compactTree(root) { + // Before doing anything to the tree, + // save the target tree root hash to chain database. + // If the tree data gets out of sync or corrupted + // the chain database knows where to resync the tree from. + this.start(); + this.put(layout.s.encode(), root); + await this.commit(); + // Rewind tree to historical commitment await this.tree.inject(root); diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index 780630c81..b966cec43 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -307,6 +307,53 @@ describe('Tree Compacting', function() { assert.bufferEqual(ns.data, Buffer.from([counter - 1])); }); + it('should recover if aborted', async () => { + // Get current counter value. + let raw = await chain.db.tree.get(nameHash); + let ns = NameState.decode(raw); + const counter = ns.data[0]; + + // Add 20 tree intervals + for (let i = counter; i <= counter + 20; i++) { + send(await wallet.sendUpdate(name, Buffer.from([i])), mempool); + await mineBlocks(treeInterval, mempool); + } + + const before = await fs.stat(treePath); + + // Rewind the tree 6 intervals and compact, but do not sync to tip yet. + const entry = await chain.getEntry(chain.height - 6 * treeInterval); + await chain.db.compactTree(entry.treeRoot); + + // Confirm tree state has been rewound + assert.notBufferEqual(chain.db.tree.rootHash(), chain.tip.treeRoot); + + // Oops, we abort before calling chain.syncTree() + await miner.close(); + await chain.close(); + await blocks.close(); + + // Restart -- chainDB used to open tree with what it thought + // was the latest tree state (saved in levelDB). If the actual + // tree on disk was still 6 intervals behind, chain.open() would + // fail with `Missing node` error. The updated logic relies on the + // tree itself to find its own state (saved in Meta nodes) then + // chain.syncTree() will catch it up from there to tip. + await blocks.open(); + await chain.open(); + await miner.open(); + + // Tree was compacted + const after = await fs.stat(treePath); + assert(before.size > after.size); + + // Tree was re-synced automatically to chain tip on restart + assert.bufferEqual(chain.db.tree.rootHash(), chain.tip.treeRoot); + raw = await chain.db.tree.get(nameHash); + ns = NameState.decode(raw); + assert.bufferEqual(ns.data, Buffer.from([counter + 20])); + }); + it(`should ${prune ? '' : 'not '}have pruned chain`, async () => { // Sanity check. Everything worked on a chain that is indeed pruning. // Start at height 2 because pruneAfterHeight == 1 From 3c13fbc17a10eaf13adec44f9d09a2d83d9d47bc Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Fri, 7 Jan 2022 12:14:01 -0500 Subject: [PATCH 07/23] chain: do not compact tree when chain is still short --- lib/blockchain/chain.js | 3 +++ test/chain-tree-compaction-test.js | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 1d4e5eb06..01e49960b 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2090,6 +2090,9 @@ class Chain extends AsyncEmitter { */ async compactTree() { + if (this.height < this.network.block.keepBlocks) + throw new Error('Chain is too short to compact tree.'); + const unlock = await this.locker.lock(); this.logger.info('Compacting Urkel Tree...'); diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index b966cec43..7bd8a2546 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -104,6 +104,13 @@ describe('Tree Compacting', function() { network.block.pruneAfterHeight = oldpruneAfterHeight; }); + it('should throw if chain is too short to compact', async () => { + await assert.rejects( + chain.compactTree(), + {message: 'Chain is too short to compact tree.'} + ); + }); + it('should fund wallet', async () => { miner.addresses.length = 0; miner.addAddress(wallet.getReceive()); From d71393f0cb99b9492c45ec3b34ccc8c6f1cd7ae1 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Fri, 7 Jan 2022 13:41:35 -0500 Subject: [PATCH 08/23] node: parse config arg to compact tree on launch --- CHANGELOG.md | 5 + lib/blockchain/chain.js | 18 +++- lib/node/fullnode.js | 3 +- test/chain-tree-compaction-test.js | 159 ++++++++++++++++++++++++++--- 4 files changed, 169 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25fdad077..626e1eb95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ sent to peers in the version packet. Strings must not contain slashes and total user-agent string must be 255 characters or less. +- `FullNode` parses new configuration option `--compact-tree` which will compact +the Urkel Tree only when the node first opens. This is the preferred method +because it will run before the node connects to the network and the processing +time will not affect peers. + ## v3.0.0 **When upgrading to this version of hsd you must pass diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 01e49960b..64c545d94 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -100,8 +100,16 @@ class Chain extends AsyncEmitter { this.setDeploymentState(state); - if (!this.options.spv) - await this.syncTree(); + if (this.options.compactTree) { + if (this.options.spv) + throw new Error('Cannot compact tree in SPV mode.'); + + // Will call syncTree() after compaction. + await this.compactTree(); + } else { + if (!this.options.spv) + await this.syncTree(); + } this.logger.memory(); @@ -3697,6 +3705,7 @@ class ChainOptions { this.maxOrphans = 20; this.checkpoints = true; this.chainMigrate = -1; + this.compactTree = false; if (options) this.fromOptions(options); @@ -3809,6 +3818,11 @@ class ChainOptions { this.chainMigrate = options.chainMigrate; } + if (options.compactTree != null) { + assert(typeof options.compactTree === 'boolean'); + this.compactTree = options.compactTree; + } + if (this.spv || this.memory) this.treePrefix = null; diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index b9cc78805..00d078418 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -66,7 +66,8 @@ class FullNode extends Node { entryCache: this.config.uint('entry-cache'), chainMigrate: this.config.uint('chain-migrate'), indexTX: this.config.bool('index-tx'), - indexAddress: this.config.bool('index-address') + indexAddress: this.config.bool('index-address'), + compactTree: this.config.bool('compact-tree') }); // Fee estimation. diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index 7bd8a2546..dd418e454 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -11,6 +11,9 @@ const blockstore = require('../lib/blockstore'); const MemWallet = require('./util/memwallet'); const rules = require('../lib/covenants/rules'); const NameState = require('../lib/covenants/namestate'); +const Address = require('../lib/primitives/address'); +const FullNode = require('../lib/node/fullnode'); +const SPVNode = require('../lib/node/spvnode'); const network = Network.get('regtest'); const { @@ -20,6 +23,22 @@ const { } = network.names; describe('Tree Compacting', function() { + const oldKeepBlocks = network.block.keepBlocks; + const oldpruneAfterHeight = network.block.pruneAfterHeight; + + before(async () => { + // Copy the 1:8 ratio from mainnet + network.block.keepBlocks = treeInterval * 8; + + // Ensure old blocks are pruned right away + network.block.pruneAfterHeight = 1; + }); + + after(async () => { + network.block.keepBlocks = oldKeepBlocks; + network.block.pruneAfterHeight = oldpruneAfterHeight; + }); + for (const prune of [true, false]) { describe(`Chain: ${prune ? 'Pruning' : 'Archival'}`, function() { const prefix = path.join( @@ -74,20 +93,10 @@ describe('Tree Compacting', function() { wallet.addBlock(entry, block.txs); } } - - const oldKeepBlocks = network.block.keepBlocks; - const oldpruneAfterHeight = network.block.pruneAfterHeight; - let name, nameHash; const treeRoots = []; before(async () => { - // Copy the 1:8 ratio from mainnet - network.block.keepBlocks = treeInterval * 8; - - // Ensure old blocks are pruned right away - network.block.pruneAfterHeight = 1; - await blocks.ensure(); await blocks.open(); await chain.open(); @@ -99,9 +108,6 @@ describe('Tree Compacting', function() { await chain.close(); await blocks.close(); await fs.rimraf(prefix); - - network.block.keepBlocks = oldKeepBlocks; - network.block.pruneAfterHeight = oldpruneAfterHeight; }); it('should throw if chain is too short to compact', async () => { @@ -376,4 +382,131 @@ describe('Tree Compacting', function() { }); }); } + + describe('SPV', function() { + it('should refuse to compact tree via RPC', async () => { + const prefix = path.join( + os.tmpdir(), + `hsd-tree-compacting-test-${Date.now()}` + ); + + const node = new SPVNode({ + prefix, + network: 'regtest', + memory: false + }); + + await node.ensure(); + await node.open(); + + await assert.rejects( + node.rpc.compactTree([]), + {message: 'Cannot compact tree in SPV mode.'} + ); + + await node.close(); + }); + }); + + describe('Full Node', function() { + it('should throw if chain is too short to compact on launch', async () => { + const prefix = path.join( + os.tmpdir(), + `hsd-tree-compacting-test-${Date.now()}` + ); + + const node = new FullNode({ + prefix, + network: 'regtest', + memory: false, + compactTree: true + }); + + await node.ensure(); + + await assert.rejects( + node.open(), + {message: 'Chain is too short to compact tree.'} + ); + }); + + it('should throw if chain is too short to compact via RPC', async () => { + const prefix = path.join( + os.tmpdir(), + `hsd-tree-compacting-test-${Date.now()}` + ); + + const node = new FullNode({ + prefix, + network: 'regtest', + memory: false + }); + + await node.ensure(); + await node.open(); + + await assert.rejects( + node.rpc.compactTree([]), + {message: 'Chain is too short to compact tree.'} + ); + + await node.close(); + }); + + it('should compact tree on launch', async () => { + const prefix = path.join( + os.tmpdir(), + `hsd-tree-compacting-test-${Date.now()}` + ); + const treePath = path.join(prefix, 'regtest', 'tree', '0000000001'); + + // Fresh start + let node = new FullNode({ + prefix, + network: 'regtest', + memory: false + }); + await node.ensure(); + await node.open(); + const fresh = await fs.stat(treePath); + + // Grow + const waiter = new Promise((resolve) => { + node.on('connect', (entry) => { + if (entry.height >= 300) + resolve(); + }); + }); + await node.rpc.generateToAddress( + [300, new Address().toString('regtest')] + ); + await waiter; + + // Tree has grown + const grown = await fs.stat(treePath); + assert(fresh.size < grown.size); + + // Relaunch with compaction argument + await node.close(); + node = new FullNode({ + prefix, + network: 'regtest', + memory: false, + compactTree: true + }); + await node.open(); + + // Tree is compacted + const compacted = await fs.stat(treePath); + assert(compacted.size < grown.size); + + // Bonus: since there are no namestate updates in this test, + // all the nodes committed to the tree during "growth" are identically + // empty. When we compact, only the original empty node will remain. + assert.strictEqual(fresh.size, compacted.size); + + // done + await node.close(); + }); + }); }); From c859c68aae690e7e34df5731e30564572ae88897 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Fri, 1 Apr 2022 13:55:58 -0400 Subject: [PATCH 09/23] test: remove rpc compacttree RPC command removed by deleting commit 66b6437df68b882f0bd539e120f96def9af92637 This commit cleans up the tests and CHANGELOG --- CHANGELOG.md | 13 ++++++----- test/chain-tree-compaction-test.js | 35 ++++-------------------------- 2 files changed, 11 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 626e1eb95..0392843ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,15 +3,16 @@ ## unreleased ### Node changes - - `FullNode` and `SPVNode` now accept the option `--agent` which adds a string + - `FullNode` and `SPVNode` now accept the option `--agent` which adds a string to the user-agent of the node (which will already contain hsd version) and is sent to peers in the version packet. Strings must not contain slashes and total - user-agent string must be 255 characters or less. + user-agent string must be 255 charact -- `FullNode` parses new configuration option `--compact-tree` which will compact -the Urkel Tree only when the node first opens. This is the preferred method -because it will run before the node connects to the network and the processing -time will not affect peers. + - `FullNode` parses new configuration option `--compact-tree` which will compact + the Urkel Tree when the node first opens, by deleting historical data. It will + keep up to the last 288 blocks worth of tree data on disk (7-8 tree intervals) + exposing the node to a similar deep reorganization vulnerability as a + chain-pruning node. ## v3.0.0 diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index dd418e454..9923e4504 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -384,7 +384,7 @@ describe('Tree Compacting', function() { } describe('SPV', function() { - it('should refuse to compact tree via RPC', async () => { + it('should ignore compact tree option', async () => { const prefix = path.join( os.tmpdir(), `hsd-tree-compacting-test-${Date.now()}` @@ -393,17 +393,13 @@ describe('Tree Compacting', function() { const node = new SPVNode({ prefix, network: 'regtest', - memory: false + memory: false, + compactTree: true, + prune: true // also ignored }); await node.ensure(); await node.open(); - - await assert.rejects( - node.rpc.compactTree([]), - {message: 'Cannot compact tree in SPV mode.'} - ); - await node.close(); }); }); @@ -430,29 +426,6 @@ describe('Tree Compacting', function() { ); }); - it('should throw if chain is too short to compact via RPC', async () => { - const prefix = path.join( - os.tmpdir(), - `hsd-tree-compacting-test-${Date.now()}` - ); - - const node = new FullNode({ - prefix, - network: 'regtest', - memory: false - }); - - await node.ensure(); - await node.open(); - - await assert.rejects( - node.rpc.compactTree([]), - {message: 'Chain is too short to compact tree.'} - ); - - await node.close(); - }); - it('should compact tree on launch', async () => { const prefix = path.join( os.tmpdir(), From ab2d0364af7af06d1c86c79c2f97db50e64399aa Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Tue, 12 Apr 2022 16:22:49 -0400 Subject: [PATCH 10/23] test: cover tree recovery using chainDB after connect failure --- test/chain-tree-compaction-test.js | 122 ++++++++++++++++++++++++++--- 1 file changed, 109 insertions(+), 13 deletions(-) diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index 9923e4504..642b535b1 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -23,21 +23,21 @@ const { } = network.names; describe('Tree Compacting', function() { - const oldKeepBlocks = network.block.keepBlocks; - const oldpruneAfterHeight = network.block.pruneAfterHeight; + const oldKeepBlocks = network.block.keepBlocks; + const oldpruneAfterHeight = network.block.pruneAfterHeight; - before(async () => { - // Copy the 1:8 ratio from mainnet - network.block.keepBlocks = treeInterval * 8; + before(async () => { + // Copy the 1:8 ratio from mainnet + network.block.keepBlocks = treeInterval * 8; - // Ensure old blocks are pruned right away - network.block.pruneAfterHeight = 1; - }); + // Ensure old blocks are pruned right away + network.block.pruneAfterHeight = 1; + }); - after(async () => { - network.block.keepBlocks = oldKeepBlocks; - network.block.pruneAfterHeight = oldpruneAfterHeight; - }); + after(async () => { + network.block.keepBlocks = oldKeepBlocks; + network.block.pruneAfterHeight = oldpruneAfterHeight; + }); for (const prune of [true, false]) { describe(`Chain: ${prune ? 'Pruning' : 'Archival'}`, function() { @@ -47,11 +47,13 @@ describe('Tree Compacting', function() { ); const treePath = path.join(prefix, 'tree', '0000000001'); + // This is the chain we are testing, + // we are going to compact its tree + // and try to corrupt its database. const blocks = blockstore.create({ prefix, network }); - const chain = new Chain({ memory: false, prefix, @@ -60,6 +62,20 @@ describe('Tree Compacting', function() { prune }); + // This second, in-memory chain is our control. + // Every block we add to the test chain will also + // be added to the memChain so we can check for consensus + // failures or other inconsistencies caused by tree compacting. + const memBlocks = blockstore.create({ + memory: true, + network + }); + const memChain = new Chain({ + memory: true, + blocks: memBlocks, + network + }); + const miner = new Miner({chain}); const cpu = miner.cpu; @@ -84,15 +100,18 @@ describe('Tree Compacting', function() { job.refresh(); const block = await job.mineAsync(); const entry = await chain.add(block); + assert(await memChain.add(block)); wallet.addBlock(entry, block.txs); for (num--; num > 0; num--) { const job = await cpu.createJob(); const block = await job.mineAsync(); const entry = await chain.add(block); + assert(await memChain.add(block)); wallet.addBlock(entry, block.txs); } } + let name, nameHash; const treeRoots = []; @@ -101,6 +120,9 @@ describe('Tree Compacting', function() { await blocks.open(); await chain.open(); await miner.open(); + + await memBlocks.open(); + await memChain.open(); }); after(async () => { @@ -108,6 +130,9 @@ describe('Tree Compacting', function() { await chain.close(); await blocks.close(); await fs.rimraf(prefix); + + await memChain.close(); + await memBlocks.close(); }); it('should throw if chain is too short to compact', async () => { @@ -367,6 +392,75 @@ describe('Tree Compacting', function() { assert.bufferEqual(ns.data, Buffer.from([counter + 20])); }); + it('should recover from failure during block connect', async () => { + // Get current counter value. + let raw = await chain.db.tree.get(nameHash); + let ns = NameState.decode(raw); + const counter = ns.data[0]; + + // Approach next tree interval so next block will commit. + const numBlocks = chain.height % treeInterval; + await mineBlocks(numBlocks); + + // Prepare UPDATE + const update = await wallet.createUpdate( + name, + Buffer.from([counter + 1]) + ); + + // Put actual batch-write function aside + const CHAIN_DB_COMMIT = chain.db.commit; + + // Current tree root before crash + const treeRoot = chain.db.treeRoot(); + + // Implement bug where node crashes before database batch is written. + // When the next block is connected, it should successfully write + // new data to the Urkel Tree but fail to write data to blockstore + // or levelDB indexes. + chain.db.commit = async () => { + // Tree root has been updated inside Urkel + const newRoot1 = chain.db.treeRoot(); + assert(!treeRoot.equals(newRoot1)); + + // Reset batch, otherwise assert(!this.current) fails + chain.db.drop(); + // Node has crashed... + await chain.close(); + }; + + // Update name and attempt to confirm + send(update, mempool); + // Will "crash" node before completing operation + await mineBlocks(1, mempool); + assert(!chain.opened); + + // Restore proper batch-write function + chain.db.commit = CHAIN_DB_COMMIT; + + // Restarting chain should recover from crash + await chain.open(); + + // Tree root has been restored from pre-crash state + const newRoot2 = chain.db.treeRoot(); + assert(treeRoot.equals(newRoot2)); + + // Try that update again with healthy chainDB + send(update, mempool); + await mineBlocks(1, mempool); + + // Tree has been updated but tree root won't be committed + // to a block header until the next block. + assert(!chain.db.tree.rootHash().equals(chain.tip.treeRoot)); + await mineBlocks(1); + + // Everything is in order + assert.bufferEqual(chain.db.tree.rootHash(), chain.tip.treeRoot); + raw = await chain.db.tree.get(nameHash); + ns = NameState.decode(raw); + assert.bufferEqual(ns.data, Buffer.from([counter + 1])); + }); + it(`should ${prune ? '' : 'not '}have pruned chain`, async () => { // Sanity check. Everything worked on a chain that is indeed pruning. // Start at height 2 because pruneAfterHeight == 1 @@ -427,6 +521,8 @@ describe('Tree Compacting', function() { }); it('should compact tree on launch', async () => { + this.timeout(10000); + const prefix = path.join( os.tmpdir(), `hsd-tree-compacting-test-${Date.now()}` From e1811524b9eba694e586b8770c3ccc4befa8652f Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Tue, 12 Apr 2022 16:28:32 -0400 Subject: [PATCH 11/23] chain: call syncTree() in open() after compactTree() --- lib/blockchain/chain.js | 17 ++++++----------- test/chain-tree-compaction-test.js | 10 +++++----- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 64c545d94..2fa85cbf7 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -100,15 +100,13 @@ class Chain extends AsyncEmitter { this.setDeploymentState(state); - if (this.options.compactTree) { - if (this.options.spv) - throw new Error('Cannot compact tree in SPV mode.'); + if (!this.options.spv) { + if (this.options.compactTree) + await this.compactTree(); - // Will call syncTree() after compaction. - await this.compactTree(); - } else { - if (!this.options.spv) - await this.syncTree(); + // Replay the blockchain from last committed tree root + // back up to the tip to rebuild tree and txn. + await this.syncTree(); } this.logger.memory(); @@ -2130,9 +2128,6 @@ class Chain extends AsyncEmitter { try { // Rewind Urkel Tree and delete all historical state. await this.db.compactTree(entry.treeRoot); - - // Replay the blockchain back up to the tip to rebuild tree. - return await this.syncTree(); } finally { unlock(); } diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index 642b535b1..8dff3b4ac 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -201,6 +201,7 @@ describe('Tree Compacting', function() { it('should compact tree', async () => { const before = await fs.stat(treePath); await chain.compactTree(); + await chain.syncTree(); const after = await fs.stat(treePath); // Urkel Tree should be smaller now. @@ -271,6 +272,7 @@ describe('Tree Compacting', function() { // it shouldn't break anything. const before = await fs.stat(treePath); await chain.compactTree(); + await chain.syncTree(); const after = await fs.stat(treePath); // Should be no change @@ -330,6 +332,7 @@ describe('Tree Compacting', function() { // Compact const before = await fs.stat(treePath); await chain.compactTree(); + await chain.syncTree(); const after = await fs.stat(treePath); assert(before.size > after.size); @@ -371,11 +374,8 @@ describe('Tree Compacting', function() { await chain.close(); await blocks.close(); - // Restart -- chainDB used to open tree with what it thought - // was the latest tree state (saved in levelDB). If the actual - // tree on disk was still 6 intervals behind, chain.open() would - // fail with `Missing node` error. The updated logic relies on the - // tree itself to find its own state (saved in Meta nodes) then + // Restart -- chainDB will open tree with what it thinks + // is the latest tree state (saved in levelDB). Then // chain.syncTree() will catch it up from there to tip. await blocks.open(); await chain.open(); From 9a8eaa9a29cbe22d7bb8a8d310e31bf6d794dcc7 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Mon, 25 Apr 2022 15:18:03 -0400 Subject: [PATCH 12/23] test: clean up tmpdirs in compact tree test --- test/chain-tree-compaction-test.js | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index 8dff3b4ac..edb9dd430 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -478,12 +478,20 @@ describe('Tree Compacting', function() { } describe('SPV', function() { - it('should ignore compact tree option', async () => { - const prefix = path.join( + let prefix; + + beforeEach(async () => { + prefix = path.join( os.tmpdir(), `hsd-tree-compacting-test-${Date.now()}` ); + }); + afterEach(async () => { + await fs.rimraf(prefix); + }); + + it('should ignore compact tree option', async () => { const node = new SPVNode({ prefix, network: 'regtest', @@ -499,12 +507,20 @@ describe('Tree Compacting', function() { }); describe('Full Node', function() { - it('should throw if chain is too short to compact on launch', async () => { - const prefix = path.join( + let prefix; + + beforeEach(async () => { + prefix = path.join( os.tmpdir(), `hsd-tree-compacting-test-${Date.now()}` ); + }); + afterEach(async () => { + await fs.rimraf(prefix); + }); + + it('should throw if chain is too short to compact on launch', async () => { const node = new FullNode({ prefix, network: 'regtest', @@ -523,10 +539,6 @@ describe('Tree Compacting', function() { it('should compact tree on launch', async () => { this.timeout(10000); - const prefix = path.join( - os.tmpdir(), - `hsd-tree-compacting-test-${Date.now()}` - ); const treePath = path.join(prefix, 'regtest', 'tree', '0000000001'); // Fresh start From a221f16c1648f36f8abec1e677204dd9076a99b8 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Thu, 5 May 2022 13:24:23 -0400 Subject: [PATCH 13/23] chain: do not sync tree deeper than pruned node could support --- lib/blockchain/chain.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 2fa85cbf7..6d1b84ca6 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -167,6 +167,7 @@ class Chain extends AsyncEmitter { break; last -= treeInterval; + assert(last > this.height - this.network.block.keepBlocks); entry = await this.db.getEntryByHeight(last + 1); assert(entry); } From 36682b6950a8010b615ec2463590df37427e4ab8 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Thu, 19 May 2022 17:38:43 +0400 Subject: [PATCH 14/23] pkg: update urkel to v1.0.1 --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 989dbb5ef..03d7f1beb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,7 +35,7 @@ "goosig": "~0.10.0", "hs-client": "~0.0.11", "n64": "~0.2.10", - "urkel": "~1.0.0" + "urkel": "~1.0.1" }, "bin": { "hs-seeder": "bin/hs-seeder", @@ -461,9 +461,9 @@ } }, "node_modules/urkel": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/urkel/-/urkel-1.0.0.tgz", - "integrity": "sha512-N1dvng7KMCr9XXKPI8yIaCwlggNvXOpY6OxYb42cuLHPiabe7QCknsU6BWklcBLUHpouyHZ/KHCceyKRrI5e7A==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/urkel/-/urkel-1.0.1.tgz", + "integrity": "sha512-/ul3w/hvvGzppHqdpDAcEFe8kS1hi6ty5h7oQalIlVLwPNUJ/tz/h7KtIyNRC0+u7ANryq2Aw96N8snq+VYEOg==", "dependencies": { "bfile": "~0.2.1", "bmutex": "~0.1.6", @@ -748,9 +748,9 @@ } }, "urkel": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/urkel/-/urkel-1.0.0.tgz", - "integrity": "sha512-N1dvng7KMCr9XXKPI8yIaCwlggNvXOpY6OxYb42cuLHPiabe7QCknsU6BWklcBLUHpouyHZ/KHCceyKRrI5e7A==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/urkel/-/urkel-1.0.1.tgz", + "integrity": "sha512-/ul3w/hvvGzppHqdpDAcEFe8kS1hi6ty5h7oQalIlVLwPNUJ/tz/h7KtIyNRC0+u7ANryq2Aw96N8snq+VYEOg==", "requires": { "bfile": "~0.2.1", "bmutex": "~0.1.6", diff --git a/package.json b/package.json index 1a6610de1..da67c1338 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "goosig": "~0.10.0", "hs-client": "~0.0.11", "n64": "~0.2.10", - "urkel": "~1.0.0" + "urkel": "~1.0.1" }, "devDependencies": { "bmocha": "^2.1.5" From 7adfc3c8b3499f4a54b37a8bb3a4b53d07e830ed Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Thu, 19 May 2022 19:19:23 +0400 Subject: [PATCH 15/23] chaindb: add tree state with compaction and migration. --- lib/blockchain/chain.js | 2 +- lib/blockchain/chaindb.js | 90 ++++++++++++++-- lib/blockchain/migrations.js | 52 ++++++++- test/chain-migration-test.js | 167 +++++++++++++++++++++++++++++ test/chain-tree-compaction-test.js | 2 +- 5 files changed, 301 insertions(+), 12 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 6d1b84ca6..cd592d639 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2128,7 +2128,7 @@ class Chain extends AsyncEmitter { try { // Rewind Urkel Tree and delete all historical state. - await this.db.compactTree(entry.treeRoot); + await this.db.compactTree(entry); } finally { unlock(); } diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 72bb19b31..ad5af47ea 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -52,7 +52,7 @@ class ChainDB { this.db = bdb.create(this.options); this.name = 'chain'; - this.version = 2; + this.version = 3; this.tree = new Tree({ hash: blake2b, bits: 256, @@ -61,6 +61,7 @@ class ChainDB { initCacheSize: -1 }); this.txn = this.tree.txn(); + this.treeState = new TreeState(); this.stateCache = new StateCache(this.network); this.state = new ChainState(); this.field = new BitField(); @@ -117,10 +118,16 @@ class ChainDB { // Grab the current tree state. if (!this.options.spv) { - const root = await this.db.get(layout.s.encode()); - assert(root && root.length === 32); + const treeState = await this.getTreeState(); + assert(treeState); + this.treeState = treeState; - await this.tree.inject(root); + if (treeState.compactionHeight !== 0) { + this.logger.warning( + `Tree is compacted at ${treeState.compactionHeight}`); + } + + await this.tree.inject(treeState.treeRoot); } // Read bitfield. @@ -661,6 +668,20 @@ class ChainDB { return ChainState.decode(data); } + /** + * Retrieve tree state from the tree record. + * @returns {Promise} + */ + + async getTreeState() { + const data = await this.db.get(layout.s.encode()); + + if (!data) + return null; + + return TreeState.decode(data); + } + /** * Write genesis block to database. * @returns {Promise} @@ -951,21 +972,24 @@ class ChainDB { * Compact the Urkel Tree. * Removes all historical state and all data not * linked directly to the provided root node hash. - * @param {Hash} root + * @param {ChainEntry} entry * @returns {Promise} */ - async compactTree(root) { + async compactTree(entry) { // Before doing anything to the tree, // save the target tree root hash to chain database. // If the tree data gets out of sync or corrupted // the chain database knows where to resync the tree from. this.start(); - this.put(layout.s.encode(), root); + this.treeState.treeRoot = entry.treeRoot; + this.treeState.compactionRoot = entry.treeRoot; + this.treeState.compactionHeight = entry.height; + this.put(layout.s.encode(), this.treeState.encode()); await this.commit(); // Rewind tree to historical commitment - await this.tree.inject(root); + await this.tree.inject(entry.treeRoot); // Delete historical data await this.tree.compact(); @@ -2007,7 +2031,8 @@ class ChainDB { else await this.txn.commit(); - this.put(layout.s.encode(), this.tree.rootHash()); + this.treeState.treeRoot = this.tree.rootHash(); + this.put(layout.s.encode(), this.treeState.encode()); } } @@ -2646,6 +2671,53 @@ class CacheUpdate { } } +/** + * Tree related state. + */ + +class TreeState extends bio.Struct { + /** + * Create tree state. + * @constructor + * @ignore + */ + + constructor() { + super(); + this.treeRoot = consensus.ZERO_HASH; + this.compactionRoot = consensus.ZERO_HASH; + this.compactionHeight = 0; + } + + inject(state) { + this.treeRoot = state.treeRoot; + this.compactionHeight = state.compactionHeight; + this.compactionRoot = state.compactionRoot; + + return this; + } + + getSize() { + return 68; + } + + write(bw) { + bw.writeHash(this.treeRoot); + bw.writeHash(this.compactionRoot); + bw.writeU32(this.compactionHeight); + + return bw; + } + + read(br) { + this.treeRoot = br.readHash(); + this.compactionRoot = br.readHash(); + this.compactionHeight = br.readU32(); + + return this; + } +} + /* * Helpers */ diff --git a/lib/blockchain/migrations.js b/lib/blockchain/migrations.js index 0e71d0bde..1cfcd49b9 100644 --- a/lib/blockchain/migrations.js +++ b/lib/blockchain/migrations.js @@ -389,6 +389,54 @@ class MigrateBlockStore extends AbstractMigration { } }; +/** + * Migrate Tree State + */ + +class MigrateTreeState extends AbstractMigration { + /** + * Create tree state migrator + * @constructor + * @param {ChainMigrator} options + */ + + constructor(options) { + super(options); + + this.options = options; + this.logger = options.logger.context('chain-migration-tree-state'); + this.db = options.db; + this.ldb = options.ldb; + } + + async check() { + if (this.options.spv) + return types.FAKE_MIGRATE; + + return types.MIGRATE; + } + + async migrate(b) { + const hash = await this.ldb.get(layout.s.encode()); + assert(hash && hash.length === 32); + + // new tree root + // see chaindb.js TreeState + const buff = Buffer.alloc(68); + hash.copy(buff); + + this.db.writeVersion(b, 3); + b.put(layout.s.encode(), buff); + } + + static info() { + return { + name: 'Migrate Tree State', + description: 'Add compaction information to the tree state.' + }; + } +} + /** * Chain Migrator * @alias module:blockchain.ChainMigrator @@ -528,12 +576,14 @@ exports = ChainMigrator; exports.migrations = { 0: MigrateMigrations, 1: MigrateChainState, - 2: MigrateBlockStore + 2: MigrateBlockStore, + 3: MigrateTreeState }; // Expose migrations exports.MigrateChainState = MigrateChainState; exports.MigrateMigrations = MigrateMigrations; exports.MigrateBlockStore = MigrateBlockStore; +exports.MigrateTreeState = MigrateTreeState; module.exports = exports; diff --git a/test/chain-migration-test.js b/test/chain-migration-test.js index a7c8e1671..9ab3bc0cb 100644 --- a/test/chain-migration-test.js +++ b/test/chain-migration-test.js @@ -24,6 +24,9 @@ const chainFlagError = (id) => { return `Restart with \`hsd --chain-migrate=${id}\``; }; +const VERSION_ERROR = 'Database version mismatch for database: "chain".' + + ' Please run a data migration before opening.'; + describe('Chain Migrations', function() { describe('General', function() { const location = testdir('migrate-chain-general'); @@ -164,14 +167,21 @@ describe('Chain Migrations', function() { it('should only migrate the migration states with flag', async () => { // set the oldest state + // NOTE: Every new migration would need to set oldest state. const genesisBlock = await chainDB.getBlock(0); const genesisHash = genesisBlock.hash(); const genesisUndo = await chainDB.getUndoCoins(genesisHash); const b = ldb.batch(); b.del(layout.M.encode()); + + // Migration blockstore b.put(layout.b.encode(genesisHash), genesisBlock.encode()); b.put(layout.u.encode(genesisHash), genesisUndo.encode()); + + // migration 3 - MigrateTreeState + b.put(layout.s.encode(), Buffer.alloc(32, 0)); + writeVersion(b, 'chain', 1); await b.write(); @@ -273,6 +283,9 @@ describe('Chain Migrations', function() { chainDB = chain.db; ldb = chainDB.db; + // Chain Version was 2 at that time. + chainDB.version = 2; + await store.open(); ChainMigrator.migrations = testMigrations; }); @@ -808,6 +821,160 @@ describe('Chain Migrations', function() { } }); }); + + describe('Migration Tree State (integration)', function() { + const location = testdir('migrate-tree-state'); + const migrationsBAK = ChainMigrator.migrations; + const store = BlockStore.create({ + memory: false, + prefix: location, + network + }); + + const workers = new WorkerPool({ + enabled: true, + size: 2 + }); + + const chainOptions = { + prefix: location, + memory: false, + blocks: store, + network, + workers + }; + + let chain, chaindb, ldb, miner, cpu; + before(async () => { + ChainMigrator.migrations = {}; + await fs.mkdirp(location); + await store.ensure(); + await workers.open(); + }); + + after(async () => { + ChainMigrator.migrations = migrationsBAK; + await rimraf(location); + await workers.close(); + }); + + beforeEach(async () => { + chain = new Chain(chainOptions); + chaindb = chain.db; + ldb = chaindb.db; + miner = new Miner({ chain }); + cpu = miner.cpu; + + chaindb.version = 3; + + await miner.open(); + await store.open(); + }); + + afterEach(async () => { + if (chain.opened) + await chain.close(); + + await store.close(); + await miner.close(); + }); + + it('should mine 10 blocks', async () => { + await chain.open(); + + for (let i = 0; i < 10; i++) { + const block = await cpu.mineBlock(); + assert(block); + assert(await chain.add(block)); + } + }); + + it('should throw version check error', async () => { + // Previous state + await chain.open(); + const b = ldb.batch(); + b.put(layout.s.encode(), Buffer.alloc(32, 0x00)); + writeVersion(b, 'chain', 2); + await b.write(); + await chain.close(); + + let error; + try { + await chain.open(); + } catch (e) { + error = e; + } + + assert(error); + assert.strictEqual(error.message, VERSION_ERROR); + }); + + it('should enable tree state migration', async () => { + ChainMigrator.migrations = { + 0: ChainMigrator.MigrateTreeState + }; + }); + + it('should throw when new migration is available', async () => { + const expected = migrationError(ChainMigrator.migrations, [0], + chainFlagError(0)); + + let error; + try { + await chain.open(); + } catch (e) { + error = e; + } + + assert(error, 'Chain must throw an error.'); + assert.strictEqual(error.message, expected); + }); + + it('should migrate tree state', async () => { + chain.options.chainMigrate = 0; + + await chain.open(); + const state = chaindb.treeState; + assert.bufferEqual(state.encode(), Buffer.alloc(68, 0)); + }); + + it('should migrate tree state (2)', async () => { + await chain.open(); + + const state = MigrationState.decode(await ldb.get(layout.M.encode())); + state.nextMigration = 0; + + // revert migration + const b = ldb.batch(); + const root = Buffer.alloc(32, 0x01); + // revert version in DB. + writeVersion(b , 'chain', 2); + // encode wrong tree state (non default) + b.put(layout.s.encode(), root); + b.put(layout.M.encode(), state.encode()); + await b.write(); + + await chain.close(); + + chain.options.chainMigrate = 0; + let error; + try { + await chain.open(); + } catch (e) { + error = e; + } + + // Now our error should be incorrect tree (after migration) + assert(error, 'Chain must throw an error.'); + assert.strictEqual(error.message, `Missing node: ${root.toString('hex')}.`); + + const version = getVersion(await ldb.get(layout.V.encode()), 'chain'); + assert.strictEqual(version, 3); + assert.bufferEqual(chaindb.treeState.treeRoot, root); + assert.bufferEqual(chaindb.treeState.compactionRoot, Buffer.alloc(32, 0)); + assert.strictEqual(chaindb.treeState.compactionHeight, 0); + }); + }); }); function writeVersion(b, name, version) { diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index edb9dd430..0ac1520ba 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -364,7 +364,7 @@ describe('Tree Compacting', function() { // Rewind the tree 6 intervals and compact, but do not sync to tip yet. const entry = await chain.getEntry(chain.height - 6 * treeInterval); - await chain.db.compactTree(entry.treeRoot); + await chain.db.compactTree(entry); // Confirm tree state has been rewound assert.notBufferEqual(chain.db.tree.rootHash(), chain.tip.treeRoot); From 95769d90b4fb3d4b7ea49a89a63a01d6c8e98af6 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Fri, 20 May 2022 01:17:48 +0400 Subject: [PATCH 16/23] chain: add compaction interval, events and rpc call. --- lib/blockchain/chain.js | 60 +++++++-- lib/node/fullnode.js | 11 +- lib/node/rpc.js | 15 +++ test/chain-tree-compaction-test.js | 197 +++++++++++++++++++++++++---- 4 files changed, 247 insertions(+), 36 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index cd592d639..e5c02b20b 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -101,12 +101,26 @@ class Chain extends AsyncEmitter { this.setDeploymentState(state); if (!this.options.spv) { - if (this.options.compactTree) - await this.compactTree(); - - // Replay the blockchain from last committed tree root - // back up to the tip to rebuild tree and txn. - await this.syncTree(); + const {txStart} = this.network; + const startFrom = txStart + this.network.block.keepBlocks; + + if (this.options.compactTreeOnInit && this.height > startFrom) { + const {compactionHeight} = await this.db.getTreeState(); + const {compactTreeInitInterval} = this.options; + const compactFrom = compactionHeight + compactTreeInitInterval; + + if (compactFrom <= this.height) { + await this.compactTree(); + } else { + this.logger.debug( + `Tree will compact at ${compactFrom} height.`); + await this.syncTree(); + } + } else { + // Replay the blockchain from last committed tree root + // back up to the tip to rebuild tree and txn. + await this.syncTree(); + } } this.logger.memory(); @@ -2118,17 +2132,28 @@ class Chain extends AsyncEmitter { const oldestBlock = this.height - this.network.block.keepBlocks; // Distance from that block to the start of the oldest tree interval. - const toNextInterval = - this.network.names.treeInterval - - (oldestBlock % this.network.names.treeInterval); + const toNextInterval = (this.network.names.treeInterval - + (oldestBlock % this.network.names.treeInterval)) + % this.network.names.treeInterval; // Get the oldest Urkel Tree root state a pruning node can recover from. const oldestTreeIntervalStart = oldestBlock + toNextInterval + 1; const entry = await this.db.getEntryByHeight(oldestTreeIntervalStart); try { + // TODO: For RPC calls, If compaction fails while compacting + // and we never hit syncTree, we need to shut down the node + // so on restart chain can recover. + // Error can also happen in syncTree, but that means the DB + // is done for. (because restart would just retry syncTree.) + // It's fine on open, open throwing would just stop the node. + // Rewind Urkel Tree and delete all historical state. + this.emit('tree compact start', entry.treeRoot, entry); await this.db.compactTree(entry); + this.emit('tree compact end', entry.treeRoot, entry); + + await this.syncTree(); } finally { unlock(); } @@ -3701,7 +3726,8 @@ class ChainOptions { this.maxOrphans = 20; this.checkpoints = true; this.chainMigrate = -1; - this.compactTree = false; + this.compactTreeOnInit = false; + this.compactTreeInitInterval = 10000; if (options) this.fromOptions(options); @@ -3814,9 +3840,17 @@ class ChainOptions { this.chainMigrate = options.chainMigrate; } - if (options.compactTree != null) { - assert(typeof options.compactTree === 'boolean'); - this.compactTree = options.compactTree; + if (options.compactTreeOnInit != null) { + assert(typeof options.compactTreeOnInit === 'boolean'); + this.compactTreeOnInit = options.compactTreeOnInit; + } + + if (options.compactTreeInitInterval != null) { + const {keepBlocks} = this.network.block; + assert(typeof options.compactTreeInitInterval === 'number'); + assert(options.compactTreeInitInterval >= keepBlocks, + `compaction interval must not be smaller than ${keepBlocks}.`); + this.compactTreeInitInterval = options.compactTreeInitInterval; } if (this.spv || this.memory) diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index 00d078418..beb781756 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -67,7 +67,8 @@ class FullNode extends Node { chainMigrate: this.config.uint('chain-migrate'), indexTX: this.config.bool('index-tx'), indexAddress: this.config.bool('index-address'), - compactTree: this.config.bool('compact-tree') + compactTreeOnInit: this.config.bool('compact-tree-on-init'), + compactTreeInitInterval: this.config.uint('compact-tree-init-interval') }); // Fee estimation. @@ -255,6 +256,14 @@ class FullNode extends Node { this.emit('reset', tip); }); + this.chain.on('tree compact start', (treeRoot, entry) => { + this.emit('tree compact start', treeRoot, entry); + }); + + this.chain.on('tree compact start', (treeRoot, entry) => { + this.emit('tree compact end', treeRoot, entry); + }); + this.loadPlugins(); } diff --git a/lib/node/rpc.js b/lib/node/rpc.js index 91d55589c..8dc959b30 100644 --- a/lib/node/rpc.js +++ b/lib/node/rpc.js @@ -177,6 +177,7 @@ class RPC extends RPCBase { this.add('gettxout', this.getTXOut); this.add('gettxoutsetinfo', this.getTXOutSetInfo); this.add('pruneblockchain', this.pruneBlockchain); + this.add('compacttree', this.compactTree); this.add('verifychain', this.verifyChain); this.add('invalidateblock', this.invalidateBlock); @@ -1104,6 +1105,20 @@ class RPC extends RPCBase { } } + async compactTree(args, help) { + if (help || args.length !== 0) + throw new RPCError(errs.MISC_ERROR, 'compacttree'); + + if (this.chain.options.spv) + throw new RPCError(errs.MISC_ERROR, 'Cannot compact tree in SPV mode.'); + + try { + await this.chain.compactTree(); + } catch (e) { + throw new RPCError(errs.DATABASE_ERROR, e.message); + } + } + async verifyChain(args, help) { if (help || args.length > 2) { throw new RPCError(errs.MISC_ERROR, diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index 0ac1520ba..cfcd181c5 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -14,6 +14,7 @@ const NameState = require('../lib/covenants/namestate'); const Address = require('../lib/primitives/address'); const FullNode = require('../lib/node/fullnode'); const SPVNode = require('../lib/node/spvnode'); +const {forEventCondition, forEvent} = require('./util/common'); const network = Network.get('regtest'); const { @@ -477,8 +478,8 @@ describe('Tree Compacting', function() { }); } - describe('SPV', function() { - let prefix; + describe('SPV Node', function() { + let prefix, node; beforeEach(async () => { prefix = path.join( @@ -488,15 +489,20 @@ describe('Tree Compacting', function() { }); afterEach(async () => { + if (node && node.opened) + await node.close(); + + node = null; + await fs.rimraf(prefix); }); it('should ignore compact tree option', async () => { - const node = new SPVNode({ + node = new SPVNode({ prefix, network: 'regtest', memory: false, - compactTree: true, + compactTreeOnInit: true, prune: true // also ignored }); @@ -504,10 +510,27 @@ describe('Tree Compacting', function() { await node.open(); await node.close(); }); + + it('should refuse to compact tree via RPC', async () => { + node = new SPVNode({ + prefix, + network: 'regtest', + memory: false + }); + + await node.ensure(); + await node.open(); + + await assert.rejects(node.rpc.compactTree([]), { + message: 'Cannot compact tree in SPV mode.' + }); + + await node.close(); + }); }); describe('Full Node', function() { - let prefix; + let prefix, node; beforeEach(async () => { prefix = path.join( @@ -517,28 +540,150 @@ describe('Tree Compacting', function() { }); afterEach(async () => { + if (node && node.opened) { + try { + await node.close(); + } catch (e) { + console.log('Error closing..', e.message); + } + } + + node = null; + await fs.rimraf(prefix); }); - it('should throw if chain is too short to compact on launch', async () => { - const node = new FullNode({ + it('should throw if compact tree interval is too small', () => { + const {keepBlocks} = network.block; + + let error; + + try { + new FullNode({ + prefix, + network: 'regtest', + memory: false, + compactTreeOnInit: true, + compactTreeInitInterval: network.block.keepBlocks - 1 + }); + } catch (e) { + error = e; + } + + assert(error, 'FullNode must throw an error.'); + assert.strictEqual(error.message, + `compaction interval must not be smaller than ${keepBlocks}.`); + }); + + it('should not throw if compact tree interval is large enough', () => { + assert.doesNotThrow(() => { + new FullNode({ + prefix, + network: 'regtest', + memory: false, + compactTreeOnInit: true, + compactTreeInitInterval: network.block.keepBlocks + }); + }); + }); + + it('should not compact tree if chain is not long enough', async () => { + const nodeOptions = { prefix, network: 'regtest', memory: false, - compactTree: true - }); + compactTreeOnInit: true, + compactTreeInitInterval: network.block.keepBlocks + }; + + node = new FullNode(nodeOptions); await node.ensure(); - await assert.rejects( - node.open(), - {message: 'Chain is too short to compact tree.'} + let compacted = false; + node.chain.compactTree = () => compacted = true; + + await node.open(); + assert.strictEqual(compacted, false); + + const blocks = network.block.keepBlocks - 1; + const waiter = forEventCondition(node, 'connect', e => e.height >= blocks); + + await node.rpc.generateToAddress( + [blocks, new Address().toString('regtest')] ); + + await waiter; + await node.close(); + + node = new FullNode(nodeOptions); + node.chain.compactTree = () => compacted = true; + + await node.open(); + assert.strictEqual(compacted, false); + await node.close(); }); - it('should compact tree on launch', async () => { - this.timeout(10000); + it('should compact tree when chain is long enought', async () => { + this.timeout(2000); + const compactTimeout = 500; + + const nodeOptions = { + prefix, + network: 'regtest', + memory: false, + compactTreeOnInit: true, + compactTreeInitInterval: network.block.keepBlocks + }; + + node = new FullNode(nodeOptions); + + await node.ensure(); + + let compacted = false; + node.chain.compactTree = () => compacted = true; + await node.open(); + assert.strictEqual(compacted, false); + + const blocks = 100; + const waiter = forEventCondition(node, 'connect', e => e.height >= blocks); + + await node.rpc.generateToAddress( + [blocks, new Address().toString('regtest')] + ); + + await waiter; + await node.close(); + + // keepBlocks.. + node = new FullNode(nodeOptions); + + const waiterStart = forEvent(node, 'tree compact start', 1, compactTimeout); + const waiterEnd = forEvent(node, 'tree compact end', 1, compactTimeout); + await node.open(); + + const [startEvent, endEvent] = await Promise.all([ + waiterStart, + waiterEnd + ]); + + assert.strictEqual(startEvent.length, 1); + assert.strictEqual(endEvent.length, 1); + + const [rootHash, entry] = endEvent[0].values; + // We don't have anything in the tree. + assert.bufferEqual(rootHash, Buffer.alloc(32, 0x00)); + // 100 - 40 = 60 + // 60 % 5 = 0 + // So nearest one is 61. + assert.strictEqual(entry.height, 61); + + await node.close(); + }); + + it('should compact tree on launch (disk sizes)', async () => { + this.timeout(4000); const treePath = path.join(prefix, 'regtest', 'tree', '0000000001'); // Fresh start @@ -552,14 +697,13 @@ describe('Tree Compacting', function() { const fresh = await fs.stat(treePath); // Grow - const waiter = new Promise((resolve) => { - node.on('connect', (entry) => { - if (entry.height >= 300) - resolve(); - }); - }); + const blocks = 300; + const waiter = forEventCondition(node, 'connect', (entry) => { + return entry.height >= blocks; + }, 2000); + await node.rpc.generateToAddress( - [300, new Address().toString('regtest')] + [blocks, new Address().toString('regtest')] ); await waiter; @@ -569,14 +713,23 @@ describe('Tree Compacting', function() { // Relaunch with compaction argument await node.close(); + node = new FullNode({ prefix, network: 'regtest', memory: false, - compactTree: true + compactTreeOnInit: true, + compactTreeInitInterval: 100 }); + + const waiterEnd = forEvent(node, 'tree compact end', 1, 1000); await node.open(); + const [endEvent] = await waiterEnd; + const [hash, entry] = endEvent.values; + assert.bufferEqual(hash, Buffer.alloc(32, 0x00)); + assert.strictEqual(entry.height, 261); + // Tree is compacted const compacted = await fs.stat(treePath); assert(compacted.size < grown.size); From 5b89d917d0ec947e5b9353944452add56f9164dc Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Fri, 20 May 2022 20:55:03 +0400 Subject: [PATCH 17/23] chain: Add tree commit height to the tree state. Add reconstruct tree RPC call. Make TreeState safe. --- lib/blockchain/chain.js | 83 ++++++---- lib/blockchain/chaindb.js | 69 ++++++-- lib/blockchain/migrations.js | 11 +- lib/node/fullnode.js | 8 + lib/node/rpc.js | 22 +++ test/chain-migration-test.js | 9 +- test/chain-tree-compaction-test.js | 252 +++++++++++++++++++++-------- 7 files changed, 338 insertions(+), 116 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index e5c02b20b..d3e6b43e8 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -148,8 +148,6 @@ class Chain extends AsyncEmitter { async syncTree() { this.logger.info('Synchronizing Tree with block history...'); - const {treeInterval} = this.network.names; - // Current state of the tree, loaded from chain database and // injected in chainDB.open(). It should be in the most // recently-committed state, which should have been at the last @@ -158,38 +156,22 @@ class Chain extends AsyncEmitter { // new blocks added to the chain since then. const currentRoot = this.db.treeRoot(); - // Get the block that corresponds to beginning of the last tree interval. - // The tree root in this block header will be the first block that - // commits to the new tree root hash, but the transactions it contains - // have not yet been added to the tree. - let last = this.height - (this.height % treeInterval); - let entry = await this.db.getEntryByHeight(last + 1); - - // Using the current tree root hash, rewind the blockchain to the beginning - // of the corresponding tree interval. If the tree has been compacted, - // this may be SEVERAL tree intervals behind the current chain tip - // (this is required to support chain reorgs to a limited depth, - // similar to pruning nodes keeping the last 288 blocks on disk). - // We can skip this only if the blockchain is less than - // one tree interval old, OR the chain is already at the tree interval, - // meaning the tree is already in sync with the chain and there is no delta. - if (!entry) { - assert(last === 0 || last === this.height); - } else { - for (;;) { - if (entry.treeRoot.equals(currentRoot)) - break; + // We store commit height for the tree in the tree state. + // commitHeight is the height of the block that committed tree root. + // Note that the block at commitHeight has different tree root. + const treeState = await this.db.getTreeState(); + const {commitHeight} = treeState; - last -= treeInterval; - assert(last > this.height - this.network.block.keepBlocks); - entry = await this.db.getEntryByHeight(last + 1); - assert(entry); - } + // sanity check + if (commitHeight < this.height) { + const entry = await this.db.getEntryByHeight(commitHeight + 1); + assert(entry.treeRoot.equals(treeState.treeRoot)); + assert(entry.treeRoot.equals(currentRoot)); } // Replay all blocks since the last tree interval to rebuild // the `txn` which is the in-memory delta between tree interval commitments. - for (let height = last + 1; height <= this.height; height++) { + for (let height = commitHeight + 1; height <= this.height; height++) { const entry = await this.db.getEntryByHeight(height); assert(entry); @@ -2106,11 +2088,13 @@ class Chain extends AsyncEmitter { * Compact the Urkel Tree. * Removes all historical state and all data not * linked directly to the provided root node hash. - * @param {Hash} root * @returns {Promise} */ async compactTree() { + if (this.options.spv) + return; + if (this.height < this.network.block.keepBlocks) throw new Error('Chain is too short to compact tree.'); @@ -2131,10 +2115,11 @@ class Chain extends AsyncEmitter { // Oldest block available to a pruning node. const oldestBlock = this.height - this.network.block.keepBlocks; + const {treeInterval} = this.network.names; + // Distance from that block to the start of the oldest tree interval. - const toNextInterval = (this.network.names.treeInterval - - (oldestBlock % this.network.names.treeInterval)) - % this.network.names.treeInterval; + const toNextInterval = (treeInterval - (oldestBlock % treeInterval)) + % treeInterval; // Get the oldest Urkel Tree root state a pruning node can recover from. const oldestTreeIntervalStart = oldestBlock + toNextInterval + 1; @@ -2151,9 +2136,41 @@ class Chain extends AsyncEmitter { // Rewind Urkel Tree and delete all historical state. this.emit('tree compact start', entry.treeRoot, entry); await this.db.compactTree(entry); + await this.syncTree(); this.emit('tree compact end', entry.treeRoot, entry); + } finally { + unlock(); + } + } + /** + * Reconstruct the Urkel Tree. + * @returns {Promise} + */ + + async reconstructTree() { + if (this.options.spv) + return; + + if (this.options.prune) + throw new Error('Cannot reconstruct tree in pruned mode.'); + + const unlock = await this.locker.lock(); + + const treeState = await this.db.getTreeState(); + + if (treeState.compactionHeight === 0) + throw new Error('Nothing to reconstruct.'); + + // Compact all the way to the first block and + // let the syncTree do its job. + const entry = await this.db.getEntryByHeight(1); + + try { + this.emit('tree reconstruct start'); + await this.db.compactTree(entry); await this.syncTree(); + this.emit('tree reconstruct end'); } finally { unlock(); } diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index ad5af47ea..9d4cf8814 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -66,6 +66,7 @@ class ChainDB { this.state = new ChainState(); this.field = new BitField(); this.pending = null; + this.pendingTreeState = null; this.current = null; this.blocksBatch = null; @@ -239,6 +240,7 @@ class ChainDB { this.current = this.db.batch(); this.pending = this.state.clone(); + this.pendingTreeState = this.treeState.clone(); if (this.blocks) this.blocksBatch = this.blocks.batch(); @@ -291,10 +293,12 @@ class ChainDB { assert(this.current); assert(this.pending); + assert(this.pendingTreeState); assert(!this.blocks || this.blocksBatch); this.current = null; this.pending = null; + this.pendingTreeState = null; this.blocksBatch = null; this.cacheHash.drop(); @@ -315,6 +319,7 @@ class ChainDB { async commit() { assert(this.current); assert(this.pending); + assert(this.pendingTreeState); try { if (this.blocks) @@ -324,6 +329,7 @@ class ChainDB { } catch (e) { this.current = null; this.pending = null; + this.pendingTreeState = null; this.cacheHash.drop(); this.cacheHeight.drop(); this.blocksBatch = null; @@ -338,8 +344,15 @@ class ChainDB { if (this.pending.committed) this.state = this.pending; + // Overwrite the entire TreeState + // if it's committed. Only happens + // on tree.commits. @see _saveNames + if (this.pendingTreeState.committed) + this.treeState = this.pendingTreeState; + this.current = null; this.pending = null; + this.pendingTreeState = null; this.cacheHash.commit(); this.cacheHeight.commit(); @@ -982,10 +995,14 @@ class ChainDB { // If the tree data gets out of sync or corrupted // the chain database knows where to resync the tree from. this.start(); - this.treeState.treeRoot = entry.treeRoot; - this.treeState.compactionRoot = entry.treeRoot; - this.treeState.compactionHeight = entry.height; - this.put(layout.s.encode(), this.treeState.encode()); + this.pendingTreeState.compact(entry.treeRoot, entry.height); + + // Note: the tree root commit height is always, + // first it's appearence - 1. + this.put(layout.s.encode(), this.pendingTreeState.commit( + entry.treeRoot, + entry.height - 1 + )); await this.commit(); // Rewind tree to historical commitment @@ -1246,7 +1263,7 @@ class ChainDB { /** * Get name state. * @param {Buffer} nameHash - * @returns {NameState} + * @returns {Promise} */ async getNameState(nameHash) { @@ -1263,7 +1280,7 @@ class ChainDB { /** * Get name state by name. * @param {Buffer} name - * @returns {NameState} + * @returns {Promise} */ async getNameStateByName(name) { @@ -1273,7 +1290,7 @@ class ChainDB { /** * Get name status. * @param {Buffer} nameHash - * @returns {NameState} + * @returns {Promise} */ async getNameStatus(nameHash, height, hardened) { @@ -1783,6 +1800,9 @@ class ChainDB { if (this.options.prune) throw new Error('Cannot reset when pruned.'); + if (this.treeState.compactionHeight !== 0) + throw new Error('Cannot reset when tree is compacted.'); + // We need to remove all alternate // chains first. This is ugly, but // it's the only safe way to reset @@ -2031,8 +2051,13 @@ class ChainDB { else await this.txn.commit(); - this.treeState.treeRoot = this.tree.rootHash(); - this.put(layout.s.encode(), this.treeState.encode()); + // Commit new tree state. + // Chain will need to recover current txn + // from treeState.commitHeight + 1 (including). + this.put(layout.s.encode(), this.pendingTreeState.commit( + this.tree.rootHash(), + entry.height + )); } } @@ -2685,24 +2710,47 @@ class TreeState extends bio.Struct { constructor() { super(); this.treeRoot = consensus.ZERO_HASH; + this.commitHeight = 0; this.compactionRoot = consensus.ZERO_HASH; this.compactionHeight = 0; + + this.committed = false; } inject(state) { this.treeRoot = state.treeRoot; + this.commitHeight = state.treeHeight; this.compactionHeight = state.compactionHeight; this.compactionRoot = state.compactionRoot; return this; } + compact(hash, height) { + assert(Buffer.isBuffer(hash)); + assert((height >>> 0) === height); + + this.compactionRoot = hash; + this.compactionHeight = height; + }; + + commit(hash, height) { + assert(Buffer.isBuffer(hash)); + assert((height >>> 0) === height); + + this.treeRoot = hash; + this.commitHeight = height; + this.committed = true; + return this.encode(); + } + getSize() { - return 68; + return 72; } write(bw) { bw.writeHash(this.treeRoot); + bw.writeU32(this.commitHeight); bw.writeHash(this.compactionRoot); bw.writeU32(this.compactionHeight); @@ -2711,6 +2759,7 @@ class TreeState extends bio.Struct { read(br) { this.treeRoot = br.readHash(); + this.commitHeight = br.readU32(); this.compactionRoot = br.readHash(); this.compactionHeight = br.readU32(); diff --git a/lib/blockchain/migrations.js b/lib/blockchain/migrations.js index 1cfcd49b9..64e1c2e57 100644 --- a/lib/blockchain/migrations.js +++ b/lib/blockchain/migrations.js @@ -8,6 +8,7 @@ const assert = require('bsert'); const Logger = require('blgr'); +const {encoding} = require('bufio'); const Network = require('../protocol/network'); const rules = require('../covenants/rules'); const Block = require('../primitives/block'); @@ -407,6 +408,7 @@ class MigrateTreeState extends AbstractMigration { this.logger = options.logger.context('chain-migration-tree-state'); this.db = options.db; this.ldb = options.ldb; + this.network = options.network; } async check() { @@ -417,13 +419,18 @@ class MigrateTreeState extends AbstractMigration { } async migrate(b) { + const {treeInterval} = this.network.names; + const state = await this.db.getState(); + const tipHeight = await this.db.getHeight(state.tip); + const lastCommitHeight = tipHeight - (tipHeight % treeInterval); const hash = await this.ldb.get(layout.s.encode()); assert(hash && hash.length === 32); // new tree root // see chaindb.js TreeState - const buff = Buffer.alloc(68); - hash.copy(buff); + const buff = Buffer.alloc(72); + encoding.writeBytes(buff, hash, 0); + encoding.writeU32(buff, lastCommitHeight, 32); this.db.writeVersion(b, 3); b.put(layout.s.encode(), buff); diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index beb781756..311374adb 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -264,6 +264,14 @@ class FullNode extends Node { this.emit('tree compact end', treeRoot, entry); }); + this.chain.on('tree reconstruct start', () => { + this.emit('tree reconstruct start'); + }); + + this.chain.on('tree reconstruct end', () => { + this.emit('tree reconstruct end'); + }); + this.loadPlugins(); } diff --git a/lib/node/rpc.js b/lib/node/rpc.js index 8dc959b30..b8ee8309a 100644 --- a/lib/node/rpc.js +++ b/lib/node/rpc.js @@ -178,6 +178,7 @@ class RPC extends RPCBase { this.add('gettxoutsetinfo', this.getTXOutSetInfo); this.add('pruneblockchain', this.pruneBlockchain); this.add('compacttree', this.compactTree); + this.add('reconstructtree', this.reconstructTree); this.add('verifychain', this.verifyChain); this.add('invalidateblock', this.invalidateBlock); @@ -1119,6 +1120,27 @@ class RPC extends RPCBase { } } + async reconstructTree(args, help) { + if (help || args.length !== 0) + throw new RPCError(errs.MISC_ERROR, 'reconstructtree'); + + if (this.chain.options.spv) { + throw new RPCError(errs.MISC_ERROR, + 'Cannot reconstruct tree in SPV mode.'); + } + + if (this.chain.options.prune) { + throw new RPCError(errs.MISC_ERROR, + 'Cannot reconstruct tree in pruned node.'); + } + + try { + await this.chain.reconstructTree(); + } catch (e) { + throw new RPCError(errs.DATABASE_ERROR, e.message); + } + } + async verifyChain(args, help) { if (help || args.length > 2) { throw new RPCError(errs.MISC_ERROR, diff --git a/test/chain-migration-test.js b/test/chain-migration-test.js index 9ab3bc0cb..cf247664e 100644 --- a/test/chain-migration-test.js +++ b/test/chain-migration-test.js @@ -2,6 +2,8 @@ const assert = require('bsert'); const fs = require('bfile'); +const {encoding} = require('bufio'); +const {ZERO_HASH} = require('../lib/protocol/consensus'); const Network = require('../lib/protocol/network'); const WorkerPool = require('../lib/workers/workerpool'); const Miner = require('../lib/mining/miner'); @@ -935,7 +937,10 @@ describe('Chain Migrations', function() { await chain.open(); const state = chaindb.treeState; - assert.bufferEqual(state.encode(), Buffer.alloc(68, 0)); + const encoded = Buffer.alloc(72, 0); + + encoding.writeU32(encoded, chain.height, 32); + assert.bufferEqual(state.encode(), encoded); }); it('should migrate tree state (2)', async () => { @@ -971,7 +976,7 @@ describe('Chain Migrations', function() { const version = getVersion(await ldb.get(layout.V.encode()), 'chain'); assert.strictEqual(version, 3); assert.bufferEqual(chaindb.treeState.treeRoot, root); - assert.bufferEqual(chaindb.treeState.compactionRoot, Buffer.alloc(32, 0)); + assert.bufferEqual(chaindb.treeState.compactionRoot, ZERO_HASH); assert.strictEqual(chaindb.treeState.compactionHeight, 0); }); }); diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index cfcd181c5..e76c370d7 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -4,6 +4,7 @@ const os = require('os'); const path = require('path'); const fs = require('bfile'); const assert = require('bsert'); +const consensus = require('../lib/protocol/consensus'); const Network = require('../lib/protocol/network'); const Miner = require('../lib/mining/miner'); const Chain = require('../lib/blockchain/chain'); @@ -113,8 +114,27 @@ describe('Tree Compacting', function() { } } - let name, nameHash; - const treeRoots = []; + let name, nameHash, listener; + let treeRoots = []; + + const checkTree = async (tree, compacted = false) => { + for (const [index, hash] of treeRoots.entries()) { + if (compacted && index < (treeRoots.length - 8)) { + // Old root node has been deleted, tree state can not be restored. + await assert.rejects( + chain.db.tree.inject(hash), + {message: `Missing node: ${hash.toString('hex')}.`} + ); + continue; + } + + // Last 8 tree roots are recovered successfully like before compaction. + await chain.db.tree.inject(hash); + const raw = await chain.db.tree.get(nameHash); + const ns = NameState.decode(raw); + assert.bufferEqual(ns.data, Buffer.from([index + 1])); + } + }; before(async () => { await blocks.ensure(); @@ -122,6 +142,15 @@ describe('Tree Compacting', function() { await chain.open(); await miner.open(); + listener = (root) => { + if (root.equals(consensus.ZERO_HASH)) + return; + + treeRoots.push(root); + }; + + chain.on('tree commit', listener); + await memBlocks.open(); await memChain.open(); }); @@ -132,6 +161,8 @@ describe('Tree Compacting', function() { await blocks.close(); await fs.rimraf(prefix); + chain.removeListener('tree commit', listener); + await memChain.close(); await memBlocks.close(); }); @@ -166,12 +197,7 @@ describe('Tree Compacting', function() { }); it('should update Urkel Tree 20 times', async () => { - let count = 0; - chain.on('tree commit', (rootHash, entry, block) => { - count++; - // Keep track of all new tree root hashes. - treeRoots.push(rootHash); - }); + treeRoots = []; for (let i = 1; i <= 20; i++) { // Every namestate update, increment the name's 1-byte data resource. @@ -179,24 +205,14 @@ describe('Tree Compacting', function() { await mineBlocks(treeInterval, mempool); } - assert.strictEqual(count, 20); + assert.strictEqual(treeRoots.length, 20); const ns = await chain.db.getNameStateByName(name); assert.bufferEqual(ns.data, Buffer.from([20])); }); it('should restore tree state from any historical root', async () => { - for (let i = 0; i < treeRoots.length; i++) { - // Restore old tree state using historical root hash. - await chain.db.tree.inject(treeRoots[i]); - - // Get old namestate from old tree state. - const raw = await chain.db.tree.get(nameHash); - const ns = NameState.decode(raw); - - // Counter in the name's data resource should match. - assert.bufferEqual(ns.data, Buffer.from([i + 1])); - } + await checkTree(chain.db.tree, false); }); it('should compact tree', async () => { @@ -250,22 +266,7 @@ describe('Tree Compacting', function() { }); it('should ONLY restore tree state from most recent roots', async () => { - for (let i = 0; i < treeRoots.length; i++) { - if (i < (treeRoots.length - 8)) { - // Old root node has been deleted, tree state can not be restored. - await assert.rejects( - chain.db.tree.inject(treeRoots[i]), - {message: `Missing node: ${treeRoots[i].toString('hex')}.`} - ); - continue; - } - - // Last 8 tree roots are recovered successfully like before compaction. - await chain.db.tree.inject(treeRoots[i]); - const raw = await chain.db.tree.get(nameHash); - const ns = NameState.decode(raw); - assert.bufferEqual(ns.data, Buffer.from([i + 1])); - } + await checkTree(chain.db.tree, true); }); it('should compact tree a second time with no new data', async () => { @@ -281,23 +282,7 @@ describe('Tree Compacting', function() { }); it('should ONLY restore tree state from most recent roots', async () => { - // Data recovery conditions are the same after second compacttree. - for (let i = 0; i < treeRoots.length; i++) { - if (i < (treeRoots.length - 8)) { - // Old root node has been deleted, tree state can not be restored. - await assert.rejects( - chain.db.tree.inject(treeRoots[i]), - {message: `Missing node: ${treeRoots[i].toString('hex')}.`} - ); - continue; - } - - // Last 8 tree roots are recovered successfully like before compaction. - await chain.db.tree.inject(treeRoots[i]); - const raw = await chain.db.tree.get(nameHash); - const ns = NameState.decode(raw); - assert.bufferEqual(ns.data, Buffer.from([i + 1])); - } + await checkTree(chain.db.tree, true); }); it('should recover txn between tree intervals', async () => { @@ -357,7 +342,7 @@ describe('Tree Compacting', function() { // Add 20 tree intervals for (let i = counter; i <= counter + 20; i++) { - send(await wallet.sendUpdate(name, Buffer.from([i])), mempool); + send(await wallet.sendUpdate(name, Buffer.from([i + 1])), mempool); await mineBlocks(treeInterval, mempool); } @@ -390,7 +375,7 @@ describe('Tree Compacting', function() { assert.bufferEqual(chain.db.tree.rootHash(), chain.tip.treeRoot); raw = await chain.db.tree.get(nameHash); ns = NameState.decode(raw); - assert.bufferEqual(ns.data, Buffer.from([counter + 20])); + assert.bufferEqual(ns.data, Buffer.from([counter + 21])); }); it('should recover from failure during block connect', async () => { @@ -462,6 +447,65 @@ describe('Tree Compacting', function() { assert.bufferEqual(ns.data, Buffer.from([counter + 1])); }); + it('should not reconstruct tree (prune)', async () => { + if (!prune) + this.skip(); + + let error; + try { + await chain.reconstructTree(); + } catch (e) { + error = e; + } + + assert(error, 'reconstructTree should throw an error in prune mode.'); + assert.strictEqual(error.message, + 'Cannot reconstruct tree in pruned mode.'); + }); + + it('should reconstruct tree (archival)', async () => { + if (prune) + this.skip(); + + await checkTree(chain.db.tree, true); + + const before = await fs.stat(treePath); + await chain.reconstructTree(); + const after = await fs.stat(treePath); + + assert(before.size < after.size); + + // Should have all roots. + await checkTree(chain.db.tree, false); + }); + + it('should recover reconstructing tree (archival)', async () => { + if (prune) + this.skip(); + + await checkTree(chain.db.tree, false); + + // let's compact again and reconstruct + const before = await fs.stat(treePath); + await chain.compactTree(); + const after = await fs.stat(treePath); + + assert(before.size > after.size); + + await checkTree(chain.db.tree, true); + }); + + it('should fail to reset when compacted', async () => { + let error = 'Cannot reset when tree is compacted.'; + + if (prune) + error = 'Cannot reset when pruned.'; + + await assert.rejects(chain.reset(0), { + message: error + }); + }); + it(`should ${prune ? '' : 'not '}have pruned chain`, async () => { // Sanity check. Everything worked on a chain that is indeed pruning. // Start at height 2 because pruneAfterHeight == 1 @@ -511,7 +555,7 @@ describe('Tree Compacting', function() { await node.close(); }); - it('should refuse to compact tree via RPC', async () => { + it('should refuse to compact/reconstruct tree via RPC', async () => { node = new SPVNode({ prefix, network: 'regtest', @@ -525,18 +569,24 @@ describe('Tree Compacting', function() { message: 'Cannot compact tree in SPV mode.' }); + await assert.rejects(node.rpc.reconstructTree([]), { + message: 'Cannot reconstruct tree in SPV mode.' + }); + await node.close(); }); }); describe('Full Node', function() { - let prefix, node; + let prefix, node, treePath; beforeEach(async () => { prefix = path.join( os.tmpdir(), `hsd-tree-compacting-test-${Date.now()}` ); + + treePath = path.join(prefix, 'regtest', 'tree', '0000000001'); }); afterEach(async () => { @@ -544,7 +594,7 @@ describe('Tree Compacting', function() { try { await node.close(); } catch (e) { - console.log('Error closing..', e.message); + ; } } @@ -624,7 +674,7 @@ describe('Tree Compacting', function() { await node.close(); }); - it('should compact tree when chain is long enought', async () => { + it('should compact tree when chain is long enough', async () => { this.timeout(2000); const compactTimeout = 500; @@ -673,7 +723,7 @@ describe('Tree Compacting', function() { const [rootHash, entry] = endEvent[0].values; // We don't have anything in the tree. - assert.bufferEqual(rootHash, Buffer.alloc(32, 0x00)); + assert.bufferEqual(rootHash, consensus.ZERO_HASH); // 100 - 40 = 60 // 60 % 5 = 0 // So nearest one is 61. @@ -684,10 +734,8 @@ describe('Tree Compacting', function() { it('should compact tree on launch (disk sizes)', async () => { this.timeout(4000); - const treePath = path.join(prefix, 'regtest', 'tree', '0000000001'); - // Fresh start - let node = new FullNode({ + node = new FullNode({ prefix, network: 'regtest', memory: false @@ -697,7 +745,7 @@ describe('Tree Compacting', function() { const fresh = await fs.stat(treePath); // Grow - const blocks = 300; + const blocks = 200; const waiter = forEventCondition(node, 'connect', (entry) => { return entry.height >= blocks; }, 2000); @@ -728,16 +776,82 @@ describe('Tree Compacting', function() { const [endEvent] = await waiterEnd; const [hash, entry] = endEvent.values; assert.bufferEqual(hash, Buffer.alloc(32, 0x00)); - assert.strictEqual(entry.height, 261); + assert.strictEqual(entry.height, 161); + + // Tree is compacted + const compacted = await fs.stat(treePath); + assert(compacted.size < grown.size); + + // Because syncTree will read tree roots + // since compaction (8 tree roots) compacted + // will be bigger than fresh. + assert(fresh.size < compacted.size); + + // done + await node.close(); + }); + + it('should compact/reconstruct tree on rpc', async () => { + this.timeout(4000); + + // Fresh start + node = new FullNode({ + prefix, + network: 'regtest', + memory: false + }); + await node.ensure(); + await node.open(); + const fresh = await fs.stat(treePath); + + // Grow + const blocks = 200; + const waiter = forEventCondition(node, 'connect', (entry) => { + return entry.height >= blocks; + }, 2000); + + await node.rpc.generateToAddress( + [blocks, new Address().toString('regtest')] + ); + await waiter; + + // Tree has grown + const grown = await fs.stat(treePath); + assert(fresh.size < grown.size); + + await node.close(); + + // Now use RPC for compaction. + node = new FullNode({ + prefix, + network: 'regtest', + memory: false + }); + + await node.open(); + + const compactEnd = forEvent(node, 'tree compact end', 1, 1000); + await node.rpc.compactTree([]); + const [endEvent] = await compactEnd; + const [hash, entry] = endEvent.values; + assert.bufferEqual(hash, consensus.ZERO_HASH); + assert.strictEqual(entry.height, 161); // Tree is compacted const compacted = await fs.stat(treePath); assert(compacted.size < grown.size); + assert(fresh.size < compacted.size); + + // Reconstruct + const reconstructEnd = forEvent(node, 'tree reconstruct end', 1, 1000); + await node.rpc.reconstructTree([]); + await reconstructEnd; + + const reconstructed = await fs.stat(treePath); + assert(reconstructed.size > compacted.size); - // Bonus: since there are no namestate updates in this test, - // all the nodes committed to the tree during "growth" are identically - // empty. When we compact, only the original empty node will remain. - assert.strictEqual(fresh.size, compacted.size); + // This is same as grown, because we did not have any reorgs. + assert.strictEqual(reconstructed.size, grown.size); // done await node.close(); From a339e39af1b1f9248fe641f19394cf8afef32772 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Sat, 21 May 2022 00:15:27 +0400 Subject: [PATCH 18/23] chaindb: try removing tmp directory before compaction. --- lib/blockchain/chaindb.js | 20 +++++++++++- test/chain-tree-compaction-test.js | 50 ++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 9d4cf8814..cf5647d04 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -1005,11 +1005,29 @@ class ChainDB { )); await this.commit(); + const tmpDir = this.options.treePrefix + '~'; + + const tmpTree = new Tree({ + hash: blake2b, + bits: 256, + prefix: tmpDir + }); + + // Make sure to remove the tmp directory first. + // There should not be directory, unless it was + // stopped in the middle of compaction. + // Otherwise compacted tree would add on top + // of the previsouly compacted db. + await tmpTree.open(); + const tmpStore = tmpTree.store; + await tmpTree.close(); + await tmpStore.destroy(); + // Rewind tree to historical commitment await this.tree.inject(entry.treeRoot); // Delete historical data - await this.tree.compact(); + await this.tree.compact(tmpDir); // Reset in-memory tree delta this.txn = this.tree.txn(); diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index e76c370d7..cb5f22f92 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -47,7 +47,8 @@ describe('Tree Compacting', function() { os.tmpdir(), `hsd-tree-compacting-test-${Date.now()}` ); - const treePath = path.join(prefix, 'tree', '0000000001'); + const treePath = path.join(prefix, 'tree'); + const treePart1 = path.join(prefix, 'tree', '0000000001'); // This is the chain we are testing, // we are going to compact its tree @@ -216,10 +217,10 @@ describe('Tree Compacting', function() { }); it('should compact tree', async () => { - const before = await fs.stat(treePath); + const before = await fs.stat(treePart1); await chain.compactTree(); await chain.syncTree(); - const after = await fs.stat(treePath); + const after = await fs.stat(treePart1); // Urkel Tree should be smaller now. // Urkel Tree files are padded to ensure that Meta nodes are written @@ -272,10 +273,10 @@ describe('Tree Compacting', function() { it('should compact tree a second time with no new data', async () => { // If user executes rpc compacttree repeatedly, // it shouldn't break anything. - const before = await fs.stat(treePath); + const before = await fs.stat(treePart1); await chain.compactTree(); await chain.syncTree(); - const after = await fs.stat(treePath); + const after = await fs.stat(treePart1); // Should be no change assert.strictEqual(before.size, after.size); @@ -316,10 +317,10 @@ describe('Tree Compacting', function() { const treeRootBefore = chain.db.tree.rootHash(); // Compact - const before = await fs.stat(treePath); + const before = await fs.stat(treePart1); await chain.compactTree(); await chain.syncTree(); - const after = await fs.stat(treePath); + const after = await fs.stat(treePart1); assert(before.size > after.size); // Check @@ -346,7 +347,7 @@ describe('Tree Compacting', function() { await mineBlocks(treeInterval, mempool); } - const before = await fs.stat(treePath); + const before = await fs.stat(treePart1); // Rewind the tree 6 intervals and compact, but do not sync to tip yet. const entry = await chain.getEntry(chain.height - 6 * treeInterval); @@ -368,7 +369,7 @@ describe('Tree Compacting', function() { await miner.open(); // Tree was compacted - const after = await fs.stat(treePath); + const after = await fs.stat(treePart1); assert(before.size > after.size); // Tree was re-synced automatically to chain tip on restart @@ -469,9 +470,9 @@ describe('Tree Compacting', function() { await checkTree(chain.db.tree, true); - const before = await fs.stat(treePath); + const before = await fs.stat(treePart1); await chain.reconstructTree(); - const after = await fs.stat(treePath); + const after = await fs.stat(treePart1); assert(before.size < after.size); @@ -486,9 +487,9 @@ describe('Tree Compacting', function() { await checkTree(chain.db.tree, false); // let's compact again and reconstruct - const before = await fs.stat(treePath); + const before = await fs.stat(treePart1); await chain.compactTree(); - const after = await fs.stat(treePath); + const after = await fs.stat(treePart1); assert(before.size > after.size); @@ -506,6 +507,29 @@ describe('Tree Compacting', function() { }); }); + it('should remove existing tmp dir', async () => { + if (prune) + this.skip(); + + const tmpPath = treePath + '~'; + const beforeRecovery = await fs.stat(treePart1); + await chain.reconstructTree(); + const afterRecovery = await fs.stat(treePart1); + assert(beforeRecovery.size < afterRecovery.size); + + await fs.copy(treePath, tmpPath); + // Normally tmp directory lock would have expired. + await fs.remove(path.join(tmpPath, 'lock')); + + await chain.compactTree(); + const afterCompaction = await fs.stat(treePart1); + + // If we don't remove existing TMP directory + // afterCompaction would be bigger than afterRecovery. + assert(afterCompaction.size < afterRecovery.size); + assert(!await fs.exists(tmpPath)); + }); + it(`should ${prune ? '' : 'not '}have pruned chain`, async () => { // Sanity check. Everything worked on a chain that is indeed pruning. // Start at height 2 because pruneAfterHeight == 1 From d297d10df82222f0edb094df10023b5deb8e7c47 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Tue, 31 May 2022 12:21:26 +0400 Subject: [PATCH 19/23] pkg: update changelog. chain: clean up chain.open --- CHANGELOG.md | 21 +++++++++------- lib/blockchain/chain.js | 56 +++++++++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0392843ef..1736d399b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,15 @@ ### Node changes - `FullNode` and `SPVNode` now accept the option `--agent` which adds a string to the user-agent of the node (which will already contain hsd version) and is - sent to peers in the version packet. Strings must not contain slashes and total - user-agent string must be 255 charact + sent to peers in the version packet. Strings must not contain slashes and + total user-agent string must be less than 255 characters. - - `FullNode` parses new configuration option `--compact-tree` which will compact - the Urkel Tree when the node first opens, by deleting historical data. It will - keep up to the last 288 blocks worth of tree data on disk (7-8 tree intervals) - exposing the node to a similar deep reorganization vulnerability as a - chain-pruning node. + - `FullNode` parses new configuration option `--compact-tree-on-init` and + `--compact-tree-init-interval` which will compact the Urkel Tree when the node + first opens, by deleting historical data. It will try to compact it again + after `tree-init-interval` has passed. Compaction will keep up to the last 288 + blocks worth of tree data on disk (7-8 tree intervals) exposing the node to a + similar deep reorganization vulnerability as a chain-pruning node. ## v3.0.0 @@ -32,8 +33,10 @@ ### Wallet API changes - New RPC methods: - - `signmessagewithname`: Like `signmessage` but uses a name instead of an address. The owner's address will be used to sign the message. - - `verifymessagewithname`: Like `verifymessage` but uses a name instead of an address. The owner's address will be used to verify the message. + - `signmessagewithname`: Like `signmessage` but uses a name instead of an + address. The owner's address will be used to sign the message. + - `verifymessagewithname`: Like `verifymessage` but uses a name instead of an + address. The owner's address will be used to verify the message. - New wallet creation accepts parameter `language` to generate the mnemonic phrase. diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index d3e6b43e8..583bdfd42 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -101,26 +101,10 @@ class Chain extends AsyncEmitter { this.setDeploymentState(state); if (!this.options.spv) { - const {txStart} = this.network; - const startFrom = txStart + this.network.block.keepBlocks; - - if (this.options.compactTreeOnInit && this.height > startFrom) { - const {compactionHeight} = await this.db.getTreeState(); - const {compactTreeInitInterval} = this.options; - const compactFrom = compactionHeight + compactTreeInitInterval; - - if (compactFrom <= this.height) { - await this.compactTree(); - } else { - this.logger.debug( - `Tree will compact at ${compactFrom} height.`); - await this.syncTree(); - } - } else { - // Replay the blockchain from last committed tree root - // back up to the tip to rebuild tree and txn. + const sync = await this.tryCompact(); + + if (sync) await this.syncTree(); - } } this.logger.memory(); @@ -141,6 +125,40 @@ class Chain extends AsyncEmitter { return this.db.close(); } + /** + * Check if we need to compact tree data. + * @returns {Promise} - Should we sync + */ + + async tryCompact() { + if (this.options.spv) + return false; + + if (!this.options.compactTreeOnInit) + return true; + + const {txStart} = this.network; + const {keepBlocks} = this.network.block; + const startFrom = txStart + keepBlocks; + + if (this.height <= startFrom) + return true; + + const {compactionHeight} = await this.db.getTreeState(); + const {compactTreeInitInterval} = this.options; + const compactFrom = compactionHeight + compactTreeInitInterval; + + if (compactFrom > this.height) { + this.logger.debug( + `Tree will compact after ${compactFrom} height.`); + return true; + } + + // Compact tree calls syncTree so we don't want to rerun it. + await this.compactTree(); + return false; + } + /** * Sync tree state. */ From 05ed36972a101613610e1f4fcb17002651dc0fbb Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Tue, 31 May 2022 23:52:12 +0400 Subject: [PATCH 20/23] chain: Fix migration for the SPV. Minor nits and updates. --- lib/blockchain/chain.js | 2 +- lib/blockchain/chaindb.js | 4 +- lib/blockchain/migrations.js | 8 ++- lib/node/fullnode.js | 2 +- test/chain-migration-test.js | 95 ++++++++++++++++++++++++++++++ test/chain-tree-compaction-test.js | 30 ++++++---- 6 files changed, 123 insertions(+), 18 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 583bdfd42..00975afa5 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -150,7 +150,7 @@ class Chain extends AsyncEmitter { if (compactFrom > this.height) { this.logger.debug( - `Tree will compact after ${compactFrom} height.`); + `Tree will compact when restarted after height ${compactFrom}.`); return true; } diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index cf5647d04..f744096ce 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -997,8 +997,8 @@ class ChainDB { this.start(); this.pendingTreeState.compact(entry.treeRoot, entry.height); - // Note: the tree root commit height is always, - // first it's appearence - 1. + // Note: the tree root commit height is always one block before its + // appearence in a header. this.put(layout.s.encode(), this.pendingTreeState.commit( entry.treeRoot, entry.height - 1 diff --git a/lib/blockchain/migrations.js b/lib/blockchain/migrations.js index 64e1c2e57..6a9039ccb 100644 --- a/lib/blockchain/migrations.js +++ b/lib/blockchain/migrations.js @@ -412,13 +412,15 @@ class MigrateTreeState extends AbstractMigration { } async check() { - if (this.options.spv) - return types.FAKE_MIGRATE; - return types.MIGRATE; } async migrate(b) { + if (this.options.spv) { + this.db.writeVersion(b, 3); + return; + } + const {treeInterval} = this.network.names; const state = await this.db.getState(); const tipHeight = await this.db.getHeight(state.tip); diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index 311374adb..a52bbb5ff 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -260,7 +260,7 @@ class FullNode extends Node { this.emit('tree compact start', treeRoot, entry); }); - this.chain.on('tree compact start', (treeRoot, entry) => { + this.chain.on('tree compact end', (treeRoot, entry) => { this.emit('tree compact end', treeRoot, entry); }); diff --git a/test/chain-migration-test.js b/test/chain-migration-test.js index cf247664e..5183e951d 100644 --- a/test/chain-migration-test.js +++ b/test/chain-migration-test.js @@ -980,6 +980,101 @@ describe('Chain Migrations', function() { assert.strictEqual(chaindb.treeState.compactionHeight, 0); }); }); + + describe('Migration Tree State SPV (integration)', function() { + const location = testdir('migrate-tree-state-spv'); + const migrationsBAK = ChainMigrator.migrations; + + const workers = new WorkerPool({ + enabled: true, + size: 2 + }); + + const chainOptions = { + prefix: location, + memory: false, + spv: true, + network, + workers + }; + + let chain, chaindb, ldb; + before(async () => { + ChainMigrator.migrations = {}; + await fs.mkdirp(location); + await workers.open(); + }); + + after(async () => { + ChainMigrator.migrations = migrationsBAK; + await rimraf(location); + await workers.close(); + }); + + beforeEach(async () => { + chain = new Chain(chainOptions); + chaindb = chain.db; + ldb = chaindb.db; + + chaindb.version = 3; + }); + + afterEach(async () => { + if (chain.opened) + await chain.close(); + }); + + it('should throw version check error', async () => { + // Previous state + await chain.open(); + const b = ldb.batch(); + writeVersion(b, 'chain', 2); + await b.write(); + await chain.close(); + + let error; + try { + await chain.open(); + } catch (e) { + error = e; + } + + assert(error); + assert.strictEqual(error.message, VERSION_ERROR); + }); + + it('should enable tree state migration', async () => { + ChainMigrator.migrations = { + 0: ChainMigrator.MigrateTreeState + }; + }); + + it('should throw when new migration is available', async () => { + const expected = migrationError(ChainMigrator.migrations, [0], + chainFlagError(0)); + + let error; + try { + await chain.open(); + } catch (e) { + error = e; + } + + assert(error, 'Chain must throw an error.'); + assert.strictEqual(error.message, expected); + }); + + it('should migrate db version', async () => { + chain.options.chainMigrate = 0; + + await chain.open(); + const state = chaindb.treeState; + const encoded = Buffer.alloc(72, 0); + + encoding.writeU32(encoded, chain.height, 32); + assert.bufferEqual(state.encode(), encoded); + }); + }); }); function writeVersion(b, name, version) { diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index cb5f22f92..dab3ff0dd 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -118,7 +118,7 @@ describe('Tree Compacting', function() { let name, nameHash, listener; let treeRoots = []; - const checkTree = async (tree, compacted = false) => { + const checkTree = async (compacted = false) => { for (const [index, hash] of treeRoots.entries()) { if (compacted && index < (treeRoots.length - 8)) { // Old root node has been deleted, tree state can not be restored. @@ -213,7 +213,7 @@ describe('Tree Compacting', function() { }); it('should restore tree state from any historical root', async () => { - await checkTree(chain.db.tree, false); + await checkTree(false); }); it('should compact tree', async () => { @@ -267,7 +267,7 @@ describe('Tree Compacting', function() { }); it('should ONLY restore tree state from most recent roots', async () => { - await checkTree(chain.db.tree, true); + await checkTree(true); }); it('should compact tree a second time with no new data', async () => { @@ -283,7 +283,7 @@ describe('Tree Compacting', function() { }); it('should ONLY restore tree state from most recent roots', async () => { - await checkTree(chain.db.tree, true); + await checkTree(true); }); it('should recover txn between tree intervals', async () => { @@ -468,7 +468,7 @@ describe('Tree Compacting', function() { if (prune) this.skip(); - await checkTree(chain.db.tree, true); + await checkTree(true); const before = await fs.stat(treePart1); await chain.reconstructTree(); @@ -477,14 +477,14 @@ describe('Tree Compacting', function() { assert(before.size < after.size); // Should have all roots. - await checkTree(chain.db.tree, false); + await checkTree(false); }); it('should recover reconstructing tree (archival)', async () => { if (prune) this.skip(); - await checkTree(chain.db.tree, false); + await checkTree(false); // let's compact again and reconstruct const before = await fs.stat(treePart1); @@ -493,7 +493,7 @@ describe('Tree Compacting', function() { assert(before.size > after.size); - await checkTree(chain.db.tree, true); + await checkTree(true); }); it('should fail to reset when compacted', async () => { @@ -720,7 +720,7 @@ describe('Tree Compacting', function() { await node.open(); assert.strictEqual(compacted, false); - const blocks = 100; + const blocks = network.block.keepBlocks + (treeInterval * 10); const waiter = forEventCondition(node, 'connect', e => e.height >= blocks); await node.rpc.generateToAddress( @@ -748,10 +748,18 @@ describe('Tree Compacting', function() { const [rootHash, entry] = endEvent[0].values; // We don't have anything in the tree. assert.bufferEqual(rootHash, consensus.ZERO_HASH); + // blocks = keepBlocks + treeInterval * 10; + // so what we can prune is `blocks - keepBlocks` + // what's left is `treeInterval * 10` + // Because we are at the edge, nearest should be + // (treeInterval * 10) + 1 + // + // e.g. if blocks was 100, keepBlocks 40 and interval 5: // 100 - 40 = 60 // 60 % 5 = 0 - // So nearest one is 61. - assert.strictEqual(entry.height, 61); + // So nearest one would be 61. + const nearest = treeInterval * 10 + 1; + assert.strictEqual(entry.height, nearest); await node.close(); }); From 55021959a4939a155a238ce4bff1d424b4139ac2 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Thu, 2 Jun 2022 08:25:23 -0400 Subject: [PATCH 21/23] test: cover tree interval boundary after compacting --- test/chain-tree-compaction-test.js | 171 ++++++++++++++++++++++++++++- 1 file changed, 168 insertions(+), 3 deletions(-) diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index dab3ff0dd..8ce9a1ad4 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -219,7 +219,6 @@ describe('Tree Compacting', function() { it('should compact tree', async () => { const before = await fs.stat(treePart1); await chain.compactTree(); - await chain.syncTree(); const after = await fs.stat(treePart1); // Urkel Tree should be smaller now. @@ -275,7 +274,6 @@ describe('Tree Compacting', function() { // it shouldn't break anything. const before = await fs.stat(treePart1); await chain.compactTree(); - await chain.syncTree(); const after = await fs.stat(treePart1); // Should be no change @@ -319,7 +317,6 @@ describe('Tree Compacting', function() { // Compact const before = await fs.stat(treePart1); await chain.compactTree(); - await chain.syncTree(); const after = await fs.stat(treePart1); assert(before.size > after.size); @@ -889,4 +886,172 @@ describe('Tree Compacting', function() { await node.close(); }); }); + + describe('Boundary checks', function() { + const prefix = path.join( + os.tmpdir(), + `hsd-tree-compacting-test-${Date.now()}` + ); + + const blocks = blockstore.create({ + memory: false, + prefix, + network + }); + const chain = new Chain({ + memory: false, + prefix, + blocks: blocks, + network + }); + const miner = new Miner({chain}); + const cpu = miner.cpu; + + const wallet = new MemWallet({network}); + wallet.getNameStatus = async (nameHash) => { + assert(Buffer.isBuffer(nameHash)); + const height = chain.height + 1; + return chain.db.getNameStatus(nameHash, height); + }; + + async function mineBlocks(num, open = false) { + for (; num > 0; num--) { + const job = await cpu.createJob(); + + // Include an OPEN for some new name in every block. + // This ensures that every single block results in a different + // tree and treeRoot without any auctions. + if (open) { + const name = rules.grindName(4, chain.height - 1, network); + const tx = await wallet.sendOpen(name); + job.pushTX(tx.toTX()); + job.refresh(); + } + + const block = await job.mineAsync(); + const entry = await chain.add(block); + wallet.addBlock(entry, block.txs); + } + } + + const treeRoots = []; + chain.on('tree commit', root => treeRoots.push(root)); + + const checkTree = async (expected) => { + for (let i = treeRoots.length - 1; i >= 0; i--) { + const root = treeRoots[i]; + + if (expected > 0) { + await chain.db.tree.inject(root); + expected--; + } else { + await assert.rejects( + chain.db.tree.inject(root), + {message: `Missing node: ${root.toString('hex')}.`} + ); + } + } + }; + + before(async () => { + await blocks.ensure(); + await blocks.open(); + await chain.open(); + }); + + after(async () => { + await chain.close(); + await blocks.close(); + }); + + it('should fund wallet', async () => { + miner.addresses.length = 0; + miner.addAddress(wallet.getReceive()); + await mineBlocks(treeInterval * 2); + }); + + it('should generate blocks to treeInterval - 3 and compact', async () => { + treeRoots.length = 0; + let num = treeInterval - (chain.height % treeInterval); + num += treeInterval * 20 - 3; + await mineBlocks(num, true); + assert(chain.height % treeInterval === treeInterval - 3); + + // All roots available before compacting + await checkTree(Infinity); + + await chain.compactTree(); + await checkTree(8); + }); + + it('should generate blocks to treeInterval - 2 and compact', async () => { + treeRoots.length = 0; + let num = treeInterval - (chain.height % treeInterval); + num += treeInterval * 20 - 2; + await mineBlocks(num, true); + assert(chain.height % treeInterval === treeInterval - 2); + + // All roots available before compacting + await checkTree(Infinity); + + await chain.compactTree(); + await checkTree(8); + }); + + it('should generate blocks to treeInterval - 1 and compact', async () => { + treeRoots.length = 0; + let num = treeInterval - (chain.height % treeInterval); + num += treeInterval * 20 - 1; + await mineBlocks(num, true); + assert(chain.height % treeInterval === treeInterval - 1); + + // All roots available before compacting + await checkTree(Infinity); + + await chain.compactTree(); + await checkTree(8); + }); + + it('should generate blocks to treeInterval + 0 and compact', async () => { + treeRoots.length = 0; + let num = treeInterval - (chain.height % treeInterval); + num += treeInterval * 20 + 0; + await mineBlocks(num, true); + assert(chain.height % treeInterval === 0); + + // All roots available before compacting + await checkTree(Infinity); + + await chain.compactTree(); + await checkTree(9); + }); + + it('should generate blocks to treeInterval + 1 and compact', async () => { + treeRoots.length = 0; + let num = treeInterval - (chain.height % treeInterval); + num += treeInterval * 20 + 1; + await mineBlocks(num, true); + assert(chain.height % treeInterval === 1); + + // All roots available before compacting + await checkTree(Infinity); + + await chain.compactTree(); + await checkTree(8); + }); + + it('should generate blocks to treeInterval + 2 and compact', async () => { + treeRoots.length = 0; + let num = treeInterval - (chain.height % treeInterval); + num += treeInterval * 20 + 2; + await mineBlocks(num, true); + assert(chain.height % treeInterval === 2); + + // All roots available before compacting + await checkTree(Infinity); + + await chain.compactTree(); + await checkTree(8); + }); + }); }); From b41c56e3e634651c815cbe90560b0678c97edabe Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Thu, 2 Jun 2022 17:01:20 +0400 Subject: [PATCH 22/23] chain: take into account the compaction depth. --- lib/blockchain/chain.js | 2 +- test/chain-tree-compaction-test.js | 82 ++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 00975afa5..e22d54ca8 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -146,7 +146,7 @@ class Chain extends AsyncEmitter { const {compactionHeight} = await this.db.getTreeState(); const {compactTreeInitInterval} = this.options; - const compactFrom = compactionHeight + compactTreeInitInterval; + const compactFrom = compactionHeight + keepBlocks + compactTreeInitInterval; if (compactFrom > this.height) { this.logger.debug( diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index 8ce9a1ad4..a2e4f2bcd 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -761,6 +761,88 @@ describe('Tree Compacting', function() { await node.close(); }); + it('should recompact tree if tree init interval passed', async () => { + const {keepBlocks} = network.block; + const compactInterval = keepBlocks; + const nodeOptions = { + prefix, + network: 'regtest', + memory: false, + compactTreeOnInit: true, + compactTreeInitInterval: compactInterval + }; + + node = new FullNode(nodeOptions); + + await node.ensure(); + + let compacted = false; + const compactWrapper = (node) => { + const compactTree = node.chain.compactTree.bind(node.chain); + + node.chain.compactTree = () => { + compacted = true; + return compactTree(); + }; + }; + + compactWrapper(node); + await node.open(); + assert.strictEqual(compacted, false); + + // get enough blocks for the compaction check. + let blocks = compactInterval + keepBlocks + 1; + let waiter = forEventCondition(node, 'connect', e => e.height >= blocks); + + await node.rpc.generateToAddress( + [blocks, new Address().toString('regtest')] + ); + + await waiter; + await node.close(); + + // Should compact, because we have enough blocks. + node = new FullNode(nodeOptions); + compactWrapper(node); + + await node.open(); + assert.strictEqual(compacted, true); + + // setup interval - 1 blocks for next test. + blocks = compactInterval + network.names.treeInterval - 1; + waiter = forEvent(node, 'connect', blocks); + await node.rpc.generateToAddress( + [blocks, new Address().toString('regtest')] + ); + await waiter; + await node.close(); + + // Should not recompact because interval has not passed. + compacted = false; + node = new FullNode(nodeOptions); + compactWrapper(node); + + await node.open(); + assert.strictEqual(compacted, false); + + waiter = forEvent(node, 'connect'); + // commit 1 to recompact. + await node.rpc.generateToAddress( + [1, new Address().toString('regtest')] + ); + await waiter; + await node.close(); + + // Should recompact because interval has passed + compacted = false; + node = new FullNode(nodeOptions); + compactWrapper(node); + + await node.open(); + assert.strictEqual(compacted, true); + await node.close(); + }); + it('should compact tree on launch (disk sizes)', async () => { this.timeout(4000); // Fresh start From 74cc006ef089208a8a715746c9bed470a24a0b70 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Thu, 2 Jun 2022 16:01:12 -0400 Subject: [PATCH 23/23] pkg: add chain migrate flag to changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1736d399b..dc3dcafb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## unreleased +**When upgrading to this version of hsd you must pass +`--chain-migrate=3` when you run it for the first time.** + ### Node changes - `FullNode` and `SPVNode` now accept the option `--agent` which adds a string to the user-agent of the node (which will already contain hsd version) and is