From 0e49cbf77f908321972a4d6de0ed5ecf190807bf Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Fri, 10 Mar 2023 20:42:06 +0200 Subject: [PATCH 1/9] Added initial changes to support multiple Merkle trees in the storage. Added assertion to check for collisions + fix. Added the address field to the InternalNodeKey's serialization. --- kvbc/cmf/categorized_kvbc_msgs.cmf | 1 + .../categorization/block_merkle_category.h | 2 +- kvbc/include/merkle_tree_db_adapter.h | 2 +- kvbc/include/merkle_tree_serialization.h | 32 ++++++++++++++++--- kvbc/include/sparse_merkle/db_reader.h | 4 ++- kvbc/include/sparse_merkle/keys.h | 15 +++++++-- kvbc/include/sparse_merkle/tree.h | 9 ++++-- kvbc/include/sparse_merkle/update_cache.h | 11 ++++--- kvbc/include/sparse_merkle/walker.h | 3 +- .../categorization/block_merkle_category.cpp | 10 +++--- kvbc/src/merkle_tree_db_adapter.cpp | 6 ++-- kvbc/src/sparse_merkle/tree.cpp | 10 +++--- kvbc/src/sparse_merkle/update_cache.cpp | 8 ++--- kvbc/src/sparse_merkle/walker.cpp | 10 +++--- kvbc/test/sparse_merkle/test_db.h | 17 ++++++---- kvbc/test/sparse_merkle/tree_test.cpp | 2 +- .../db_adapter_unit_test.cpp | 2 +- .../serialization_unit_test.cpp | 6 ++-- 18 files changed, 99 insertions(+), 51 deletions(-) diff --git a/kvbc/cmf/categorized_kvbc_msgs.cmf b/kvbc/cmf/categorized_kvbc_msgs.cmf index 46be39088e..cd95c1ccce 100644 --- a/kvbc/cmf/categorized_kvbc_msgs.cmf +++ b/kvbc/cmf/categorized_kvbc_msgs.cmf @@ -201,6 +201,7 @@ Msg BatchedInternalNodeKey 5002 { # The version of this key is the *tree* version, not block version uint64 version NibblePath path + string address } Msg LeafChild 5003 { diff --git a/kvbc/include/categorization/block_merkle_category.h b/kvbc/include/categorization/block_merkle_category.h index b079abc732..c372a65216 100644 --- a/kvbc/include/categorization/block_merkle_category.h +++ b/kvbc/include/categorization/block_merkle_category.h @@ -202,7 +202,7 @@ class BlockMerkleCategory { Reader(const storage::rocksdb::NativeClient& db) : db_{db} {} // Return the latest root node in the system. - sparse_merkle::BatchedInternalNode get_latest_root() const override; + sparse_merkle::BatchedInternalNode get_latest_root(std::string address = "") const override; // Retrieve a BatchedInternalNode given an InternalNodeKey. // diff --git a/kvbc/include/merkle_tree_db_adapter.h b/kvbc/include/merkle_tree_db_adapter.h index 78d3b8524f..d2075a171e 100644 --- a/kvbc/include/merkle_tree_db_adapter.h +++ b/kvbc/include/merkle_tree_db_adapter.h @@ -219,7 +219,7 @@ class DBAdapter : public IDbAdapter { Reader(const DBAdapter &adapter) : adapter_{adapter} {} // Return the latest root node in the system. - sparse_merkle::BatchedInternalNode get_latest_root() const override; + sparse_merkle::BatchedInternalNode get_latest_root(std::string address = "") const override; // Retrieve a BatchedInternalNode given an InternalNodeKey. // diff --git a/kvbc/include/merkle_tree_serialization.h b/kvbc/include/merkle_tree_serialization.h index 5062a8f46d..ebefc94b49 100644 --- a/kvbc/include/merkle_tree_serialization.h +++ b/kvbc/include/merkle_tree_serialization.h @@ -88,6 +88,12 @@ inline void serializeImp(const sparse_merkle::NibblePath &path, std::string &out out.append(std::cbegin(path.data()), std::cend(path.data())); } +inline void serializeImp(const std::string &address, std::string &out) { + ConcordAssert(address.size() < std::numeric_limits::max()); + out.append(1, static_cast(address.size())); + out.append(std::cbegin(address), std::cend(address)); +} + inline std::string serializeImp(const sparse_merkle::NibblePath &path) { auto out = std::string{}; serializeImp(path, out); @@ -107,6 +113,7 @@ inline std::string serializeImp(const sparse_merkle::Hash &hash) { inline void serializeImp(const sparse_merkle::InternalNodeKey &key, std::string &out) { serializeImp(key.version().value(), out); serializeImp(key.path(), out); + serializeImp(key.address(), out); } inline std::string serializeImp(const sparse_merkle::InternalNodeKey &key) { @@ -318,6 +325,19 @@ inline sparse_merkle::NibblePath deserialize(const co return sparse_merkle::NibblePath{nibbleCount, path}; } +template <> +inline std::string deserialize(const concordUtils::Sliver &buf) { + ConcordAssert(buf.length() >= sizeof(std::uint8_t)); + // NOLINTNEXTLINE(bugprone-signed-char-misuse) + const auto addrSize = static_cast(buf[0]); + std::string address; + address.reserve(addrSize); + for (auto i = 0u; i < addrSize; ++i) { + address.push_back(buf[i]); + } + return address; +} + template <> inline sparse_merkle::LeafKey deserialize(const concordUtils::Sliver &buf) { ConcordAssert(buf.length() >= sparse_merkle::LeafKey::SIZE_IN_BYTES); @@ -328,10 +348,14 @@ inline sparse_merkle::LeafKey deserialize(const concordU template <> inline sparse_merkle::InternalNodeKey deserialize(const concordUtils::Sliver &buf) { - return sparse_merkle::InternalNodeKey{ - deserialize(buf), - deserialize(concordUtils::Sliver{ - buf, sparse_merkle::Version::SIZE_IN_BYTES, buf.length() - sparse_merkle::Version::SIZE_IN_BYTES})}; + auto version = deserialize(buf); + auto nibblePath = deserialize(concordUtils::Sliver{ + buf, sparse_merkle::Version::SIZE_IN_BYTES, buf.length() - sparse_merkle::Version::SIZE_IN_BYTES}); + std::string address = + deserialize(concordUtils::Sliver{buf, + sparse_merkle::Version::SIZE_IN_BYTES + nibblePath.length(), + buf.length() - sparse_merkle::Version::SIZE_IN_BYTES}); + return sparse_merkle::InternalNodeKey{std::move(version), std::move(nibblePath), std::move(address)}; } template <> diff --git a/kvbc/include/sparse_merkle/db_reader.h b/kvbc/include/sparse_merkle/db_reader.h index a23850d773..16b23b282b 100644 --- a/kvbc/include/sparse_merkle/db_reader.h +++ b/kvbc/include/sparse_merkle/db_reader.h @@ -12,6 +12,8 @@ #pragma once +#include + namespace concord { namespace kvbc { namespace sparse_merkle { @@ -28,7 +30,7 @@ class IDBReader { virtual ~IDBReader() = default; // Return the latest root node in the system - virtual BatchedInternalNode get_latest_root() const = 0; + virtual BatchedInternalNode get_latest_root(std::string address = "") const = 0; // Retrieve a BatchedInternalNode given an InternalNodeKey. // diff --git a/kvbc/include/sparse_merkle/keys.h b/kvbc/include/sparse_merkle/keys.h index 52575ee1f4..d702ac7c0b 100644 --- a/kvbc/include/sparse_merkle/keys.h +++ b/kvbc/include/sparse_merkle/keys.h @@ -24,15 +24,23 @@ namespace sparse_merkle { // are part of the batch by pointing to their containing BatchedInternalNode. class InternalNodeKey { public: - InternalNodeKey(Version version, const NibblePath& path) : version_(version), path_(path) {} + template + InternalNodeKey(Version version, const NibblePath& path, T&& address) + : version_(version), path_(path), address_(std::forward(address)) {} // Return the root of a sparse merkle tree at a given version. - static InternalNodeKey root(Version version) { return InternalNodeKey(version, NibblePath()); } + template + static InternalNodeKey root(Version version, T&& address) { + return InternalNodeKey(version, NibblePath(), std::forward(address)); + } bool operator==(const InternalNodeKey& other) const { return version_ == other.version_ && path_ == other.path_; } // Compare by version then by path bool operator<(const InternalNodeKey& other) const { + if (version_ == other.version_ && path_ == other.path_) { + return address_ < other.address_; + } if (version_ == other.version_) { return path_ < other.path_; } @@ -48,12 +56,15 @@ class InternalNodeKey { Version version() const { return version_; } + const std::string& address() const { return address_; } + const NibblePath& path() const { return path_; } NibblePath& path() { return path_; } private: Version version_; NibblePath path_; + std::string address_; }; // The key for a leaf node of a sparse merkle tree. diff --git a/kvbc/include/sparse_merkle/tree.h b/kvbc/include/sparse_merkle/tree.h index 59911a36d6..842fddde69 100644 --- a/kvbc/include/sparse_merkle/tree.h +++ b/kvbc/include/sparse_merkle/tree.h @@ -43,11 +43,15 @@ namespace sparse_merkle { class Tree { public: Tree() = default; - explicit Tree(std::shared_ptr db_reader) : db_reader_(db_reader) { reset(); } + explicit Tree(std::shared_ptr db_reader, std::string address = "") + : db_reader_(db_reader), address_(address) { + reset(); + } const Hash& get_root_hash() const { return root_.hash(); } Version get_version() const { return root_.version(); } bool empty() const { return root_.numChildren() == 0; } + std::string toString() { return root_.toString(); } // Add or update key-value pairs given in `updates`, and remove keys in // `deleted_keys`. @@ -74,7 +78,7 @@ class Tree { // // This is necessary to do before updates, as we only allow updating the // latest tree. - void reset() { root_ = db_reader_->get_latest_root(); } + void reset() { root_ = db_reader_->get_latest_root(address_); } UpdateBatch update_impl(const concord::kvbc::SetOfKeyValuePairs& updates, const concord::kvbc::KeysVector& deleted_keys, @@ -82,6 +86,7 @@ class Tree { std::shared_ptr db_reader_; BatchedInternalNode root_; + std::string address_; }; } // namespace sparse_merkle diff --git a/kvbc/include/sparse_merkle/update_cache.h b/kvbc/include/sparse_merkle/update_cache.h index bea85595e6..9386e053d4 100644 --- a/kvbc/include/sparse_merkle/update_cache.h +++ b/kvbc/include/sparse_merkle/update_cache.h @@ -34,8 +34,8 @@ namespace detail { // A new instance of this structure is created during every update call. class UpdateCache { public: - UpdateCache(const BatchedInternalNode& root, const std::shared_ptr& db_reader) - : version_(root.version() + 1), db_reader_(db_reader), original_root_(root) {} + UpdateCache(const BatchedInternalNode& root, const std::shared_ptr& db_reader, std::string address) + : version_(root.version() + 1), db_reader_(db_reader), original_root_(root), address_(address) {} const StaleNodeIndexes& stale() const { return stale_; } const auto& internalNodes() const { return internal_nodes_; } @@ -55,8 +55,8 @@ class UpdateCache { void putStale(const std::optional& key); void putStale(const InternalNodeKey& key); - void put(const NibblePath& path, const BatchedInternalNode& node); - void remove(const NibblePath& path); + void put(const InternalNodeKey& key, const BatchedInternalNode& node); + void remove(const InternalNodeKey& key); private: // The version of the tree after this update is complete. @@ -73,7 +73,8 @@ class UpdateCache { // // These nodes are mutable and are all being updated to a single version. // Therefore we key them by their NibblePath alone, without a version. - std::map internal_nodes_; + std::map internal_nodes_; + std::string address_; }; } // namespace detail diff --git a/kvbc/include/sparse_merkle/walker.h b/kvbc/include/sparse_merkle/walker.h index 854cb2cf82..f07efb4eae 100644 --- a/kvbc/include/sparse_merkle/walker.h +++ b/kvbc/include/sparse_merkle/walker.h @@ -28,7 +28,7 @@ typedef std::stack> NodeSt // A class for ascending and descending the tree. This is used for updates. class Walker { public: - Walker(UpdateCache& cache) : cache_(cache), current_node_(cache.getRoot()) { + Walker(UpdateCache& cache, std::string address) : cache_(cache), current_node_(cache.getRoot()), address_(address) { // Save the version of the root node, in case we only update the root node. stale_version_ = current_node_.version(); } @@ -74,6 +74,7 @@ class Walker { NodeStack stack_; BatchedInternalNode current_node_; NibblePath nibble_path_; + std::string address_; }; } // namespace detail diff --git a/kvbc/src/categorization/block_merkle_category.cpp b/kvbc/src/categorization/block_merkle_category.cpp index abf13aa439..ee35d46f3d 100644 --- a/kvbc/src/categorization/block_merkle_category.cpp +++ b/kvbc/src/categorization/block_merkle_category.cpp @@ -76,12 +76,12 @@ BatchedInternalNodeKey toBatchedInternalNodeKey(const sparse_merkle::InternalNod BatchedInternalNodeKey toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey&& key) { auto path = NibblePath{static_cast(key.path().length()), key.path().move_data()}; - return BatchedInternalNodeKey{key.version().value(), std::move(path)}; + return BatchedInternalNodeKey{key.version().value(), std::move(path), key.address()}; } -std::vector rootKey(uint64_t version) { +std::vector rootKey(uint64_t version, std::string address = "") { auto v = sparse_merkle::Version(version); - return serialize(toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey::root(v))); + return serialize(toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey::root(v, address))); } // A key used in tree_.update() @@ -775,8 +775,8 @@ uint64_t BlockMerkleCategory::getLatestTreeVersion() const { return 0; } -sparse_merkle::BatchedInternalNode BlockMerkleCategory::Reader::get_latest_root() const { - if (auto latest_root_key = db_.get(BLOCK_MERKLE_INTERNAL_NODES_CF, rootKey(0))) { +sparse_merkle::BatchedInternalNode BlockMerkleCategory::Reader::get_latest_root(std::string address) const { + if (auto latest_root_key = db_.get(BLOCK_MERKLE_INTERNAL_NODES_CF, rootKey(0, address))) { if (auto serialized = db_.get(BLOCK_MERKLE_INTERNAL_NODES_CF, *latest_root_key)) { return deserializeBatchedInternalNode(*serialized); } diff --git a/kvbc/src/merkle_tree_db_adapter.cpp b/kvbc/src/merkle_tree_db_adapter.cpp index c799c8b823..1006ca9868 100644 --- a/kvbc/src/merkle_tree_db_adapter.cpp +++ b/kvbc/src/merkle_tree_db_adapter.cpp @@ -446,7 +446,7 @@ std::pair DBAdap return smTree_.update_with_cache(updates, KeysVector{std::cbegin(deletes), std::cend(deletes)}); } -BatchedInternalNode DBAdapter::Reader::get_latest_root() const { +BatchedInternalNode DBAdapter::Reader::get_latest_root(std::string address) const { const auto lastBlock = adapter_.getLastReachableBlockId(); if (lastBlock == 0) { return BatchedInternalNode{}; @@ -462,7 +462,7 @@ BatchedInternalNode DBAdapter::Reader::get_latest_root() const { return BatchedInternalNode{}; } - return get_internal(InternalNodeKey::root(stateRootVersion)); + return get_internal(InternalNodeKey::root(stateRootVersion, address)); } BatchedInternalNode DBAdapter::Reader::get_internal(const InternalNodeKey &key) const { @@ -708,7 +708,7 @@ KeysVector DBAdapter::internalProvableKeysForVersion(const Version &version) con // Rely on the fact that root internal keys always precede non-root ones - due to lexicographical ordering and root // internal keys having empty nibble paths. See InternalNodeKey serialization code. return keysForVersion(db_, - DBKeyManipulator::genInternalDbKey(InternalNodeKey::root(version)), + DBKeyManipulator::genInternalDbKey(InternalNodeKey::root(version, "")), version, EKeySubtype::Internal, [](const Key &key) { return DBKeyManipulator::extractVersionFromInternalKey(key); }); diff --git a/kvbc/src/sparse_merkle/tree.cpp b/kvbc/src/sparse_merkle/tree.cpp index 238f7e241c..cb91652582 100644 --- a/kvbc/src/sparse_merkle/tree.cpp +++ b/kvbc/src/sparse_merkle/tree.cpp @@ -133,14 +133,14 @@ UpdateBatch Tree::update(const concord::kvbc::SetOfKeyValuePairs& updates, histograms.num_deleted_keys->record(deleted_keys.size()); TimeRecorder scoped_timer(*histograms.update); reset(); - UpdateCache cache(root_, db_reader_); + UpdateCache cache(root_, db_reader_, address_); return update_impl(updates, deleted_keys, cache); } std::pair Tree::update_with_cache(const concord::kvbc::SetOfKeyValuePairs& updates, const concord::kvbc::KeysVector& deleted_keys) { reset(); - UpdateCache cache(root_, db_reader_); + UpdateCache cache(root_, db_reader_, address_); auto batch = update_impl(updates, deleted_keys, cache); return std::make_pair(batch, cache); } @@ -155,7 +155,7 @@ UpdateBatch Tree::update_impl(const concord::kvbc::SetOfKeyValuePairs& updates, // Deletes come before inserts because it makes more semantic sense. A user can delete a key and then write a new // version, but it makes no sense to add a new version and then delete a key. for (auto& key : deleted_keys) { - Walker walker(cache); + Walker walker(cache, address_); auto key_hash = hasher.hash(key.data(), key.length()); sparse_merkle::remove(walker, key_hash); } @@ -171,7 +171,7 @@ UpdateBatch Tree::update_impl(const concord::kvbc::SetOfKeyValuePairs& updates, LeafNode leaf_node{val}; LeafKey leaf_key{hasher.hash(key.data(), key.length()), version}; LeafChild child{leaf_hash, leaf_key}; - Walker walker(cache); + Walker walker(cache, address_); insert(walker, child); batch.leaf_nodes.emplace_back(leaf_key, leaf_node); } @@ -180,7 +180,7 @@ UpdateBatch Tree::update_impl(const concord::kvbc::SetOfKeyValuePairs& updates, batch.stale = cache.stale(); batch.stale.stale_since_version = version; for (auto& it : cache.internalNodes()) { - batch.internal_nodes.emplace_back(InternalNodeKey(version, it.first), it.second); + batch.internal_nodes.emplace_back(it.first, it.second); } updateBatchHistograms(batch); diff --git a/kvbc/src/sparse_merkle/update_cache.cpp b/kvbc/src/sparse_merkle/update_cache.cpp index 2d1e2a2248..b8455a7a17 100644 --- a/kvbc/src/sparse_merkle/update_cache.cpp +++ b/kvbc/src/sparse_merkle/update_cache.cpp @@ -15,7 +15,7 @@ namespace concord::kvbc::sparse_merkle::detail { BatchedInternalNode UpdateCache::getInternalNode(const InternalNodeKey& key) { - auto it = internal_nodes_.find(key.path()); + auto it = internal_nodes_.find(key); if (it != internal_nodes_.end()) { return it->second; } @@ -29,17 +29,17 @@ void UpdateCache::putStale(const std::optional& key) { } const BatchedInternalNode& UpdateCache::getRoot() { - auto it = internal_nodes_.find(NibblePath()); + auto it = internal_nodes_.find({version_, NibblePath(), address_}); if (it != internal_nodes_.end()) { return it->second; } return original_root_; } -void UpdateCache::put(const NibblePath& path, const BatchedInternalNode& node) { internal_nodes_[path] = node; } +void UpdateCache::put(const InternalNodeKey& key, const BatchedInternalNode& node) { internal_nodes_[key] = node; } void UpdateCache::putStale(const InternalNodeKey& key) { stale_.internal_keys.insert(key); } -void UpdateCache::remove(const NibblePath& path) { internal_nodes_.erase(path); } +void UpdateCache::remove(const InternalNodeKey& key) { internal_nodes_.erase(key); } } // namespace concord::kvbc::sparse_merkle::detail diff --git a/kvbc/src/sparse_merkle/walker.cpp b/kvbc/src/sparse_merkle/walker.cpp index 5687f934b0..353cfcc2ec 100644 --- a/kvbc/src/sparse_merkle/walker.cpp +++ b/kvbc/src/sparse_merkle/walker.cpp @@ -36,7 +36,7 @@ void Walker::descend(const Hash& key, Version next_version) { stack_.push(current_node_); Nibble next_nibble = key.getNibble(depth()); nibble_path_.append(next_nibble); - InternalNodeKey next_internal_key{next_version, nibble_path_}; + InternalNodeKey next_internal_key{next_version, nibble_path_, address_}; current_node_ = cache_.getInternalNode(next_internal_key); stale_version_ = current_node_.version(); } @@ -68,7 +68,7 @@ void Walker::ascend() { std::optional Walker::removeCurrentNode() { markCurrentNodeStale(); - cache_.remove(nibble_path_); + cache_.remove({stale_version_, nibble_path_, address_}); if (!atRoot()) { return pop().first; } @@ -78,7 +78,7 @@ std::optional Walker::removeCurrentNode() { void Walker::insertEmptyRootAtCurrentVersion() { auto children = BatchedInternalNode::Children{}; children[0] = InternalChild{PLACEHOLDER_HASH, cache_.version()}; - cache_.put(NibblePath{}, BatchedInternalNode{children}); + cache_.put({cache_.version(), NibblePath{}, address_}, BatchedInternalNode{children}); } std::pair Walker::pop() { @@ -97,10 +97,10 @@ void Walker::markCurrentNodeStale() { // Don't mark the initial root stale, and don't mark an already cached node // stale. if (stale_version_ != Version(0) && stale_version_ != version()) { - cache_.putStale(InternalNodeKey(stale_version_, nibble_path_)); + cache_.putStale(InternalNodeKey(stale_version_, nibble_path_, address_)); } } -void Walker::cacheCurrentNode() { cache_.put(nibble_path_, current_node_); } +void Walker::cacheCurrentNode() { cache_.put({cache_.version(), nibble_path_, address_}, current_node_); } } // namespace concord::kvbc::sparse_merkle::detail diff --git a/kvbc/test/sparse_merkle/test_db.h b/kvbc/test/sparse_merkle/test_db.h index bac6126db5..368f700b7e 100644 --- a/kvbc/test/sparse_merkle/test_db.h +++ b/kvbc/test/sparse_merkle/test_db.h @@ -15,32 +15,35 @@ #include "sparse_merkle/keys.h" #include "sparse_merkle/db_reader.h" #include "sparse_merkle/internal_node.h" +#include "kvbc/include/categorization/details.h" using namespace std; using namespace concord::kvbc::sparse_merkle; -class TestDB : public concord::kvbc::sparse_merkle::IDBReader { +class TestDB : public IDBReader { public: void put(const LeafKey& key, const LeafNode& val) { leaf_nodes_[key] = val; } void put(const InternalNodeKey& key, const BatchedInternalNode& val) { + assert(internal_nodes_.find(key) == internal_nodes_.end()); + internal_nodes_[key] = val; - if (latest_version_ < key.version()) { - latest_version_ = key.version(); + if (latest_version_[key.address()] < key.version()) { + latest_version_[key.address()] = key.version(); } } - BatchedInternalNode get_latest_root() const override { - if (latest_version_ == 0) { + BatchedInternalNode get_latest_root(std::string address = "") const override { + if (latest_version_.find(address) == latest_version_.end()) { return BatchedInternalNode(); } - auto root_key = InternalNodeKey::root(latest_version_); + auto root_key = InternalNodeKey::root(latest_version_.at(address), address); return internal_nodes_.at(root_key); } BatchedInternalNode get_internal(const InternalNodeKey& key) const override { return internal_nodes_.at(key); } private: - Version latest_version_ = 0; + map latest_version_; map leaf_nodes_; map internal_nodes_; }; diff --git a/kvbc/test/sparse_merkle/tree_test.cpp b/kvbc/test/sparse_merkle/tree_test.cpp index 1bc24e4e68..217c162bfb 100644 --- a/kvbc/test/sparse_merkle/tree_test.cpp +++ b/kvbc/test/sparse_merkle/tree_test.cpp @@ -274,7 +274,7 @@ TEST(tree_tests, insert_single_leaf_node) { // Only a single node was created. It's key is made up of an empty nibble // path, since there are no collisions requring going deeper into the tree. auto& root = batch.internal_nodes.at(0); - ASSERT_EQ(InternalNodeKey(1, NibblePath()), root.first); + ASSERT_EQ(InternalNodeKey(1, NibblePath(), ""), root.first); // There should be a root and a single leaf ASSERT_EQ(2, root.second.numChildren()); diff --git a/kvbc/test/sparse_merkle_storage/db_adapter_unit_test.cpp b/kvbc/test/sparse_merkle_storage/db_adapter_unit_test.cpp index b0c86a7a93..120f04052a 100644 --- a/kvbc/test/sparse_merkle_storage/db_adapter_unit_test.cpp +++ b/kvbc/test/sparse_merkle_storage/db_adapter_unit_test.cpp @@ -113,7 +113,7 @@ bool hasProvableStaleIndexKeysSince(const std::shared_ptr &db, const bool hasInternalKeysForVersion(const std::shared_ptr &db, const Version &version) { auto iter = db->getIteratorGuard(); - const auto key = iter->seekAtLeast(DBKeyManipulator::genInternalDbKey(InternalNodeKey::root(version))).first; + const auto key = iter->seekAtLeast(DBKeyManipulator::genInternalDbKey(InternalNodeKey::root(version, ""))).first; if (!key.empty() && DBKeyManipulator::getDBKeyType(key) == EDBKeyType::Key && DBKeyManipulator::getKeySubtype(key) == EKeySubtype::Internal && DBKeyManipulator::extractVersionFromInternalKey(key) == version) { diff --git a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp index 0a6828afc0..49060297d1 100644 --- a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp +++ b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp @@ -182,7 +182,7 @@ TEST(key_manipulator, internal_key_even) { // second byte of the nibble path is 0x34 (by appending 0x03 and 0x04) path.append(0x03); path.append(0x04); - const auto internalKey = InternalNodeKey{defaultVersion, path}; + const auto internalKey = InternalNodeKey{defaultVersion, path, ""}; const auto key = DBKeyManipulator::genInternalDbKey(internalKey); // Expect that two bytes of nibbles have been written. const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Internal) + @@ -201,7 +201,7 @@ TEST(key_manipulator, internal_key_odd) { path.append(0x02); // second byte of the nibble path is 0x30 (by appending 0x03) path.append(0x03); - const auto internalKey = InternalNodeKey{defaultVersion, path}; + const auto internalKey = InternalNodeKey{defaultVersion, path, ""}; const auto key = DBKeyManipulator::genInternalDbKey(internalKey); // Expect that two bytes of nibbles have been written. const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Internal) + @@ -292,7 +292,7 @@ TEST(key_manipulator, stale_db_key_internal) { // second byte of the nibble path is 0x34 (by appending 0x03 and 0x04) path.append(0x03); path.append(0x04); - const auto internalKey = InternalNodeKey{defaultVersion, path}; + const auto internalKey = InternalNodeKey{defaultVersion, path, ""}; const auto key = DBKeyManipulator::genStaleDbKey(internalKey, defaultVersion); const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::ProvableStale) + From 7341f5a38aa0745fc81e6ec67e513f060bc1d55e Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Tue, 14 Mar 2023 15:08:12 +0200 Subject: [PATCH 2/9] Fix for sparse_merkle_storage_serialization_unit_test. Fix to take in consideration the address field of the InternalNodeKey. --- .../serialization_unit_test.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp index 49060297d1..4c696529d7 100644 --- a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp +++ b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp @@ -187,8 +187,9 @@ TEST(key_manipulator, internal_key_even) { // Expect that two bytes of nibbles have been written. const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Internal) + serializeIntegral(internalKey.version().value()) + serializeIntegral(std::uint8_t{4}) + - serializeIntegral(std::uint8_t{0x12}) + serializeIntegral(std::uint8_t{0x34})); - ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 2); + serializeIntegral(std::uint8_t{0x12}) + serializeIntegral(std::uint8_t{0x34}) + + serializeIntegral(std::uint8_t{0x0})); + ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 2 + 1); ASSERT_TRUE(key == expected); } @@ -206,8 +207,9 @@ TEST(key_manipulator, internal_key_odd) { // Expect that two bytes of nibbles have been written. const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Internal) + serializeIntegral(internalKey.version().value()) + serializeIntegral(std::uint8_t{3}) + - serializeIntegral(std::uint8_t{0x12}) + serializeIntegral(std::uint8_t{0x30})); - ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 2); + serializeIntegral(std::uint8_t{0x12}) + serializeIntegral(std::uint8_t{0x30}) + + serializeIntegral(std::uint8_t{0x0})); + ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 2 + 1); ASSERT_TRUE(key == expected); } @@ -297,7 +299,7 @@ TEST(key_manipulator, stale_db_key_internal) { const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::ProvableStale) + serializeIntegral(defaultBlockId) + DBKeyManipulator::genInternalDbKey(internalKey).toString()); - ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 1 + 8 + 1 + 2); + ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 1 + 8 + 1 + 2 + 1); ASSERT_TRUE(key == expected); } From 3309a76fbf82e2bb6fc03ed7fadbf54b4c0132d4 Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Thu, 16 Mar 2023 11:18:49 +0200 Subject: [PATCH 3/9] Serialization fixes. * Fix for correct reading of the size of nibbles. * Changes to the way we deserialize nibbles - we no longer rely on them being the last component in the buffer. * Fix for deserialization of the address field. * Added a non empty address in unit test serialization_unit_test/stale_db_key_internal for better code coverage. --- kvbc/include/merkle_tree_serialization.h | 29 ++++++++++++------- kvbc/include/sparse_merkle/base_types.h | 7 ++++- .../serialization_unit_test.cpp | 5 ++-- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/kvbc/include/merkle_tree_serialization.h b/kvbc/include/merkle_tree_serialization.h index ebefc94b49..441cbd280e 100644 --- a/kvbc/include/merkle_tree_serialization.h +++ b/kvbc/include/merkle_tree_serialization.h @@ -317,11 +317,16 @@ inline std::vector deserialize>(const co template <> inline sparse_merkle::NibblePath deserialize(const concordUtils::Sliver &buf) { - ConcordAssert(buf.length() >= sizeof(std::uint8_t)); + static const int nibblePathSizePrefixInBytes = sizeof(std::uint8_t); + ConcordAssert(buf.length() >= nibblePathSizePrefixInBytes); // NOLINTNEXTLINE(bugprone-signed-char-misuse) - const auto nibbleCount = static_cast(buf[0]); - const auto path = deserialize>( - concordUtils::Sliver{buf, sizeof(std::uint8_t), buf.length() - sizeof(std::uint8_t)}); + size_t nibbleCount = static_cast(buf[0]); + size_t pathBytesCount = sparse_merkle::NibblePath::pathLengthInBytes(nibbleCount); + std::vector path; + if (pathBytesCount > 0) { + path = + deserialize>(concordUtils::Sliver{buf, nibblePathSizePrefixInBytes, pathBytesCount}); + } return sparse_merkle::NibblePath{nibbleCount, path}; } @@ -329,10 +334,11 @@ template <> inline std::string deserialize(const concordUtils::Sliver &buf) { ConcordAssert(buf.length() >= sizeof(std::uint8_t)); // NOLINTNEXTLINE(bugprone-signed-char-misuse) - const auto addrSize = static_cast(buf[0]); + const auto addrSize = static_cast(buf[0]); + ConcordAssert(buf.length() > addrSize); std::string address; address.reserve(addrSize); - for (auto i = 0u; i < addrSize; ++i) { + for (auto i = 1u; i <= addrSize; ++i) { address.push_back(buf[i]); } return address; @@ -348,13 +354,16 @@ inline sparse_merkle::LeafKey deserialize(const concordU template <> inline sparse_merkle::InternalNodeKey deserialize(const concordUtils::Sliver &buf) { + static const int nibblePathSizePrefixInBytes = sizeof(std::uint8_t); // 1 byte for the nibbleCount auto version = deserialize(buf); auto nibblePath = deserialize(concordUtils::Sliver{ buf, sparse_merkle::Version::SIZE_IN_BYTES, buf.length() - sparse_merkle::Version::SIZE_IN_BYTES}); - std::string address = - deserialize(concordUtils::Sliver{buf, - sparse_merkle::Version::SIZE_IN_BYTES + nibblePath.length(), - buf.length() - sparse_merkle::Version::SIZE_IN_BYTES}); + std::string address = deserialize( + concordUtils::Sliver{buf, + sparse_merkle::Version::SIZE_IN_BYTES + nibblePathSizePrefixInBytes + + sparse_merkle::NibblePath::pathLengthInBytes(nibblePath.length()), + buf.length() - sparse_merkle::Version::SIZE_IN_BYTES - nibblePathSizePrefixInBytes - + sparse_merkle::NibblePath::pathLengthInBytes(nibblePath.length())}); return sparse_merkle::InternalNodeKey{std::move(version), std::move(nibblePath), std::move(address)}; } diff --git a/kvbc/include/sparse_merkle/base_types.h b/kvbc/include/sparse_merkle/base_types.h index b03412f08e..3d0c60d020 100644 --- a/kvbc/include/sparse_merkle/base_types.h +++ b/kvbc/include/sparse_merkle/base_types.h @@ -243,7 +243,7 @@ class NibblePath { NibblePath() : num_nibbles_(0) {} NibblePath(size_t num_nibbles, const std::vector& path) : num_nibbles_(num_nibbles), path_(path) { ConcordAssert(num_nibbles < Hash::MAX_NIBBLES); - ConcordAssert(path.size() == (num_nibbles % 2 ? (num_nibbles + 1) / 2 : num_nibbles / 2)); + ConcordAssert(path.size() == pathLengthInBytes(num_nibbles)); } bool operator==(const NibblePath& other) const { return num_nibbles_ == other.num_nibbles_ && path_ == other.path_; } @@ -263,6 +263,11 @@ class NibblePath { return false; } + // Return the count of elements (bytes) in path_ + static size_t pathLengthInBytes(size_t num_nibbles) { + return (num_nibbles % 2 ? (num_nibbles + 1) / 2 : num_nibbles / 2); + } + // Return the length of the path_ in nibbles size_t length() const { return num_nibbles_; } diff --git a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp index 4c696529d7..83785b4b9a 100644 --- a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp +++ b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp @@ -294,12 +294,13 @@ TEST(key_manipulator, stale_db_key_internal) { // second byte of the nibble path is 0x34 (by appending 0x03 and 0x04) path.append(0x03); path.append(0x04); - const auto internalKey = InternalNodeKey{defaultVersion, path, ""}; + std::string address = "abcdefg123"; + const auto internalKey = InternalNodeKey{defaultVersion, path, address}; const auto key = DBKeyManipulator::genStaleDbKey(internalKey, defaultVersion); const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::ProvableStale) + serializeIntegral(defaultBlockId) + DBKeyManipulator::genInternalDbKey(internalKey).toString()); - ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 1 + 8 + 1 + 2 + 1); + ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 1 + 8 + 1 + 2 + 1 + address.size()); ASSERT_TRUE(key == expected); } From 519dd97c1845a9bb712291d503b0db67b1ef7cad Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Thu, 16 Mar 2023 14:55:39 +0200 Subject: [PATCH 4/9] Addressed clang-tidy warnings. --- kvbc/src/categorization/block_merkle_category.cpp | 2 +- kvbc/test/sparse_merkle/test_db.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kvbc/src/categorization/block_merkle_category.cpp b/kvbc/src/categorization/block_merkle_category.cpp index ee35d46f3d..2fa6ce6a09 100644 --- a/kvbc/src/categorization/block_merkle_category.cpp +++ b/kvbc/src/categorization/block_merkle_category.cpp @@ -79,7 +79,7 @@ BatchedInternalNodeKey toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey&& return BatchedInternalNodeKey{key.version().value(), std::move(path), key.address()}; } -std::vector rootKey(uint64_t version, std::string address = "") { +std::vector rootKey(uint64_t version, const std::string& address = "") { auto v = sparse_merkle::Version(version); return serialize(toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey::root(v, address))); } diff --git a/kvbc/test/sparse_merkle/test_db.h b/kvbc/test/sparse_merkle/test_db.h index 368f700b7e..18d4feb90f 100644 --- a/kvbc/test/sparse_merkle/test_db.h +++ b/kvbc/test/sparse_merkle/test_db.h @@ -24,7 +24,7 @@ class TestDB : public IDBReader { public: void put(const LeafKey& key, const LeafNode& val) { leaf_nodes_[key] = val; } void put(const InternalNodeKey& key, const BatchedInternalNode& val) { - assert(internal_nodes_.find(key) == internal_nodes_.end()); + ConcordAssert(internal_nodes_.find(key) == internal_nodes_.end()); internal_nodes_[key] = val; if (latest_version_[key.address()] < key.version()) { From 85e64f89362b5e747ba25d25343188ee1e6cfba6 Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Fri, 17 Mar 2023 19:37:19 +0200 Subject: [PATCH 5/9] Moved address to be the first field in InternalNodeKey. --- kvbc/cmf/categorized_kvbc_msgs.cmf | 2 +- kvbc/include/merkle_tree_serialization.h | 23 ++++++++++--------- kvbc/include/sparse_merkle/keys.h | 10 ++++---- .../categorization/block_merkle_category.cpp | 6 ++--- kvbc/src/merkle_tree_db_adapter.cpp | 4 ++-- kvbc/src/sparse_merkle/update_cache.cpp | 2 +- kvbc/src/sparse_merkle/walker.cpp | 10 ++++---- kvbc/test/sparse_merkle/test_db.h | 2 +- kvbc/test/sparse_merkle/tree_test.cpp | 2 +- .../db_adapter_unit_test.cpp | 2 +- .../serialization_unit_test.cpp | 23 +++++++++++-------- 11 files changed, 45 insertions(+), 41 deletions(-) diff --git a/kvbc/cmf/categorized_kvbc_msgs.cmf b/kvbc/cmf/categorized_kvbc_msgs.cmf index cd95c1ccce..b8333e00a1 100644 --- a/kvbc/cmf/categorized_kvbc_msgs.cmf +++ b/kvbc/cmf/categorized_kvbc_msgs.cmf @@ -199,9 +199,9 @@ Msg NibblePath 5001 { Msg BatchedInternalNodeKey 5002 { # The version of this key is the *tree* version, not block version + string address uint64 version NibblePath path - string address } Msg LeafChild 5003 { diff --git a/kvbc/include/merkle_tree_serialization.h b/kvbc/include/merkle_tree_serialization.h index 441cbd280e..b8c25c2af8 100644 --- a/kvbc/include/merkle_tree_serialization.h +++ b/kvbc/include/merkle_tree_serialization.h @@ -111,9 +111,9 @@ inline std::string serializeImp(const sparse_merkle::Hash &hash) { } inline void serializeImp(const sparse_merkle::InternalNodeKey &key, std::string &out) { + serializeImp(key.address(), out); serializeImp(key.version().value(), out); serializeImp(key.path(), out); - serializeImp(key.address(), out); } inline std::string serializeImp(const sparse_merkle::InternalNodeKey &key) { @@ -354,17 +354,18 @@ inline sparse_merkle::LeafKey deserialize(const concordU template <> inline sparse_merkle::InternalNodeKey deserialize(const concordUtils::Sliver &buf) { - static const int nibblePathSizePrefixInBytes = sizeof(std::uint8_t); // 1 byte for the nibbleCount - auto version = deserialize(buf); - auto nibblePath = deserialize(concordUtils::Sliver{ - buf, sparse_merkle::Version::SIZE_IN_BYTES, buf.length() - sparse_merkle::Version::SIZE_IN_BYTES}); - std::string address = deserialize( + static const int addressPathSizePrefixInBytes = sizeof(std::uint8_t); // 1 byte for the address size + std::string address = deserialize(buf); + auto version = deserialize( concordUtils::Sliver{buf, - sparse_merkle::Version::SIZE_IN_BYTES + nibblePathSizePrefixInBytes + - sparse_merkle::NibblePath::pathLengthInBytes(nibblePath.length()), - buf.length() - sparse_merkle::Version::SIZE_IN_BYTES - nibblePathSizePrefixInBytes - - sparse_merkle::NibblePath::pathLengthInBytes(nibblePath.length())}); - return sparse_merkle::InternalNodeKey{std::move(version), std::move(nibblePath), std::move(address)}; + addressPathSizePrefixInBytes + address.size(), + buf.length() - addressPathSizePrefixInBytes - address.size()}); + auto nibblePath = deserialize(concordUtils::Sliver{ + buf, + addressPathSizePrefixInBytes + address.size() + sparse_merkle::Version::SIZE_IN_BYTES, + buf.length() - addressPathSizePrefixInBytes - address.size() - sparse_merkle::Version::SIZE_IN_BYTES}); + + return sparse_merkle::InternalNodeKey{std::move(address), std::move(version), std::move(nibblePath)}; } template <> diff --git a/kvbc/include/sparse_merkle/keys.h b/kvbc/include/sparse_merkle/keys.h index d702ac7c0b..cd364fd3e5 100644 --- a/kvbc/include/sparse_merkle/keys.h +++ b/kvbc/include/sparse_merkle/keys.h @@ -25,13 +25,13 @@ namespace sparse_merkle { class InternalNodeKey { public: template - InternalNodeKey(Version version, const NibblePath& path, T&& address) - : version_(version), path_(path), address_(std::forward(address)) {} + InternalNodeKey(T&& address, Version version, const NibblePath& path) + : address_(std::forward(address)), version_(version), path_(path) {} // Return the root of a sparse merkle tree at a given version. template - static InternalNodeKey root(Version version, T&& address) { - return InternalNodeKey(version, NibblePath(), std::forward(address)); + static InternalNodeKey root(T&& address, Version version) { + return InternalNodeKey(std::forward(address), version, NibblePath()); } bool operator==(const InternalNodeKey& other) const { return version_ == other.version_ && path_ == other.path_; } @@ -62,9 +62,9 @@ class InternalNodeKey { NibblePath& path() { return path_; } private: + std::string address_; Version version_; NibblePath path_; - std::string address_; }; // The key for a leaf node of a sparse merkle tree. diff --git a/kvbc/src/categorization/block_merkle_category.cpp b/kvbc/src/categorization/block_merkle_category.cpp index 2fa6ce6a09..7fd456b54a 100644 --- a/kvbc/src/categorization/block_merkle_category.cpp +++ b/kvbc/src/categorization/block_merkle_category.cpp @@ -71,17 +71,17 @@ VersionedKey leafKeyToVersionedKey(const sparse_merkle::LeafKey& leaf_key) { // favor of the one immediately below it. BatchedInternalNodeKey toBatchedInternalNodeKey(const sparse_merkle::InternalNodeKey& key) { auto path = NibblePath{static_cast(key.path().length()), key.path().data()}; - return BatchedInternalNodeKey{key.version().value(), std::move(path)}; + return BatchedInternalNodeKey{key.address(), key.version().value(), std::move(path)}; } BatchedInternalNodeKey toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey&& key) { auto path = NibblePath{static_cast(key.path().length()), key.path().move_data()}; - return BatchedInternalNodeKey{key.version().value(), std::move(path), key.address()}; + return BatchedInternalNodeKey{key.address(), key.version().value(), std::move(path)}; } std::vector rootKey(uint64_t version, const std::string& address = "") { auto v = sparse_merkle::Version(version); - return serialize(toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey::root(v, address))); + return serialize(toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey::root(address, v))); } // A key used in tree_.update() diff --git a/kvbc/src/merkle_tree_db_adapter.cpp b/kvbc/src/merkle_tree_db_adapter.cpp index 1006ca9868..b42c530d7c 100644 --- a/kvbc/src/merkle_tree_db_adapter.cpp +++ b/kvbc/src/merkle_tree_db_adapter.cpp @@ -462,7 +462,7 @@ BatchedInternalNode DBAdapter::Reader::get_latest_root(std::string address) cons return BatchedInternalNode{}; } - return get_internal(InternalNodeKey::root(stateRootVersion, address)); + return get_internal(InternalNodeKey::root(address, stateRootVersion)); } BatchedInternalNode DBAdapter::Reader::get_internal(const InternalNodeKey &key) const { @@ -708,7 +708,7 @@ KeysVector DBAdapter::internalProvableKeysForVersion(const Version &version) con // Rely on the fact that root internal keys always precede non-root ones - due to lexicographical ordering and root // internal keys having empty nibble paths. See InternalNodeKey serialization code. return keysForVersion(db_, - DBKeyManipulator::genInternalDbKey(InternalNodeKey::root(version, "")), + DBKeyManipulator::genInternalDbKey(InternalNodeKey::root("", version)), version, EKeySubtype::Internal, [](const Key &key) { return DBKeyManipulator::extractVersionFromInternalKey(key); }); diff --git a/kvbc/src/sparse_merkle/update_cache.cpp b/kvbc/src/sparse_merkle/update_cache.cpp index b8455a7a17..a66c10fc19 100644 --- a/kvbc/src/sparse_merkle/update_cache.cpp +++ b/kvbc/src/sparse_merkle/update_cache.cpp @@ -29,7 +29,7 @@ void UpdateCache::putStale(const std::optional& key) { } const BatchedInternalNode& UpdateCache::getRoot() { - auto it = internal_nodes_.find({version_, NibblePath(), address_}); + auto it = internal_nodes_.find({address_, version_, NibblePath()}); if (it != internal_nodes_.end()) { return it->second; } diff --git a/kvbc/src/sparse_merkle/walker.cpp b/kvbc/src/sparse_merkle/walker.cpp index 353cfcc2ec..b8f601f73b 100644 --- a/kvbc/src/sparse_merkle/walker.cpp +++ b/kvbc/src/sparse_merkle/walker.cpp @@ -36,7 +36,7 @@ void Walker::descend(const Hash& key, Version next_version) { stack_.push(current_node_); Nibble next_nibble = key.getNibble(depth()); nibble_path_.append(next_nibble); - InternalNodeKey next_internal_key{next_version, nibble_path_, address_}; + InternalNodeKey next_internal_key{address_, next_version, nibble_path_}; current_node_ = cache_.getInternalNode(next_internal_key); stale_version_ = current_node_.version(); } @@ -68,7 +68,7 @@ void Walker::ascend() { std::optional Walker::removeCurrentNode() { markCurrentNodeStale(); - cache_.remove({stale_version_, nibble_path_, address_}); + cache_.remove({address_, stale_version_, nibble_path_}); if (!atRoot()) { return pop().first; } @@ -78,7 +78,7 @@ std::optional Walker::removeCurrentNode() { void Walker::insertEmptyRootAtCurrentVersion() { auto children = BatchedInternalNode::Children{}; children[0] = InternalChild{PLACEHOLDER_HASH, cache_.version()}; - cache_.put({cache_.version(), NibblePath{}, address_}, BatchedInternalNode{children}); + cache_.put({address_, cache_.version(), NibblePath{}}, BatchedInternalNode{children}); } std::pair Walker::pop() { @@ -97,10 +97,10 @@ void Walker::markCurrentNodeStale() { // Don't mark the initial root stale, and don't mark an already cached node // stale. if (stale_version_ != Version(0) && stale_version_ != version()) { - cache_.putStale(InternalNodeKey(stale_version_, nibble_path_, address_)); + cache_.putStale(InternalNodeKey(address_, stale_version_, nibble_path_)); } } -void Walker::cacheCurrentNode() { cache_.put({cache_.version(), nibble_path_, address_}, current_node_); } +void Walker::cacheCurrentNode() { cache_.put({address_, cache_.version(), nibble_path_}, current_node_); } } // namespace concord::kvbc::sparse_merkle::detail diff --git a/kvbc/test/sparse_merkle/test_db.h b/kvbc/test/sparse_merkle/test_db.h index 18d4feb90f..ea540c642b 100644 --- a/kvbc/test/sparse_merkle/test_db.h +++ b/kvbc/test/sparse_merkle/test_db.h @@ -36,7 +36,7 @@ class TestDB : public IDBReader { if (latest_version_.find(address) == latest_version_.end()) { return BatchedInternalNode(); } - auto root_key = InternalNodeKey::root(latest_version_.at(address), address); + auto root_key = InternalNodeKey::root(address, latest_version_.at(address)); return internal_nodes_.at(root_key); } diff --git a/kvbc/test/sparse_merkle/tree_test.cpp b/kvbc/test/sparse_merkle/tree_test.cpp index 217c162bfb..61b8cc3b56 100644 --- a/kvbc/test/sparse_merkle/tree_test.cpp +++ b/kvbc/test/sparse_merkle/tree_test.cpp @@ -274,7 +274,7 @@ TEST(tree_tests, insert_single_leaf_node) { // Only a single node was created. It's key is made up of an empty nibble // path, since there are no collisions requring going deeper into the tree. auto& root = batch.internal_nodes.at(0); - ASSERT_EQ(InternalNodeKey(1, NibblePath(), ""), root.first); + ASSERT_EQ(InternalNodeKey("", 1, NibblePath()), root.first); // There should be a root and a single leaf ASSERT_EQ(2, root.second.numChildren()); diff --git a/kvbc/test/sparse_merkle_storage/db_adapter_unit_test.cpp b/kvbc/test/sparse_merkle_storage/db_adapter_unit_test.cpp index 120f04052a..148103cb9f 100644 --- a/kvbc/test/sparse_merkle_storage/db_adapter_unit_test.cpp +++ b/kvbc/test/sparse_merkle_storage/db_adapter_unit_test.cpp @@ -113,7 +113,7 @@ bool hasProvableStaleIndexKeysSince(const std::shared_ptr &db, const bool hasInternalKeysForVersion(const std::shared_ptr &db, const Version &version) { auto iter = db->getIteratorGuard(); - const auto key = iter->seekAtLeast(DBKeyManipulator::genInternalDbKey(InternalNodeKey::root(version, ""))).first; + const auto key = iter->seekAtLeast(DBKeyManipulator::genInternalDbKey(InternalNodeKey::root("", version))).first; if (!key.empty() && DBKeyManipulator::getDBKeyType(key) == EDBKeyType::Key && DBKeyManipulator::getKeySubtype(key) == EKeySubtype::Internal && DBKeyManipulator::extractVersionFromInternalKey(key) == version) { diff --git a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp index 83785b4b9a..ecb84bc05d 100644 --- a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp +++ b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp @@ -182,13 +182,13 @@ TEST(key_manipulator, internal_key_even) { // second byte of the nibble path is 0x34 (by appending 0x03 and 0x04) path.append(0x03); path.append(0x04); - const auto internalKey = InternalNodeKey{defaultVersion, path, ""}; + const auto internalKey = InternalNodeKey{"", defaultVersion, path}; const auto key = DBKeyManipulator::genInternalDbKey(internalKey); // Expect that two bytes of nibbles have been written. const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Internal) + + serializeIntegral(std::uint8_t{0x0}) + serializeIntegral(internalKey.version().value()) + serializeIntegral(std::uint8_t{4}) + - serializeIntegral(std::uint8_t{0x12}) + serializeIntegral(std::uint8_t{0x34}) + - serializeIntegral(std::uint8_t{0x0})); + serializeIntegral(std::uint8_t{0x12}) + serializeIntegral(std::uint8_t{0x34})); ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 2 + 1); ASSERT_TRUE(key == expected); } @@ -202,14 +202,17 @@ TEST(key_manipulator, internal_key_odd) { path.append(0x02); // second byte of the nibble path is 0x30 (by appending 0x03) path.append(0x03); - const auto internalKey = InternalNodeKey{defaultVersion, path, ""}; + std::string address = "test"; + const auto internalKey = InternalNodeKey{address, defaultVersion, path}; const auto key = DBKeyManipulator::genInternalDbKey(internalKey); // Expect that two bytes of nibbles have been written. - const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Internal) + - serializeIntegral(internalKey.version().value()) + serializeIntegral(std::uint8_t{3}) + - serializeIntegral(std::uint8_t{0x12}) + serializeIntegral(std::uint8_t{0x30}) + - serializeIntegral(std::uint8_t{0x0})); - ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 2 + 1); + const auto expected = + toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Internal) + + serializeIntegral(std::uint8_t{0x4}) + serializeIntegral(address[0]) + serializeIntegral(address[1]) + + serializeIntegral(address[2]) + serializeIntegral(address[3]) + + serializeIntegral(internalKey.version().value()) + serializeIntegral(std::uint8_t{3}) + + serializeIntegral(std::uint8_t{0x12}) + serializeIntegral(std::uint8_t{0x30})); + ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 2 + 1 + address.size()); ASSERT_TRUE(key == expected); } @@ -295,7 +298,7 @@ TEST(key_manipulator, stale_db_key_internal) { path.append(0x03); path.append(0x04); std::string address = "abcdefg123"; - const auto internalKey = InternalNodeKey{defaultVersion, path, address}; + const auto internalKey = InternalNodeKey{address, defaultVersion, path}; const auto key = DBKeyManipulator::genStaleDbKey(internalKey, defaultVersion); const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::ProvableStale) + From d3fd201a9cecb6f5441c4739123d9637047d0e6b Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Mon, 27 Mar 2023 19:18:51 +0300 Subject: [PATCH 6/9] Added Address field to LeafKey. Created types (Address and PaddedAddress) to hold addresses in LeafKey and InternalNodeKey. --- kvbc/include/merkle_tree_serialization.h | 64 +++++++++++++------ kvbc/include/sparse_merkle/base_types.h | 51 +++++++++++++++ kvbc/include/sparse_merkle/keys.h | 48 ++++++++------ kvbc/include/sparse_merkle/walker.h | 2 + .../categorization/block_merkle_category.cpp | 4 +- kvbc/src/sparse_merkle/keys.cpp | 2 +- kvbc/src/sparse_merkle/tree.cpp | 6 +- kvbc/test/sparse_merkle/test_db.h | 4 +- 8 files changed, 134 insertions(+), 47 deletions(-) diff --git a/kvbc/include/merkle_tree_serialization.h b/kvbc/include/merkle_tree_serialization.h index b8c25c2af8..d522ba22e5 100644 --- a/kvbc/include/merkle_tree_serialization.h +++ b/kvbc/include/merkle_tree_serialization.h @@ -88,10 +88,22 @@ inline void serializeImp(const sparse_merkle::NibblePath &path, std::string &out out.append(std::cbegin(path.data()), std::cend(path.data())); } -inline void serializeImp(const std::string &address, std::string &out) { +inline void serializeImp(const sparse_merkle::Address &address, std::string &out) { ConcordAssert(address.size() < std::numeric_limits::max()); - out.append(1, static_cast(address.size())); - out.append(std::cbegin(address), std::cend(address)); + static const auto SIZE_PREFIX_BYTES = sparse_merkle::Address::SIZE_PREFIX_BYTES; + using SIZE_PREFIX_TYPE = sparse_merkle::Address::SIZE_PREFIX_TYPE; + out.append(SIZE_PREFIX_BYTES, static_cast(address.size())); + out.append(std::cbegin(address.value()), std::cend(address.value())); +} + +inline void serializeImp(const sparse_merkle::PaddedAddress &address, std::string &out) { + ConcordAssert(address.size() < std::numeric_limits::max()); + static const auto SIZE_PREFIX_BYTES = sparse_merkle::PaddedAddress::SIZE_PREFIX_BYTES; + using SIZE_PREFIX_TYPE = sparse_merkle::PaddedAddress::SIZE_PREFIX_TYPE; + out.append(SIZE_PREFIX_BYTES, static_cast(address.size())); + for (auto i = 0u; i < sparse_merkle::PaddedAddress::SIZE_IN_BYTES; i++) { + out.append(1, static_cast(address.get(i))); + } } inline std::string serializeImp(const sparse_merkle::NibblePath &path) { @@ -125,6 +137,7 @@ inline std::string serializeImp(const sparse_merkle::InternalNodeKey &key) { inline void serializeImp(const sparse_merkle::LeafKey &key, std::string &out) { serializeImp(key.hash(), out); serializeImp(key.version().value(), out); + serializeImp(key.address(), out); } inline std::string serializeImp(const sparse_merkle::LeafKey &key) { @@ -331,39 +344,48 @@ inline sparse_merkle::NibblePath deserialize(const co } template <> -inline std::string deserialize(const concordUtils::Sliver &buf) { - ConcordAssert(buf.length() >= sizeof(std::uint8_t)); +inline sparse_merkle::Address deserialize(const concordUtils::Sliver &buf) { + static const auto SIZE_PREFIX_BYTES = sparse_merkle::Address::SIZE_PREFIX_BYTES; + using SIZE_PREFIX_TYPE = sparse_merkle::Address::SIZE_PREFIX_TYPE; + ConcordAssert(buf.length() >= SIZE_PREFIX_BYTES); // NOLINTNEXTLINE(bugprone-signed-char-misuse) - const auto addrSize = static_cast(buf[0]); + const auto addrSize = static_cast(buf[0]); ConcordAssert(buf.length() > addrSize); - std::string address; - address.reserve(addrSize); - for (auto i = 1u; i <= addrSize; ++i) { - address.push_back(buf[i]); + sparse_merkle::Address address(addrSize); + for (auto i = SIZE_PREFIX_BYTES; i <= addrSize; ++i) { + address.set(i - SIZE_PREFIX_BYTES, buf[i]); } return address; } +template <> +inline sparse_merkle::PaddedAddress deserialize(const concordUtils::Sliver &buf) { + return sparse_merkle::PaddedAddress(deserialize(buf)); +} + template <> inline sparse_merkle::LeafKey deserialize(const concordUtils::Sliver &buf) { ConcordAssert(buf.length() >= sparse_merkle::LeafKey::SIZE_IN_BYTES); - return sparse_merkle::LeafKey{deserialize(buf), - deserialize(concordUtils::Sliver{ - buf, sparse_merkle::Hash::SIZE_IN_BYTES, sparse_merkle::Version::SIZE_IN_BYTES})}; + return sparse_merkle::LeafKey{ + deserialize(buf), + deserialize( + concordUtils::Sliver{buf, sparse_merkle::Hash::SIZE_IN_BYTES, sparse_merkle::Version::SIZE_IN_BYTES}), + deserialize(concordUtils::Sliver{ + buf, + sparse_merkle::Hash::SIZE_IN_BYTES + sparse_merkle::Version::SIZE_IN_BYTES, + buf.length() - sparse_merkle::Hash::SIZE_IN_BYTES - sparse_merkle::Version::SIZE_IN_BYTES})}; } template <> inline sparse_merkle::InternalNodeKey deserialize(const concordUtils::Sliver &buf) { - static const int addressPathSizePrefixInBytes = sizeof(std::uint8_t); // 1 byte for the address size - std::string address = deserialize(buf); - auto version = deserialize( - concordUtils::Sliver{buf, - addressPathSizePrefixInBytes + address.size(), - buf.length() - addressPathSizePrefixInBytes - address.size()}); + static const auto ADDR_SIZE_PREFIX_BYTES = sparse_merkle::Address::SIZE_PREFIX_BYTES; // 1 byte for the address size + auto address = deserialize(buf); + auto version = deserialize(concordUtils::Sliver{ + buf, ADDR_SIZE_PREFIX_BYTES + address.size(), buf.length() - ADDR_SIZE_PREFIX_BYTES - address.size()}); auto nibblePath = deserialize(concordUtils::Sliver{ buf, - addressPathSizePrefixInBytes + address.size() + sparse_merkle::Version::SIZE_IN_BYTES, - buf.length() - addressPathSizePrefixInBytes - address.size() - sparse_merkle::Version::SIZE_IN_BYTES}); + ADDR_SIZE_PREFIX_BYTES + address.size() + sparse_merkle::Version::SIZE_IN_BYTES, + buf.length() - ADDR_SIZE_PREFIX_BYTES - address.size() - sparse_merkle::Version::SIZE_IN_BYTES}); return sparse_merkle::InternalNodeKey{std::move(address), std::move(version), std::move(nibblePath)}; } diff --git a/kvbc/include/sparse_merkle/base_types.h b/kvbc/include/sparse_merkle/base_types.h index 3d0c60d020..94349dd4e3 100644 --- a/kvbc/include/sparse_merkle/base_types.h +++ b/kvbc/include/sparse_merkle/base_types.h @@ -53,6 +53,57 @@ class Version { Type value_ = 0; }; +// A type safe address wrapper +class Address { + public: + using Type = std::string; + using SIZE_PREFIX_TYPE = std::uint8_t; + static constexpr auto SIZE_IN_BYTES = 20 * sizeof(std::uint8_t); + static constexpr auto SIZE_PREFIX_BYTES = sizeof(std::uint8_t); + static constexpr auto TOTAL_SIZE = SIZE_PREFIX_BYTES + SIZE_IN_BYTES; + + public: + Address() = default; + Address(size_t size) { + ConcordAssert(size <= SIZE_IN_BYTES); + addr_.assign('\0', size); + } + Address(const Type& val) : addr_(val) { ConcordAssert(val.size() <= SIZE_IN_BYTES); } + Address(Type&& val) : addr_(std::move(val)) { ConcordAssert(val.size() <= SIZE_IN_BYTES); } + + void set(std::uint8_t pos, char c) { + ConcordAssert(pos < addr_.size()); + addr_[pos] = c; + } + char get(std::uint8_t pos) const { + ConcordAssert(pos < addr_.size()); + return addr_[pos]; + } + bool operator==(const Address& other) const { return addr_ == other.addr_; } + bool operator!=(const Address& other) const { return addr_ != other.addr_; } + bool operator<(const Address& other) const { return addr_ < other.addr_; } + + const Type& value() const { return addr_; } + size_t size() const { return addr_.size(); } + + protected: + Type addr_; +}; + +class PaddedAddress : public Address { + public: + template + PaddedAddress(T&& arg) : Address(std::forward(arg)) {} + char get(std::uint8_t pos) const { + ConcordAssert(pos < SIZE_IN_BYTES); + if (pos < addr_.size()) { + return addr_[pos]; + } else { + return '\0'; + } + } +}; + // A nibble is 4 bits, stored in the lower bits of a byte. class Nibble { public: diff --git a/kvbc/include/sparse_merkle/keys.h b/kvbc/include/sparse_merkle/keys.h index cd364fd3e5..c45a142bde 100644 --- a/kvbc/include/sparse_merkle/keys.h +++ b/kvbc/include/sparse_merkle/keys.h @@ -34,35 +34,37 @@ class InternalNodeKey { return InternalNodeKey(std::forward(address), version, NibblePath()); } - bool operator==(const InternalNodeKey& other) const { return version_ == other.version_ && path_ == other.path_; } + bool operator==(const InternalNodeKey& other) const { + return address_ == other.address_ && version_ == other.version_ && path_ == other.path_; + } - // Compare by version then by path + // Compare by address, version and then by path bool operator<(const InternalNodeKey& other) const { - if (version_ == other.version_ && path_ == other.path_) { + if (address_ != other.address_) { return address_ < other.address_; } - if (version_ == other.version_) { - return path_ < other.path_; + if (version_ != other.version_) { + return version_ < other.version_; } - return version_ < other.version_; + return path_ < other.path_; } std::string toString() const { if (path_.empty()) { - return std::string("") + "-" + version_.toString(); + return address_.value() + std::string("-") + "-" + version_.toString(); } - return path_.toString() + "-" + version_.toString(); + return address_.value() + "-" + path_.toString() + "-" + version_.toString(); } Version version() const { return version_; } - const std::string& address() const { return address_; } + const Address& address() const { return address_; } const NibblePath& path() const { return path_; } NibblePath& path() { return path_; } private: - std::string address_; + Address address_; Version version_; NibblePath path_; }; @@ -74,29 +76,38 @@ class InternalNodeKey { // merkle tree. class LeafKey { public: - static constexpr auto SIZE_IN_BYTES = Hash::SIZE_IN_BYTES + Version::SIZE_IN_BYTES; + static constexpr auto SIZE_IN_BYTES = PaddedAddress::TOTAL_SIZE + Hash::SIZE_IN_BYTES + Version::SIZE_IN_BYTES; public: - LeafKey(Hash key, Version version) : key_(key), version_(version) {} + template + LeafKey(Hash key, Version version, T&& address) : key_(key), version_(version), address_(std::forward(address)) {} + LeafKey(Hash key, Version version) : key_(key), version_(version), address_("") {} - bool operator==(const LeafKey& other) const { return key_ == other.key_ && version_ == other.version_; } + bool operator==(const LeafKey& other) const { + return address_ == other.address_ && key_ == other.key_ && version_ == other.version_; + } bool operator!=(const LeafKey& other) const { return !(*this == other); } - // Compare by key_ then version_ + // Compare by addres, key_ and then version_ bool operator<(const LeafKey& other) const { - if (key_ < other.key_) { - return true; + if (address_ != other.address_) { + return address_ < other.address_; + } + if (key_ != other.key_) { + return key_ < other.key_; } - return key_ == other.key_ && version_ < other.version_; + return version_ < other.version_; } - std::string toString() const { return key_.toString() + "-" + version_.toString(); } + std::string toString() const { return key_.toString() + "-" + version_.toString() + "-" + address_.value(); } Nibble getNibble(const size_t n) const { ConcordAssert(n < Hash::MAX_NIBBLES); return key_.getNibble(n); } + const PaddedAddress& address() const { return address_; } + Version version() const { return version_; } const Hash& hash() const { return key_; } @@ -104,6 +115,7 @@ class LeafKey { private: Hash key_; Version version_; + PaddedAddress address_; }; std::ostream& operator<<(std::ostream& os, const LeafKey&); diff --git a/kvbc/include/sparse_merkle/walker.h b/kvbc/include/sparse_merkle/walker.h index f07efb4eae..60a1b6d9f8 100644 --- a/kvbc/include/sparse_merkle/walker.h +++ b/kvbc/include/sparse_merkle/walker.h @@ -56,6 +56,8 @@ class Walker { void insertEmptyRootAtCurrentVersion(); + const std::string& address() const { return address_; } + private: void ascend(); std::pair pop(); diff --git a/kvbc/src/categorization/block_merkle_category.cpp b/kvbc/src/categorization/block_merkle_category.cpp index 7fd456b54a..c201668b8c 100644 --- a/kvbc/src/categorization/block_merkle_category.cpp +++ b/kvbc/src/categorization/block_merkle_category.cpp @@ -71,12 +71,12 @@ VersionedKey leafKeyToVersionedKey(const sparse_merkle::LeafKey& leaf_key) { // favor of the one immediately below it. BatchedInternalNodeKey toBatchedInternalNodeKey(const sparse_merkle::InternalNodeKey& key) { auto path = NibblePath{static_cast(key.path().length()), key.path().data()}; - return BatchedInternalNodeKey{key.address(), key.version().value(), std::move(path)}; + return BatchedInternalNodeKey{key.address().value(), key.version().value(), std::move(path)}; } BatchedInternalNodeKey toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey&& key) { auto path = NibblePath{static_cast(key.path().length()), key.path().move_data()}; - return BatchedInternalNodeKey{key.address(), key.version().value(), std::move(path)}; + return BatchedInternalNodeKey{key.address().value(), key.version().value(), std::move(path)}; } std::vector rootKey(uint64_t version, const std::string& address = "") { diff --git a/kvbc/src/sparse_merkle/keys.cpp b/kvbc/src/sparse_merkle/keys.cpp index fd9de2e134..52a01f22a4 100644 --- a/kvbc/src/sparse_merkle/keys.cpp +++ b/kvbc/src/sparse_merkle/keys.cpp @@ -15,7 +15,7 @@ namespace concord::kvbc::sparse_merkle { std::ostream& operator<<(std::ostream& os, const LeafKey& key) { - os << key.hash() << "-" << key.version(); + os << key.address().value() << "-" << key.hash() << "-" << key.version(); return os; } diff --git a/kvbc/src/sparse_merkle/tree.cpp b/kvbc/src/sparse_merkle/tree.cpp index cb91652582..f198f3ecd1 100644 --- a/kvbc/src/sparse_merkle/tree.cpp +++ b/kvbc/src/sparse_merkle/tree.cpp @@ -101,7 +101,7 @@ void remove(Walker& walker, const Hash& key_hash) { if (auto rv = std::get_if(&result)) { histograms.remove_depth->record(walker.depth()); - auto stale = LeafKey(key_hash, rv->version); + auto stale = LeafKey(key_hash, rv->version, walker.address()); return walker.ascendToRoot(stale); } @@ -111,7 +111,7 @@ void remove(Walker& walker, const Hash& key_hash) { } if (auto rv = std::get_if(&result)) { - walker.markStale(LeafKey(key_hash, rv->removed_version)); + walker.markStale(LeafKey(key_hash, rv->removed_version, walker.address())); return removeBatchedInternalNode(walker, rv->promoted); } @@ -169,7 +169,7 @@ UpdateBatch Tree::update_impl(const concord::kvbc::SetOfKeyValuePairs& updates, leaf_hash = hasher.hash(val.data(), val.length()); } LeafNode leaf_node{val}; - LeafKey leaf_key{hasher.hash(key.data(), key.length()), version}; + LeafKey leaf_key{hasher.hash(key.data(), key.length()), version, address_}; LeafChild child{leaf_hash, leaf_key}; Walker walker(cache, address_); insert(walker, child); diff --git a/kvbc/test/sparse_merkle/test_db.h b/kvbc/test/sparse_merkle/test_db.h index ea540c642b..99d2385573 100644 --- a/kvbc/test/sparse_merkle/test_db.h +++ b/kvbc/test/sparse_merkle/test_db.h @@ -27,8 +27,8 @@ class TestDB : public IDBReader { ConcordAssert(internal_nodes_.find(key) == internal_nodes_.end()); internal_nodes_[key] = val; - if (latest_version_[key.address()] < key.version()) { - latest_version_[key.address()] = key.version(); + if (latest_version_[key.address().value()] < key.version()) { + latest_version_[key.address().value()] = key.version(); } } From 566705144cc8a6df484d7ada54af421cde0e42e5 Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Mon, 27 Mar 2023 19:23:38 +0300 Subject: [PATCH 7/9] Fix for kvbc/test/sparse_merkle_storage/serialization_unit_test. We need to take in consideration the newly added PaddedAddress field. --- .../serialization_unit_test.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp index ecb84bc05d..ba9c5c9038 100644 --- a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp +++ b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp @@ -155,9 +155,10 @@ TEST(key_manipulator, extract_key_from_non_provable_stale_key) { TEST(key_manipulator, data_key_leaf) { const auto leafKey = LeafKey{defaultHash, defaultVersion}; const auto key = DBKeyManipulator::genDataDbKey(leafKey); + std::string emptyAddress(concord::kvbc::sparse_merkle::PaddedAddress::TOTAL_SIZE, '\0'); const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Leaf) + defaultHashStrBuf + - serializeIntegral(leafKey.version().value())); - ASSERT_EQ(key.length(), 1 + 1 + 32 + 8); + serializeIntegral(leafKey.version().value()) + emptyAddress); + ASSERT_EQ(key.length(), 1 + 1 + 32 + 8 + 21); ASSERT_TRUE(key == expected); } @@ -166,9 +167,10 @@ TEST(key_manipulator, data_key_leaf) { TEST(key_manipulator, data_key_sliver) { const auto version = 42ul; const auto key = DBKeyManipulator::genDataDbKey(defaultSliver, version); + std::string emptyAddress(concord::kvbc::sparse_merkle::PaddedAddress::TOTAL_SIZE, '\0'); const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Leaf) + defaultHashStrBuf + - serializeIntegral(version)); - ASSERT_EQ(key.length(), 1 + 1 + 32 + 8); + serializeIntegral(version) + emptyAddress); + ASSERT_EQ(key.length(), 1 + 1 + 32 + 8 + 21); ASSERT_TRUE(key == expected); } @@ -316,7 +318,7 @@ TEST(key_manipulator, stale_db_key_leaf) { const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::ProvableStale) + serializeIntegral(defaultBlockId) + DBKeyManipulator::genDataDbKey(leafKey).toString()); - ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 1 + 32 + 8); + ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 1 + 32 + 8 + 21); ASSERT_TRUE(key == expected); } From ec0309d4b6f7469933944639dc2e115b666f8840 Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Mon, 27 Mar 2023 23:24:54 +0300 Subject: [PATCH 8/9] Clang tidy report fixes. --- kvbc/include/sparse_merkle/internal_node.h | 4 ++-- kvbc/src/sparse_merkle/internal_node.cpp | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/kvbc/include/sparse_merkle/internal_node.h b/kvbc/include/sparse_merkle/internal_node.h index a587154571..52e9e7ca3a 100644 --- a/kvbc/include/sparse_merkle/internal_node.h +++ b/kvbc/include/sparse_merkle/internal_node.h @@ -387,8 +387,8 @@ class BatchedInternalNode { Version version, Nibble child_key, size_t prefix_bits_in_common, - LeafChild child1, - LeafChild child2); + const LeafChild& child1, + const LeafChild& child2); // Remove the LeafChild at the given index. Updates of children should be set // to new_version. diff --git a/kvbc/src/sparse_merkle/internal_node.cpp b/kvbc/src/sparse_merkle/internal_node.cpp index 193b46ecdb..1dc6d1115c 100644 --- a/kvbc/src/sparse_merkle/internal_node.cpp +++ b/kvbc/src/sparse_merkle/internal_node.cpp @@ -173,8 +173,12 @@ size_t BatchedInternalNode::insertInternalChildren(size_t index, return index; } -BatchedInternalNode::InsertResult BatchedInternalNode::insertTwoLeafChildren( - size_t index, Version version, Nibble child_key, size_t prefix_bits_in_common, LeafChild child1, LeafChild child2) { +BatchedInternalNode::InsertResult BatchedInternalNode::insertTwoLeafChildren(size_t index, + Version version, + Nibble child_key, + size_t prefix_bits_in_common, + const LeafChild& child1, + const LeafChild& child2) { size_t child1_index = 0; size_t child2_index = 0; // prefix_bits_in_common start from MSB. At most there can be 3 bits in common From 1f605ed8391c02990cbd6491632270934f5a5ada Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Tue, 28 Mar 2023 14:21:51 +0300 Subject: [PATCH 9/9] Renamed address to custom_prefix. --- kvbc/cmf/categorized_kvbc_msgs.cmf | 2 +- .../categorization/block_merkle_category.h | 2 +- kvbc/include/merkle_tree_db_adapter.h | 2 +- kvbc/include/merkle_tree_serialization.h | 79 ++++++++++--------- kvbc/include/sparse_merkle/base_types.h | 42 +++++----- kvbc/include/sparse_merkle/db_reader.h | 2 +- kvbc/include/sparse_merkle/keys.h | 42 +++++----- kvbc/include/sparse_merkle/tree.h | 8 +- kvbc/include/sparse_merkle/update_cache.h | 6 +- kvbc/include/sparse_merkle/walker.h | 7 +- .../categorization/block_merkle_category.cpp | 12 +-- kvbc/src/merkle_tree_db_adapter.cpp | 4 +- kvbc/src/sparse_merkle/keys.cpp | 2 +- kvbc/src/sparse_merkle/tree.cpp | 14 ++-- kvbc/src/sparse_merkle/update_cache.cpp | 2 +- kvbc/src/sparse_merkle/walker.cpp | 10 +-- kvbc/test/sparse_merkle/test_db.h | 10 +-- .../serialization_unit_test.cpp | 32 ++++---- 18 files changed, 140 insertions(+), 138 deletions(-) diff --git a/kvbc/cmf/categorized_kvbc_msgs.cmf b/kvbc/cmf/categorized_kvbc_msgs.cmf index b8333e00a1..278ebfbaa9 100644 --- a/kvbc/cmf/categorized_kvbc_msgs.cmf +++ b/kvbc/cmf/categorized_kvbc_msgs.cmf @@ -199,7 +199,7 @@ Msg NibblePath 5001 { Msg BatchedInternalNodeKey 5002 { # The version of this key is the *tree* version, not block version - string address + string custom_prefix uint64 version NibblePath path } diff --git a/kvbc/include/categorization/block_merkle_category.h b/kvbc/include/categorization/block_merkle_category.h index c372a65216..36aaaddc3b 100644 --- a/kvbc/include/categorization/block_merkle_category.h +++ b/kvbc/include/categorization/block_merkle_category.h @@ -202,7 +202,7 @@ class BlockMerkleCategory { Reader(const storage::rocksdb::NativeClient& db) : db_{db} {} // Return the latest root node in the system. - sparse_merkle::BatchedInternalNode get_latest_root(std::string address = "") const override; + sparse_merkle::BatchedInternalNode get_latest_root(std::string custom_prefix = "") const override; // Retrieve a BatchedInternalNode given an InternalNodeKey. // diff --git a/kvbc/include/merkle_tree_db_adapter.h b/kvbc/include/merkle_tree_db_adapter.h index d2075a171e..56472d40fb 100644 --- a/kvbc/include/merkle_tree_db_adapter.h +++ b/kvbc/include/merkle_tree_db_adapter.h @@ -219,7 +219,7 @@ class DBAdapter : public IDbAdapter { Reader(const DBAdapter &adapter) : adapter_{adapter} {} // Return the latest root node in the system. - sparse_merkle::BatchedInternalNode get_latest_root(std::string address = "") const override; + sparse_merkle::BatchedInternalNode get_latest_root(std::string custom_prefix = "") const override; // Retrieve a BatchedInternalNode given an InternalNodeKey. // diff --git a/kvbc/include/merkle_tree_serialization.h b/kvbc/include/merkle_tree_serialization.h index d522ba22e5..565f2615d6 100644 --- a/kvbc/include/merkle_tree_serialization.h +++ b/kvbc/include/merkle_tree_serialization.h @@ -88,21 +88,21 @@ inline void serializeImp(const sparse_merkle::NibblePath &path, std::string &out out.append(std::cbegin(path.data()), std::cend(path.data())); } -inline void serializeImp(const sparse_merkle::Address &address, std::string &out) { - ConcordAssert(address.size() < std::numeric_limits::max()); - static const auto SIZE_PREFIX_BYTES = sparse_merkle::Address::SIZE_PREFIX_BYTES; - using SIZE_PREFIX_TYPE = sparse_merkle::Address::SIZE_PREFIX_TYPE; - out.append(SIZE_PREFIX_BYTES, static_cast(address.size())); - out.append(std::cbegin(address.value()), std::cend(address.value())); -} - -inline void serializeImp(const sparse_merkle::PaddedAddress &address, std::string &out) { - ConcordAssert(address.size() < std::numeric_limits::max()); - static const auto SIZE_PREFIX_BYTES = sparse_merkle::PaddedAddress::SIZE_PREFIX_BYTES; - using SIZE_PREFIX_TYPE = sparse_merkle::PaddedAddress::SIZE_PREFIX_TYPE; - out.append(SIZE_PREFIX_BYTES, static_cast(address.size())); - for (auto i = 0u; i < sparse_merkle::PaddedAddress::SIZE_IN_BYTES; i++) { - out.append(1, static_cast(address.get(i))); +inline void serializeImp(const sparse_merkle::CustomPrefix &custom_prefix, std::string &out) { + ConcordAssert(custom_prefix.size() < std::numeric_limits::max()); + static const auto SIZE_PREFIX_BYTES = sparse_merkle::CustomPrefix::SIZE_PREFIX_BYTES; + using SIZE_PREFIX_TYPE = sparse_merkle::CustomPrefix::SIZE_PREFIX_TYPE; + out.append(SIZE_PREFIX_BYTES, static_cast(custom_prefix.size())); + out.append(std::cbegin(custom_prefix.value()), std::cend(custom_prefix.value())); +} + +inline void serializeImp(const sparse_merkle::PaddedCustomPrefix &custom_prefix, std::string &out) { + ConcordAssert(custom_prefix.size() < std::numeric_limits::max()); + static const auto SIZE_PREFIX_BYTES = sparse_merkle::PaddedCustomPrefix::SIZE_PREFIX_BYTES; + using SIZE_PREFIX_TYPE = sparse_merkle::PaddedCustomPrefix::SIZE_PREFIX_TYPE; + out.append(SIZE_PREFIX_BYTES, static_cast(custom_prefix.size())); + for (auto i = 0u; i < sparse_merkle::PaddedCustomPrefix::SIZE_IN_BYTES; i++) { + out.append(1, static_cast(custom_prefix.get(i))); } } @@ -123,7 +123,7 @@ inline std::string serializeImp(const sparse_merkle::Hash &hash) { } inline void serializeImp(const sparse_merkle::InternalNodeKey &key, std::string &out) { - serializeImp(key.address(), out); + serializeImp(key.customPrefix(), out); serializeImp(key.version().value(), out); serializeImp(key.path(), out); } @@ -137,7 +137,7 @@ inline std::string serializeImp(const sparse_merkle::InternalNodeKey &key) { inline void serializeImp(const sparse_merkle::LeafKey &key, std::string &out) { serializeImp(key.hash(), out); serializeImp(key.version().value(), out); - serializeImp(key.address(), out); + serializeImp(key.customPrefix(), out); } inline std::string serializeImp(const sparse_merkle::LeafKey &key) { @@ -344,23 +344,24 @@ inline sparse_merkle::NibblePath deserialize(const co } template <> -inline sparse_merkle::Address deserialize(const concordUtils::Sliver &buf) { - static const auto SIZE_PREFIX_BYTES = sparse_merkle::Address::SIZE_PREFIX_BYTES; - using SIZE_PREFIX_TYPE = sparse_merkle::Address::SIZE_PREFIX_TYPE; +inline sparse_merkle::CustomPrefix deserialize(const concordUtils::Sliver &buf) { + static const auto SIZE_PREFIX_BYTES = sparse_merkle::CustomPrefix::SIZE_PREFIX_BYTES; + using SIZE_PREFIX_TYPE = sparse_merkle::CustomPrefix::SIZE_PREFIX_TYPE; ConcordAssert(buf.length() >= SIZE_PREFIX_BYTES); // NOLINTNEXTLINE(bugprone-signed-char-misuse) - const auto addrSize = static_cast(buf[0]); - ConcordAssert(buf.length() > addrSize); - sparse_merkle::Address address(addrSize); - for (auto i = SIZE_PREFIX_BYTES; i <= addrSize; ++i) { - address.set(i - SIZE_PREFIX_BYTES, buf[i]); + const auto prefixSize = static_cast(buf[0]); + ConcordAssert(buf.length() > prefixSize); + sparse_merkle::CustomPrefix prefix(prefixSize); + for (auto i = SIZE_PREFIX_BYTES; i <= prefixSize; ++i) { + prefix.set(i - SIZE_PREFIX_BYTES, buf[i]); } - return address; + return prefix; } template <> -inline sparse_merkle::PaddedAddress deserialize(const concordUtils::Sliver &buf) { - return sparse_merkle::PaddedAddress(deserialize(buf)); +inline sparse_merkle::PaddedCustomPrefix deserialize( + const concordUtils::Sliver &buf) { + return sparse_merkle::PaddedCustomPrefix(deserialize(buf)); } template <> @@ -370,7 +371,7 @@ inline sparse_merkle::LeafKey deserialize(const concordU deserialize(buf), deserialize( concordUtils::Sliver{buf, sparse_merkle::Hash::SIZE_IN_BYTES, sparse_merkle::Version::SIZE_IN_BYTES}), - deserialize(concordUtils::Sliver{ + deserialize(concordUtils::Sliver{ buf, sparse_merkle::Hash::SIZE_IN_BYTES + sparse_merkle::Version::SIZE_IN_BYTES, buf.length() - sparse_merkle::Hash::SIZE_IN_BYTES - sparse_merkle::Version::SIZE_IN_BYTES})}; @@ -378,16 +379,16 @@ inline sparse_merkle::LeafKey deserialize(const concordU template <> inline sparse_merkle::InternalNodeKey deserialize(const concordUtils::Sliver &buf) { - static const auto ADDR_SIZE_PREFIX_BYTES = sparse_merkle::Address::SIZE_PREFIX_BYTES; // 1 byte for the address size - auto address = deserialize(buf); - auto version = deserialize(concordUtils::Sliver{ - buf, ADDR_SIZE_PREFIX_BYTES + address.size(), buf.length() - ADDR_SIZE_PREFIX_BYTES - address.size()}); - auto nibblePath = deserialize(concordUtils::Sliver{ - buf, - ADDR_SIZE_PREFIX_BYTES + address.size() + sparse_merkle::Version::SIZE_IN_BYTES, - buf.length() - ADDR_SIZE_PREFIX_BYTES - address.size() - sparse_merkle::Version::SIZE_IN_BYTES}); - - return sparse_merkle::InternalNodeKey{std::move(address), std::move(version), std::move(nibblePath)}; + static const auto SIZE_PREFIX_BYTES = sparse_merkle::CustomPrefix::SIZE_PREFIX_BYTES; // 1 byte for the address size + auto prefix = deserialize(buf); + auto version = deserialize( + concordUtils::Sliver{buf, SIZE_PREFIX_BYTES + prefix.size(), buf.length() - SIZE_PREFIX_BYTES - prefix.size()}); + auto nibblePath = deserialize( + concordUtils::Sliver{buf, + SIZE_PREFIX_BYTES + prefix.size() + sparse_merkle::Version::SIZE_IN_BYTES, + buf.length() - SIZE_PREFIX_BYTES - prefix.size() - sparse_merkle::Version::SIZE_IN_BYTES}); + + return sparse_merkle::InternalNodeKey{std::move(prefix), std::move(version), std::move(nibblePath)}; } template <> diff --git a/kvbc/include/sparse_merkle/base_types.h b/kvbc/include/sparse_merkle/base_types.h index 94349dd4e3..47d564b1fb 100644 --- a/kvbc/include/sparse_merkle/base_types.h +++ b/kvbc/include/sparse_merkle/base_types.h @@ -53,8 +53,8 @@ class Version { Type value_ = 0; }; -// A type safe address wrapper -class Address { +// A type safe custom prefix wrapper +class CustomPrefix { public: using Type = std::string; using SIZE_PREFIX_TYPE = std::uint8_t; @@ -63,41 +63,41 @@ class Address { static constexpr auto TOTAL_SIZE = SIZE_PREFIX_BYTES + SIZE_IN_BYTES; public: - Address() = default; - Address(size_t size) { + CustomPrefix() = default; + CustomPrefix(size_t size) { ConcordAssert(size <= SIZE_IN_BYTES); - addr_.assign('\0', size); + prefix_.assign('\0', size); } - Address(const Type& val) : addr_(val) { ConcordAssert(val.size() <= SIZE_IN_BYTES); } - Address(Type&& val) : addr_(std::move(val)) { ConcordAssert(val.size() <= SIZE_IN_BYTES); } + CustomPrefix(const Type& val) : prefix_(val) { ConcordAssert(val.size() <= SIZE_IN_BYTES); } + CustomPrefix(Type&& val) : prefix_(std::move(val)) { ConcordAssert(val.size() <= SIZE_IN_BYTES); } void set(std::uint8_t pos, char c) { - ConcordAssert(pos < addr_.size()); - addr_[pos] = c; + ConcordAssert(pos < prefix_.size()); + prefix_[pos] = c; } char get(std::uint8_t pos) const { - ConcordAssert(pos < addr_.size()); - return addr_[pos]; + ConcordAssert(pos < prefix_.size()); + return prefix_[pos]; } - bool operator==(const Address& other) const { return addr_ == other.addr_; } - bool operator!=(const Address& other) const { return addr_ != other.addr_; } - bool operator<(const Address& other) const { return addr_ < other.addr_; } + bool operator==(const CustomPrefix& other) const { return prefix_ == other.prefix_; } + bool operator!=(const CustomPrefix& other) const { return prefix_ != other.prefix_; } + bool operator<(const CustomPrefix& other) const { return prefix_ < other.prefix_; } - const Type& value() const { return addr_; } - size_t size() const { return addr_.size(); } + const Type& value() const { return prefix_; } + size_t size() const { return prefix_.size(); } protected: - Type addr_; + Type prefix_; }; -class PaddedAddress : public Address { +class PaddedCustomPrefix : public CustomPrefix { public: template - PaddedAddress(T&& arg) : Address(std::forward(arg)) {} + PaddedCustomPrefix(T&& arg) : CustomPrefix(std::forward(arg)) {} char get(std::uint8_t pos) const { ConcordAssert(pos < SIZE_IN_BYTES); - if (pos < addr_.size()) { - return addr_[pos]; + if (pos < prefix_.size()) { + return prefix_[pos]; } else { return '\0'; } diff --git a/kvbc/include/sparse_merkle/db_reader.h b/kvbc/include/sparse_merkle/db_reader.h index 16b23b282b..9909190385 100644 --- a/kvbc/include/sparse_merkle/db_reader.h +++ b/kvbc/include/sparse_merkle/db_reader.h @@ -30,7 +30,7 @@ class IDBReader { virtual ~IDBReader() = default; // Return the latest root node in the system - virtual BatchedInternalNode get_latest_root(std::string address = "") const = 0; + virtual BatchedInternalNode get_latest_root(std::string custom_prefix = "") const = 0; // Retrieve a BatchedInternalNode given an InternalNodeKey. // diff --git a/kvbc/include/sparse_merkle/keys.h b/kvbc/include/sparse_merkle/keys.h index c45a142bde..4161ccc5f4 100644 --- a/kvbc/include/sparse_merkle/keys.h +++ b/kvbc/include/sparse_merkle/keys.h @@ -25,23 +25,23 @@ namespace sparse_merkle { class InternalNodeKey { public: template - InternalNodeKey(T&& address, Version version, const NibblePath& path) - : address_(std::forward(address)), version_(version), path_(path) {} + InternalNodeKey(T&& custom_prefix, Version version, const NibblePath& path) + : custom_prefix_(std::forward(custom_prefix)), version_(version), path_(path) {} // Return the root of a sparse merkle tree at a given version. template - static InternalNodeKey root(T&& address, Version version) { - return InternalNodeKey(std::forward(address), version, NibblePath()); + static InternalNodeKey root(T&& custom_prefix, Version version) { + return InternalNodeKey(std::forward(custom_prefix), version, NibblePath()); } bool operator==(const InternalNodeKey& other) const { - return address_ == other.address_ && version_ == other.version_ && path_ == other.path_; + return custom_prefix_ == other.custom_prefix_ && version_ == other.version_ && path_ == other.path_; } // Compare by address, version and then by path bool operator<(const InternalNodeKey& other) const { - if (address_ != other.address_) { - return address_ < other.address_; + if (custom_prefix_ != other.custom_prefix_) { + return custom_prefix_ < other.custom_prefix_; } if (version_ != other.version_) { return version_ < other.version_; @@ -51,20 +51,20 @@ class InternalNodeKey { std::string toString() const { if (path_.empty()) { - return address_.value() + std::string("-") + "-" + version_.toString(); + return custom_prefix_.value() + std::string("-") + "-" + version_.toString(); } - return address_.value() + "-" + path_.toString() + "-" + version_.toString(); + return custom_prefix_.value() + "-" + path_.toString() + "-" + version_.toString(); } Version version() const { return version_; } - const Address& address() const { return address_; } + const CustomPrefix& customPrefix() const { return custom_prefix_; } const NibblePath& path() const { return path_; } NibblePath& path() { return path_; } private: - Address address_; + CustomPrefix custom_prefix_; Version version_; NibblePath path_; }; @@ -76,22 +76,22 @@ class InternalNodeKey { // merkle tree. class LeafKey { public: - static constexpr auto SIZE_IN_BYTES = PaddedAddress::TOTAL_SIZE + Hash::SIZE_IN_BYTES + Version::SIZE_IN_BYTES; + static constexpr auto SIZE_IN_BYTES = PaddedCustomPrefix::TOTAL_SIZE + Hash::SIZE_IN_BYTES + Version::SIZE_IN_BYTES; public: template - LeafKey(Hash key, Version version, T&& address) : key_(key), version_(version), address_(std::forward(address)) {} - LeafKey(Hash key, Version version) : key_(key), version_(version), address_("") {} + LeafKey(Hash key, Version version, T&& address) : key_(key), version_(version), prefix_(std::forward(address)) {} + LeafKey(Hash key, Version version) : key_(key), version_(version), prefix_("") {} bool operator==(const LeafKey& other) const { - return address_ == other.address_ && key_ == other.key_ && version_ == other.version_; + return prefix_ == other.prefix_ && key_ == other.key_ && version_ == other.version_; } bool operator!=(const LeafKey& other) const { return !(*this == other); } - // Compare by addres, key_ and then version_ + // Compare by prefix_, key_ and then version_ bool operator<(const LeafKey& other) const { - if (address_ != other.address_) { - return address_ < other.address_; + if (prefix_ != other.prefix_) { + return prefix_ < other.prefix_; } if (key_ != other.key_) { return key_ < other.key_; @@ -99,14 +99,14 @@ class LeafKey { return version_ < other.version_; } - std::string toString() const { return key_.toString() + "-" + version_.toString() + "-" + address_.value(); } + std::string toString() const { return key_.toString() + "-" + version_.toString() + "-" + prefix_.value(); } Nibble getNibble(const size_t n) const { ConcordAssert(n < Hash::MAX_NIBBLES); return key_.getNibble(n); } - const PaddedAddress& address() const { return address_; } + const PaddedCustomPrefix& customPrefix() const { return prefix_; } Version version() const { return version_; } @@ -115,7 +115,7 @@ class LeafKey { private: Hash key_; Version version_; - PaddedAddress address_; + PaddedCustomPrefix prefix_; }; std::ostream& operator<<(std::ostream& os, const LeafKey&); diff --git a/kvbc/include/sparse_merkle/tree.h b/kvbc/include/sparse_merkle/tree.h index 842fddde69..0497a9af9a 100644 --- a/kvbc/include/sparse_merkle/tree.h +++ b/kvbc/include/sparse_merkle/tree.h @@ -43,8 +43,8 @@ namespace sparse_merkle { class Tree { public: Tree() = default; - explicit Tree(std::shared_ptr db_reader, std::string address = "") - : db_reader_(db_reader), address_(address) { + explicit Tree(std::shared_ptr db_reader, std::string custom_prefix = "") + : db_reader_(db_reader), custom_prefix_(custom_prefix) { reset(); } @@ -78,7 +78,7 @@ class Tree { // // This is necessary to do before updates, as we only allow updating the // latest tree. - void reset() { root_ = db_reader_->get_latest_root(address_); } + void reset() { root_ = db_reader_->get_latest_root(custom_prefix_); } UpdateBatch update_impl(const concord::kvbc::SetOfKeyValuePairs& updates, const concord::kvbc::KeysVector& deleted_keys, @@ -86,7 +86,7 @@ class Tree { std::shared_ptr db_reader_; BatchedInternalNode root_; - std::string address_; + std::string custom_prefix_; }; } // namespace sparse_merkle diff --git a/kvbc/include/sparse_merkle/update_cache.h b/kvbc/include/sparse_merkle/update_cache.h index 9386e053d4..668bcdc0a4 100644 --- a/kvbc/include/sparse_merkle/update_cache.h +++ b/kvbc/include/sparse_merkle/update_cache.h @@ -34,8 +34,8 @@ namespace detail { // A new instance of this structure is created during every update call. class UpdateCache { public: - UpdateCache(const BatchedInternalNode& root, const std::shared_ptr& db_reader, std::string address) - : version_(root.version() + 1), db_reader_(db_reader), original_root_(root), address_(address) {} + UpdateCache(const BatchedInternalNode& root, const std::shared_ptr& db_reader, std::string custom_prefix) + : version_(root.version() + 1), db_reader_(db_reader), original_root_(root), custom_prefix_(custom_prefix) {} const StaleNodeIndexes& stale() const { return stale_; } const auto& internalNodes() const { return internal_nodes_; } @@ -74,7 +74,7 @@ class UpdateCache { // These nodes are mutable and are all being updated to a single version. // Therefore we key them by their NibblePath alone, without a version. std::map internal_nodes_; - std::string address_; + std::string custom_prefix_; }; } // namespace detail diff --git a/kvbc/include/sparse_merkle/walker.h b/kvbc/include/sparse_merkle/walker.h index 60a1b6d9f8..02431a1091 100644 --- a/kvbc/include/sparse_merkle/walker.h +++ b/kvbc/include/sparse_merkle/walker.h @@ -28,7 +28,8 @@ typedef std::stack> NodeSt // A class for ascending and descending the tree. This is used for updates. class Walker { public: - Walker(UpdateCache& cache, std::string address) : cache_(cache), current_node_(cache.getRoot()), address_(address) { + Walker(UpdateCache& cache, std::string custom_prefix) + : cache_(cache), current_node_(cache.getRoot()), custom_prefix_(custom_prefix) { // Save the version of the root node, in case we only update the root node. stale_version_ = current_node_.version(); } @@ -56,7 +57,7 @@ class Walker { void insertEmptyRootAtCurrentVersion(); - const std::string& address() const { return address_; } + const std::string& customPrefix() const { return custom_prefix_; } private: void ascend(); @@ -76,7 +77,7 @@ class Walker { NodeStack stack_; BatchedInternalNode current_node_; NibblePath nibble_path_; - std::string address_; + std::string custom_prefix_; }; } // namespace detail diff --git a/kvbc/src/categorization/block_merkle_category.cpp b/kvbc/src/categorization/block_merkle_category.cpp index c201668b8c..4d19fa4163 100644 --- a/kvbc/src/categorization/block_merkle_category.cpp +++ b/kvbc/src/categorization/block_merkle_category.cpp @@ -71,17 +71,17 @@ VersionedKey leafKeyToVersionedKey(const sparse_merkle::LeafKey& leaf_key) { // favor of the one immediately below it. BatchedInternalNodeKey toBatchedInternalNodeKey(const sparse_merkle::InternalNodeKey& key) { auto path = NibblePath{static_cast(key.path().length()), key.path().data()}; - return BatchedInternalNodeKey{key.address().value(), key.version().value(), std::move(path)}; + return BatchedInternalNodeKey{key.customPrefix().value(), key.version().value(), std::move(path)}; } BatchedInternalNodeKey toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey&& key) { auto path = NibblePath{static_cast(key.path().length()), key.path().move_data()}; - return BatchedInternalNodeKey{key.address().value(), key.version().value(), std::move(path)}; + return BatchedInternalNodeKey{key.customPrefix().value(), key.version().value(), std::move(path)}; } -std::vector rootKey(uint64_t version, const std::string& address = "") { +std::vector rootKey(uint64_t version, const std::string& custom_prefix = "") { auto v = sparse_merkle::Version(version); - return serialize(toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey::root(address, v))); + return serialize(toBatchedInternalNodeKey(sparse_merkle::InternalNodeKey::root(custom_prefix, v))); } // A key used in tree_.update() @@ -775,8 +775,8 @@ uint64_t BlockMerkleCategory::getLatestTreeVersion() const { return 0; } -sparse_merkle::BatchedInternalNode BlockMerkleCategory::Reader::get_latest_root(std::string address) const { - if (auto latest_root_key = db_.get(BLOCK_MERKLE_INTERNAL_NODES_CF, rootKey(0, address))) { +sparse_merkle::BatchedInternalNode BlockMerkleCategory::Reader::get_latest_root(std::string custom_prefix) const { + if (auto latest_root_key = db_.get(BLOCK_MERKLE_INTERNAL_NODES_CF, rootKey(0, custom_prefix))) { if (auto serialized = db_.get(BLOCK_MERKLE_INTERNAL_NODES_CF, *latest_root_key)) { return deserializeBatchedInternalNode(*serialized); } diff --git a/kvbc/src/merkle_tree_db_adapter.cpp b/kvbc/src/merkle_tree_db_adapter.cpp index b42c530d7c..c5fc5eeb07 100644 --- a/kvbc/src/merkle_tree_db_adapter.cpp +++ b/kvbc/src/merkle_tree_db_adapter.cpp @@ -446,7 +446,7 @@ std::pair DBAdap return smTree_.update_with_cache(updates, KeysVector{std::cbegin(deletes), std::cend(deletes)}); } -BatchedInternalNode DBAdapter::Reader::get_latest_root(std::string address) const { +BatchedInternalNode DBAdapter::Reader::get_latest_root(std::string custom_prefix) const { const auto lastBlock = adapter_.getLastReachableBlockId(); if (lastBlock == 0) { return BatchedInternalNode{}; @@ -462,7 +462,7 @@ BatchedInternalNode DBAdapter::Reader::get_latest_root(std::string address) cons return BatchedInternalNode{}; } - return get_internal(InternalNodeKey::root(address, stateRootVersion)); + return get_internal(InternalNodeKey::root(custom_prefix, stateRootVersion)); } BatchedInternalNode DBAdapter::Reader::get_internal(const InternalNodeKey &key) const { diff --git a/kvbc/src/sparse_merkle/keys.cpp b/kvbc/src/sparse_merkle/keys.cpp index 52a01f22a4..221593b042 100644 --- a/kvbc/src/sparse_merkle/keys.cpp +++ b/kvbc/src/sparse_merkle/keys.cpp @@ -15,7 +15,7 @@ namespace concord::kvbc::sparse_merkle { std::ostream& operator<<(std::ostream& os, const LeafKey& key) { - os << key.address().value() << "-" << key.hash() << "-" << key.version(); + os << key.hash() << "-" << key.version() << "-" << key.customPrefix().value(); return os; } diff --git a/kvbc/src/sparse_merkle/tree.cpp b/kvbc/src/sparse_merkle/tree.cpp index f198f3ecd1..22c1853f5f 100644 --- a/kvbc/src/sparse_merkle/tree.cpp +++ b/kvbc/src/sparse_merkle/tree.cpp @@ -101,7 +101,7 @@ void remove(Walker& walker, const Hash& key_hash) { if (auto rv = std::get_if(&result)) { histograms.remove_depth->record(walker.depth()); - auto stale = LeafKey(key_hash, rv->version, walker.address()); + auto stale = LeafKey(key_hash, rv->version, walker.customPrefix()); return walker.ascendToRoot(stale); } @@ -111,7 +111,7 @@ void remove(Walker& walker, const Hash& key_hash) { } if (auto rv = std::get_if(&result)) { - walker.markStale(LeafKey(key_hash, rv->removed_version, walker.address())); + walker.markStale(LeafKey(key_hash, rv->removed_version, walker.customPrefix())); return removeBatchedInternalNode(walker, rv->promoted); } @@ -133,14 +133,14 @@ UpdateBatch Tree::update(const concord::kvbc::SetOfKeyValuePairs& updates, histograms.num_deleted_keys->record(deleted_keys.size()); TimeRecorder scoped_timer(*histograms.update); reset(); - UpdateCache cache(root_, db_reader_, address_); + UpdateCache cache(root_, db_reader_, custom_prefix_); return update_impl(updates, deleted_keys, cache); } std::pair Tree::update_with_cache(const concord::kvbc::SetOfKeyValuePairs& updates, const concord::kvbc::KeysVector& deleted_keys) { reset(); - UpdateCache cache(root_, db_reader_, address_); + UpdateCache cache(root_, db_reader_, custom_prefix_); auto batch = update_impl(updates, deleted_keys, cache); return std::make_pair(batch, cache); } @@ -155,7 +155,7 @@ UpdateBatch Tree::update_impl(const concord::kvbc::SetOfKeyValuePairs& updates, // Deletes come before inserts because it makes more semantic sense. A user can delete a key and then write a new // version, but it makes no sense to add a new version and then delete a key. for (auto& key : deleted_keys) { - Walker walker(cache, address_); + Walker walker(cache, custom_prefix_); auto key_hash = hasher.hash(key.data(), key.length()); sparse_merkle::remove(walker, key_hash); } @@ -169,9 +169,9 @@ UpdateBatch Tree::update_impl(const concord::kvbc::SetOfKeyValuePairs& updates, leaf_hash = hasher.hash(val.data(), val.length()); } LeafNode leaf_node{val}; - LeafKey leaf_key{hasher.hash(key.data(), key.length()), version, address_}; + LeafKey leaf_key{hasher.hash(key.data(), key.length()), version, custom_prefix_}; LeafChild child{leaf_hash, leaf_key}; - Walker walker(cache, address_); + Walker walker(cache, custom_prefix_); insert(walker, child); batch.leaf_nodes.emplace_back(leaf_key, leaf_node); } diff --git a/kvbc/src/sparse_merkle/update_cache.cpp b/kvbc/src/sparse_merkle/update_cache.cpp index a66c10fc19..be0996dfcb 100644 --- a/kvbc/src/sparse_merkle/update_cache.cpp +++ b/kvbc/src/sparse_merkle/update_cache.cpp @@ -29,7 +29,7 @@ void UpdateCache::putStale(const std::optional& key) { } const BatchedInternalNode& UpdateCache::getRoot() { - auto it = internal_nodes_.find({address_, version_, NibblePath()}); + auto it = internal_nodes_.find({custom_prefix_, version_, NibblePath()}); if (it != internal_nodes_.end()) { return it->second; } diff --git a/kvbc/src/sparse_merkle/walker.cpp b/kvbc/src/sparse_merkle/walker.cpp index b8f601f73b..40b322c2a1 100644 --- a/kvbc/src/sparse_merkle/walker.cpp +++ b/kvbc/src/sparse_merkle/walker.cpp @@ -36,7 +36,7 @@ void Walker::descend(const Hash& key, Version next_version) { stack_.push(current_node_); Nibble next_nibble = key.getNibble(depth()); nibble_path_.append(next_nibble); - InternalNodeKey next_internal_key{address_, next_version, nibble_path_}; + InternalNodeKey next_internal_key{custom_prefix_, next_version, nibble_path_}; current_node_ = cache_.getInternalNode(next_internal_key); stale_version_ = current_node_.version(); } @@ -68,7 +68,7 @@ void Walker::ascend() { std::optional Walker::removeCurrentNode() { markCurrentNodeStale(); - cache_.remove({address_, stale_version_, nibble_path_}); + cache_.remove({custom_prefix_, stale_version_, nibble_path_}); if (!atRoot()) { return pop().first; } @@ -78,7 +78,7 @@ std::optional Walker::removeCurrentNode() { void Walker::insertEmptyRootAtCurrentVersion() { auto children = BatchedInternalNode::Children{}; children[0] = InternalChild{PLACEHOLDER_HASH, cache_.version()}; - cache_.put({address_, cache_.version(), NibblePath{}}, BatchedInternalNode{children}); + cache_.put({custom_prefix_, cache_.version(), NibblePath{}}, BatchedInternalNode{children}); } std::pair Walker::pop() { @@ -97,10 +97,10 @@ void Walker::markCurrentNodeStale() { // Don't mark the initial root stale, and don't mark an already cached node // stale. if (stale_version_ != Version(0) && stale_version_ != version()) { - cache_.putStale(InternalNodeKey(address_, stale_version_, nibble_path_)); + cache_.putStale(InternalNodeKey(custom_prefix_, stale_version_, nibble_path_)); } } -void Walker::cacheCurrentNode() { cache_.put({address_, cache_.version(), nibble_path_}, current_node_); } +void Walker::cacheCurrentNode() { cache_.put({custom_prefix_, cache_.version(), nibble_path_}, current_node_); } } // namespace concord::kvbc::sparse_merkle::detail diff --git a/kvbc/test/sparse_merkle/test_db.h b/kvbc/test/sparse_merkle/test_db.h index 99d2385573..69ed0946e5 100644 --- a/kvbc/test/sparse_merkle/test_db.h +++ b/kvbc/test/sparse_merkle/test_db.h @@ -27,16 +27,16 @@ class TestDB : public IDBReader { ConcordAssert(internal_nodes_.find(key) == internal_nodes_.end()); internal_nodes_[key] = val; - if (latest_version_[key.address().value()] < key.version()) { - latest_version_[key.address().value()] = key.version(); + if (latest_version_[key.customPrefix().value()] < key.version()) { + latest_version_[key.customPrefix().value()] = key.version(); } } - BatchedInternalNode get_latest_root(std::string address = "") const override { - if (latest_version_.find(address) == latest_version_.end()) { + BatchedInternalNode get_latest_root(std::string custom_prefix = "") const override { + if (latest_version_.find(custom_prefix) == latest_version_.end()) { return BatchedInternalNode(); } - auto root_key = InternalNodeKey::root(address, latest_version_.at(address)); + auto root_key = InternalNodeKey::root(custom_prefix, latest_version_.at(custom_prefix)); return internal_nodes_.at(root_key); } diff --git a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp index ba9c5c9038..711959b3ff 100644 --- a/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp +++ b/kvbc/test/sparse_merkle_storage/serialization_unit_test.cpp @@ -155,9 +155,9 @@ TEST(key_manipulator, extract_key_from_non_provable_stale_key) { TEST(key_manipulator, data_key_leaf) { const auto leafKey = LeafKey{defaultHash, defaultVersion}; const auto key = DBKeyManipulator::genDataDbKey(leafKey); - std::string emptyAddress(concord::kvbc::sparse_merkle::PaddedAddress::TOTAL_SIZE, '\0'); + std::string emptyCustomPrefix(concord::kvbc::sparse_merkle::PaddedCustomPrefix::TOTAL_SIZE, '\0'); const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Leaf) + defaultHashStrBuf + - serializeIntegral(leafKey.version().value()) + emptyAddress); + serializeIntegral(leafKey.version().value()) + emptyCustomPrefix); ASSERT_EQ(key.length(), 1 + 1 + 32 + 8 + 21); ASSERT_TRUE(key == expected); } @@ -167,9 +167,9 @@ TEST(key_manipulator, data_key_leaf) { TEST(key_manipulator, data_key_sliver) { const auto version = 42ul; const auto key = DBKeyManipulator::genDataDbKey(defaultSliver, version); - std::string emptyAddress(concord::kvbc::sparse_merkle::PaddedAddress::TOTAL_SIZE, '\0'); + std::string emptyCustomPrefix(concord::kvbc::sparse_merkle::PaddedCustomPrefix::TOTAL_SIZE, '\0'); const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Leaf) + defaultHashStrBuf + - serializeIntegral(version) + emptyAddress); + serializeIntegral(version) + emptyCustomPrefix); ASSERT_EQ(key.length(), 1 + 1 + 32 + 8 + 21); ASSERT_TRUE(key == expected); } @@ -204,17 +204,17 @@ TEST(key_manipulator, internal_key_odd) { path.append(0x02); // second byte of the nibble path is 0x30 (by appending 0x03) path.append(0x03); - std::string address = "test"; - const auto internalKey = InternalNodeKey{address, defaultVersion, path}; + std::string custom_prefix = "test"; + const auto internalKey = InternalNodeKey{custom_prefix, defaultVersion, path}; const auto key = DBKeyManipulator::genInternalDbKey(internalKey); // Expect that two bytes of nibbles have been written. - const auto expected = - toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Internal) + - serializeIntegral(std::uint8_t{0x4}) + serializeIntegral(address[0]) + serializeIntegral(address[1]) + - serializeIntegral(address[2]) + serializeIntegral(address[3]) + - serializeIntegral(internalKey.version().value()) + serializeIntegral(std::uint8_t{3}) + - serializeIntegral(std::uint8_t{0x12}) + serializeIntegral(std::uint8_t{0x30})); - ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 2 + 1 + address.size()); + const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::Internal) + + serializeIntegral(std::uint8_t{0x4}) + serializeIntegral(custom_prefix[0]) + + serializeIntegral(custom_prefix[1]) + serializeIntegral(custom_prefix[2]) + + serializeIntegral(custom_prefix[3]) + + serializeIntegral(internalKey.version().value()) + serializeIntegral(std::uint8_t{3}) + + serializeIntegral(std::uint8_t{0x12}) + serializeIntegral(std::uint8_t{0x30})); + ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 2 + 1 + custom_prefix.size()); ASSERT_TRUE(key == expected); } @@ -299,13 +299,13 @@ TEST(key_manipulator, stale_db_key_internal) { // second byte of the nibble path is 0x34 (by appending 0x03 and 0x04) path.append(0x03); path.append(0x04); - std::string address = "abcdefg123"; - const auto internalKey = InternalNodeKey{address, defaultVersion, path}; + std::string custom_prefix = "abcdefg123"; + const auto internalKey = InternalNodeKey{custom_prefix, defaultVersion, path}; const auto key = DBKeyManipulator::genStaleDbKey(internalKey, defaultVersion); const auto expected = toSliver(serializeEnum(EDBKeyType::Key) + serializeEnum(EKeySubtype::ProvableStale) + serializeIntegral(defaultBlockId) + DBKeyManipulator::genInternalDbKey(internalKey).toString()); - ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 1 + 8 + 1 + 2 + 1 + address.size()); + ASSERT_EQ(key.length(), 1 + 1 + 8 + 1 + 1 + 8 + 1 + 2 + 1 + custom_prefix.size()); ASSERT_TRUE(key == expected); }