From af9b74fcc7d0eddc57b6d92aec7c14aa67e4d1b0 Mon Sep 17 00:00:00 2001 From: j-berman Date: Thu, 23 May 2024 23:56:23 -0700 Subject: [PATCH] start LMDB grow_tree --- src/blockchain_db/CMakeLists.txt | 1 + src/blockchain_db/blockchain_db.h | 4 + src/blockchain_db/lmdb/db_lmdb.cpp | 148 +++++++++++++++++++++++++++-- src/blockchain_db/lmdb/db_lmdb.h | 18 +++- src/blockchain_db/testdb.h | 2 + src/fcmp/curve_trees.cpp | 2 +- src/fcmp/curve_trees.h | 2 +- tests/unit_tests/blockchain_db.cpp | 34 +++++++ tests/unit_tests/curve_trees.cpp | 17 +--- tests/unit_tests/curve_trees.h | 18 ++++ 10 files changed, 217 insertions(+), 29 deletions(-) diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index e94705b22..9c55cebaa 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -45,6 +45,7 @@ target_link_libraries(blockchain_db PUBLIC common cncrypto + fcmp ringct ${LMDB_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 3e953da30..b5224127e 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -40,6 +40,7 @@ #include "cryptonote_basic/difficulty.h" #include "cryptonote_basic/hardfork.h" #include "cryptonote_protocol/enums.h" +#include "fcmp/curve_trees.h" /** \file * Cryptonote Blockchain Database Interface @@ -1764,6 +1765,9 @@ public: */ virtual bool for_all_alt_blocks(std::function f, bool include_blob = false) const = 0; + // TODO: description and make private + virtual void grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, + const std::vector &new_leaves) = 0; // // Hard fork related storage diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index af456b00d..35d041473 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -200,7 +200,7 @@ namespace * spent_keys input hash - * * leaves leaf_idx {O.x, I.x, C.x} - * branches layer_idx [{branch_idx, branch_hash}...] + * layers layer_idx [{child_chunk_idx, child_chunk_hash}...] * * txpool_meta txn hash txn metadata * txpool_blob txn hash txn blob @@ -213,7 +213,7 @@ namespace * attached as a prefix on the Data to serve as the DUPSORT key. * (DUPFIXED saves 8 bytes per record.) * - * The output_amounts and branches tables don't use a dummy key, but use DUPSORT + * The output_amounts and layers tables don't use a dummy key, but use DUPSORT */ const char* const LMDB_BLOCKS = "blocks"; const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; @@ -233,7 +233,7 @@ const char* const LMDB_SPENT_KEYS = "spent_keys"; // Curve trees tree types const char* const LMDB_LEAVES = "leaves"; -const char* const LMDB_BRANCHES = "branches"; +const char* const LMDB_LAYERS = "layers"; const char* const LMDB_TXPOOL_META = "txpool_meta"; const char* const LMDB_TXPOOL_BLOB = "txpool_blob"; @@ -358,6 +358,11 @@ typedef struct outtx { uint64_t local_index; } outtx; +typedef struct layer_val { + uint64_t child_chunk_idx; + std::array child_chunk_hash; +} layer_val; + std::atomic mdb_txn_safe::num_active_txns{0}; std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; @@ -1295,6 +1300,135 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) } } +template +void BlockchainLMDB::grow_layer(const std::vector> &layer_extensions, + const std::size_t ext_idx, + const std::size_t layer_idx, + const fcmp::curve_trees::LastChunkData *last_chunk_ptr) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + + CURSOR(layers) + + CHECK_AND_ASSERT_THROW_MES(ext_idx < layer_extensions.size(), "unexpected layer extension"); + const auto &ext = layer_extensions[ext_idx]; + + CHECK_AND_ASSERT_THROW_MES(!ext.hashes.empty(), "empty layer extension"); + + // TODO: make sure last_chunk_ptr->parent_layer_size is correct + + // Check if we started at 0 if fresh layer, or if started 1 after the last element in the layer + const bool started_after_tip = (ext.start_idx == 0 && last_chunk_ptr == nullptr) + || (last_chunk_ptr != nullptr && ext.start_idx == last_chunk_ptr->parent_layer_size); + + // Check if we updated the last element in the layer + const bool started_at_tip = (last_chunk_ptr != nullptr + && (ext.start_idx + 1) == last_chunk_ptr->parent_layer_size); + + CHECK_AND_ASSERT_THROW_MES(started_after_tip || started_at_tip, "unexpected layer start"); + + MDB_val_copy k(layer_idx); + + if (started_at_tip) + { + // We updated the last hash, so update it + layer_val lv; + lv.child_chunk_idx = ext.start_idx; + lv.child_chunk_hash = std::array(); // ext.hashes.front(); // TODO + MDB_val_set(v, lv); + + // We expect to overwrite the existing hash + // TODO: make sure the hash already exists + int result = mdb_cursor_put(m_cur_layers, &k, &v, 0); + if (result != MDB_SUCCESS) + throw0(DB_ERROR(lmdb_error("Failed to update chunk hash: ", result).c_str())); + } + + // Now add all the new hashes found in the extension + for (std::size_t i = started_at_tip ? 1 : 0; i < ext.hashes.size(); ++i) + { + layer_val lv; + lv.child_chunk_idx = i + ext.start_idx; + lv.child_chunk_hash = std::array(); // ext.hashes[i]; // TODO + MDB_val_set(v, lv); + + // TODO: according to the docs, MDB_APPENDDUP isn't supposed to perform any key comparisons to maximize efficiency. + // Adding MDB_NODUPDATA I assume re-introduces a key comparison. Benchmark MDB_NODUPDATA here + // MDB_NODUPDATA makes sure key/data pair doesn't already exist + int result = mdb_cursor_put(m_cur_layers, &k, &v, MDB_APPENDDUP | MDB_NODUPDATA); + if (result != MDB_SUCCESS) + throw0(DB_ERROR(lmdb_error("Failed to add hash: ", result).c_str())); + } +} + +void BlockchainLMDB::grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, + const std::vector &new_leaves) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + + CURSOR(leaves) + + // TODO: read every layer's last chunks + const auto last_chunks = fcmp::curve_trees::CurveTreesV1::LastChunks{}; + + const auto tree_extension = curve_trees.get_tree_extension(last_chunks, new_leaves); + + // Insert the leaves + const auto &leaves = tree_extension.leaves; + for (std::size_t i = 0; i < leaves.tuples.size(); ++i) + { + MDB_val_copy k(i + leaves.start_idx); + MDB_val_set(v, leaves.tuples[i]); + + // TODO: according to the docs, MDB_APPENDDUP isn't supposed to perform any key comparisons to maximize efficiency. + // Adding MDB_NOOVERWRITE I assume re-introduces a key comparison. Benchmark NOOVERWRITE here + // MDB_NOOVERWRITE makes sure key doesn't already exist + int result = mdb_cursor_put(m_cur_leaves, &k, &v, MDB_APPENDDUP | MDB_NOOVERWRITE); + if (result != MDB_SUCCESS) + throw0(DB_ERROR(lmdb_error("Failed to add leaf: ", result).c_str())); + } + + // Grow the layers + const auto &c2_extensions = tree_extension.c2_layer_extensions; + const auto &c1_extensions = tree_extension.c1_layer_extensions; + CHECK_AND_ASSERT_THROW_MES(!c2_extensions.empty(), "empty c2 extensions"); + + bool use_c2 = true; + std::size_t c2_idx = 0; + std::size_t c1_idx = 0; + for (std::size_t i = 0; i < (c2_extensions.size() + c1_extensions.size()); ++i) + { + const std::size_t layer_idx = c2_idx + c1_idx; + + if (use_c2) + { + const auto *c2_last_chunk_ptr = (c2_idx >= last_chunks.c2_last_chunks.size()) + ? nullptr + : &last_chunks.c2_last_chunks[c2_idx]; + + this->grow_layer(c2_extensions, c2_idx, layer_idx, c2_last_chunk_ptr); + + ++c2_idx; + } + else + { + const auto *c1_last_chunk_ptr = (c1_idx >= last_chunks.c1_last_chunks.size()) + ? nullptr + : &last_chunks.c1_last_chunks[c1_idx]; + + this->grow_layer(c1_extensions, c1_idx, layer_idx, c1_last_chunk_ptr); + + ++c1_idx; + } + + use_c2 = !use_c2; + } +} + BlockchainLMDB::~BlockchainLMDB() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1445,7 +1579,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); lmdb_db_open(txn, LMDB_LEAVES, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_leaves, "Failed to open db handle for m_leaves"); - lmdb_db_open(txn, LMDB_BRANCHES, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_branches, "Failed to open db handle for m_branches"); + lmdb_db_open(txn, LMDB_LAYERS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_layers, "Failed to open db handle for m_layers"); lmdb_db_open(txn, LMDB_TXPOOL_META, MDB_CREATE, m_txpool_meta, "Failed to open db handle for m_txpool_meta"); lmdb_db_open(txn, LMDB_TXPOOL_BLOB, MDB_CREATE, m_txpool_blob, "Failed to open db handle for m_txpool_blob"); @@ -1467,7 +1601,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) mdb_set_dupsort(txn, m_tx_indices, compare_hash32); mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_leaves, compare_uint64); - mdb_set_dupsort(txn, m_branches, compare_uint64); + mdb_set_dupsort(txn, m_layers, compare_uint64); mdb_set_dupsort(txn, m_output_txs, compare_uint64); mdb_set_dupsort(txn, m_block_info, compare_uint64); if (!(mdb_flags & MDB_RDONLY)) @@ -1647,8 +1781,8 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to drop m_spent_keys: ", result).c_str())); if (auto result = mdb_drop(txn, m_leaves, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_leaves: ", result).c_str())); - if (auto result = mdb_drop(txn, m_branches, 0)) - throw0(DB_ERROR(lmdb_error("Failed to drop m_branches: ", result).c_str())); + if (auto result = mdb_drop(txn, m_layers, 0)) + throw0(DB_ERROR(lmdb_error("Failed to drop m_layers: ", result).c_str())); (void)mdb_drop(txn, m_hf_starting_heights, 0); // this one is dropped in new code if (auto result = mdb_drop(txn, m_hf_versions, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_hf_versions: ", result).c_str())); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index c31250af2..688f4f998 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -65,7 +65,7 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_spent_keys; MDB_cursor *m_txc_leaves; - MDB_cursor *m_txc_branches; + MDB_cursor *m_txc_layers; MDB_cursor *m_txc_txpool_meta; MDB_cursor *m_txc_txpool_blob; @@ -91,7 +91,7 @@ typedef struct mdb_txn_cursors #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_spent_keys m_cursors->m_txc_spent_keys #define m_cur_leaves m_cursors->m_txc_leaves -#define m_cur_branches m_cursors->m_txc_branches +#define m_cur_layers m_cursors->m_txc_layers #define m_cur_txpool_meta m_cursors->m_txc_txpool_meta #define m_cur_txpool_blob m_cursors->m_txc_txpool_blob #define m_cur_alt_blocks m_cursors->m_txc_alt_blocks @@ -115,7 +115,7 @@ typedef struct mdb_rflags bool m_rf_tx_outputs; bool m_rf_spent_keys; bool m_rf_leaves; - bool m_rf_branches; + bool m_rf_layers; bool m_rf_txpool_meta; bool m_rf_txpool_blob; bool m_rf_alt_blocks; @@ -363,6 +363,10 @@ public: static int compare_hash32(const MDB_val *a, const MDB_val *b); static int compare_string(const MDB_val *a, const MDB_val *b); + // make private + virtual void grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, + const std::vector &new_leaves); + private: void do_resize(uint64_t size_increase=0); @@ -406,6 +410,12 @@ private: virtual void remove_spent_key(const crypto::key_image& k_image); + template + void grow_layer(const std::vector> &layer_extensions, + const std::size_t c_idx, + const std::size_t layer_idx, + const fcmp::curve_trees::LastChunkData *last_chunk_data); + uint64_t num_outputs() const; // Hard fork @@ -471,7 +481,7 @@ private: MDB_dbi m_spent_keys; MDB_dbi m_leaves; - MDB_dbi m_branches; + MDB_dbi m_layers; MDB_dbi m_txpool_meta; MDB_dbi m_txpool_blob; diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index 308bdd4c2..f05338e1f 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -116,6 +116,8 @@ public: virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices) override {} virtual void add_spent_key(const crypto::key_image& k_image) override {} virtual void remove_spent_key(const crypto::key_image& k_image) override {} + virtual void grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, + const std::vector &new_leaves) override {}; virtual bool for_all_key_images(std::function) const override { return true; } virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function) const override { return true; } diff --git a/src/fcmp/curve_trees.cpp b/src/fcmp/curve_trees.cpp index 0fd0fb913..6801475bc 100644 --- a/src/fcmp/curve_trees.cpp +++ b/src/fcmp/curve_trees.cpp @@ -243,7 +243,7 @@ CurveTrees::LeafTuple CurveTrees::output_to_leaf template typename CurveTrees::TreeExtension CurveTrees::get_tree_extension( const LastChunks &existing_last_chunks, - const std::vector &new_leaf_tuples) + const std::vector &new_leaf_tuples) const { TreeExtension tree_extension; diff --git a/src/fcmp/curve_trees.h b/src/fcmp/curve_trees.h index 903a6cc53..642ff9231 100644 --- a/src/fcmp/curve_trees.h +++ b/src/fcmp/curve_trees.h @@ -143,7 +143,7 @@ public: // Take in the existing last chunks of each layer in the tree, as well as new leaves to add to the tree, // and return a tree extension struct that can be used to extend a global tree TreeExtension get_tree_extension(const LastChunks &existing_last_chunks, - const std::vector &new_leaf_tuples); + const std::vector &new_leaf_tuples) const; //private member functions private: diff --git a/tests/unit_tests/blockchain_db.cpp b/tests/unit_tests/blockchain_db.cpp index 66219322e..e5085bccb 100644 --- a/tests/unit_tests/blockchain_db.cpp +++ b/tests/unit_tests/blockchain_db.cpp @@ -39,6 +39,7 @@ #include "blockchain_db/blockchain_db.h" #include "blockchain_db/lmdb/db_lmdb.h" #include "cryptonote_basic/cryptonote_format_utils.h" +#include "curve_trees.h" using namespace cryptonote; using epee::string_tools::pod_to_hex; @@ -341,4 +342,37 @@ TYPED_TEST(BlockchainDBTest, RetrieveBlockData) ASSERT_HASH_EQ(get_block_hash(this->m_blocks[1].first), hashes[1]); } +TYPED_TEST(BlockchainDBTest, GrowCurveTrees) +{ + boost::filesystem::path tempPath = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + std::string dirPath = tempPath.string(); + + this->set_prefix(dirPath); + + // make sure open does not throw + ASSERT_NO_THROW(this->m_db->open(dirPath)); + this->get_filenames(); + this->init_hard_fork(); + + db_wtxn_guard guard(this->m_db); + + CHECK_AND_ASSERT_THROW_MES(HELIOS_GENERATORS_LEN >= HELIOS_CHUNK_WIDTH, "helios generators < chunk width"); + CHECK_AND_ASSERT_THROW_MES(SELENE_GENERATORS_LEN >= (SELENE_CHUNK_WIDTH * CurveTreesV1::LEAF_TUPLE_SIZE), + "selene generators < max chunk width"); + + Helios helios(HELIOS_GENERATORS, HELIOS_HASH_INIT_POINT); + Selene selene(SELENE_GENERATORS, SELENE_HASH_INIT_POINT); + + auto curve_trees = CurveTreesV1( + helios, + selene, + HELIOS_CHUNK_WIDTH, + SELENE_CHUNK_WIDTH); + + // Grow tree by 1 leaf + ASSERT_NO_THROW(this->m_db->grow_tree(curve_trees, generate_random_leaves(curve_trees, 1))); + + // TODO: Validate the tree +} + } // anonymous namespace diff --git a/tests/unit_tests/curve_trees.cpp b/tests/unit_tests/curve_trees.cpp index 44f123b7a..e2a588c44 100644 --- a/tests/unit_tests/curve_trees.cpp +++ b/tests/unit_tests/curve_trees.cpp @@ -521,7 +521,7 @@ void CurveTreesUnitTest::log_tree(const CurveTreesUnitTest::Tree &tree) //---------------------------------------------------------------------------------------------------------------------- // Test helpers //---------------------------------------------------------------------------------------------------------------------- -static const std::vector generate_random_leaves(const CurveTreesV1 &curve_trees, +const std::vector generate_random_leaves(const CurveTreesV1 &curve_trees, const std::size_t num_leaves) { std::vector tuples; @@ -653,25 +653,10 @@ static void grow_tree_test(Helios &helios, //---------------------------------------------------------------------------------------------------------------------- TEST(curve_trees, grow_tree) { - // TODO: use static constant generators and hash init points - const std::size_t HELIOS_GENERATORS_LEN = 128; - const std::size_t SELENE_GENERATORS_LEN = 256; - - // https://github.com/kayabaNerve/fcmp-plus-plus/blob - // /b2742e86f3d18155fd34dd1ed69cb8f79b900fce/crypto/fcmps/src/tests.rs#L81-L82 - const std::size_t HELIOS_CHUNK_WIDTH = 38; - const std::size_t SELENE_CHUNK_WIDTH = 18; - CHECK_AND_ASSERT_THROW_MES(HELIOS_GENERATORS_LEN >= HELIOS_CHUNK_WIDTH, "helios generators < chunk width"); CHECK_AND_ASSERT_THROW_MES(SELENE_GENERATORS_LEN >= (SELENE_CHUNK_WIDTH * CurveTreesV1::LEAF_TUPLE_SIZE), "selene generators < max chunk width"); - const Helios::Generators HELIOS_GENERATORS = fcmp::tower_cycle::random_helios_generators(HELIOS_GENERATORS_LEN); - const Selene::Generators SELENE_GENERATORS = fcmp::tower_cycle::random_selene_generators(SELENE_GENERATORS_LEN); - - const Helios::Point HELIOS_HASH_INIT_POINT = fcmp::tower_cycle::random_helios_hash_init_point(); - const Selene::Point SELENE_HASH_INIT_POINT = fcmp::tower_cycle::random_selene_hash_init_point(); - Helios helios(HELIOS_GENERATORS, HELIOS_HASH_INIT_POINT); Selene selene(SELENE_GENERATORS, SELENE_HASH_INIT_POINT); diff --git a/tests/unit_tests/curve_trees.h b/tests/unit_tests/curve_trees.h index cd3d8f370..0be03172d 100644 --- a/tests/unit_tests/curve_trees.h +++ b/tests/unit_tests/curve_trees.h @@ -74,3 +74,21 @@ public: private: CurveTreesV1 &m_curve_trees; }; + +const std::vector generate_random_leaves(const CurveTreesV1 &curve_trees, + const std::size_t num_leaves); + +// TODO: use static constant generators and hash init points +const std::size_t HELIOS_GENERATORS_LEN = 128; +const std::size_t SELENE_GENERATORS_LEN = 256; + +// https://github.com/kayabaNerve/fcmp-plus-plus/blob +// /b2742e86f3d18155fd34dd1ed69cb8f79b900fce/crypto/fcmps/src/tests.rs#L81-L82 +const std::size_t HELIOS_CHUNK_WIDTH = 38; +const std::size_t SELENE_CHUNK_WIDTH = 18; + +const Helios::Generators HELIOS_GENERATORS = fcmp::tower_cycle::random_helios_generators(HELIOS_GENERATORS_LEN); +const Selene::Generators SELENE_GENERATORS = fcmp::tower_cycle::random_selene_generators(SELENE_GENERATORS_LEN); + +const Helios::Point HELIOS_HASH_INIT_POINT = fcmp::tower_cycle::random_helios_hash_init_point(); +const Selene::Point SELENE_HASH_INIT_POINT = fcmp::tower_cycle::random_selene_hash_init_point();