Mirrors
/
monero
mirror of https://github.com/monero-project/monero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

start LMDB grow_tree

This commit is contained in:
j-berman 2024-05-23 23:56:23 -07:00
parent e68ea2e054
commit af9b74fcc7
10 changed files with 217 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/blockchain_db/CMakeLists.txt
View File

@ -45,6 +45,7 @@ target_link_libraries(blockchain_db
PUBLIC
common
cncrypto
fcmp
ringct
${LMDB_LIBRARY}
${Boost_FILESYSTEM_LIBRARY}

4
src/blockchain_db/blockchain_db.h
View File

@ -40,6 +40,7 @@
#include "cryptonote_basic/difficulty.h"
#include "cryptonote_basic/hardfork.h"
#include "cryptonote_protocol/enums.h"
#include "fcmp/curve_trees.h"
/** \file
* Cryptonote Blockchain Database Interface
@ -1764,6 +1765,9 @@ public:
*/
virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const = 0;
// TODO: description and make private
virtual void grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees,
const std::vector<fcmp::curve_trees::CurveTreesV1::LeafTuple> &new_leaves) = 0;
//
// Hard fork related storage

148
src/blockchain_db/lmdb/db_lmdb.cpp
View File

@ -200,7 +200,7 @@ namespace
* spent_keys input hash -
*
* leaves leaf_idx {O.x, I.x, C.x}
* branches layer_idx [{branch_idx, branch_hash}...]
* layers layer_idx [{child_chunk_idx, child_chunk_hash}...]
*
* txpool_meta txn hash txn metadata
* txpool_blob txn hash txn blob
@ -213,7 +213,7 @@ namespace
* attached as a prefix on the Data to serve as the DUPSORT key.
* (DUPFIXED saves 8 bytes per record.)
*
* The output_amounts and branches tables don't use a dummy key, but use DUPSORT
* The output_amounts and layers tables don't use a dummy key, but use DUPSORT
*/
const char* const LMDB_BLOCKS = "blocks";
const char* const LMDB_BLOCK_HEIGHTS = "block_heights";
@ -233,7 +233,7 @@ const char* const LMDB_SPENT_KEYS = "spent_keys";
// Curve trees tree types
const char* const LMDB_LEAVES = "leaves";
const char* const LMDB_BRANCHES = "branches";
const char* const LMDB_LAYERS = "layers";
const char* const LMDB_TXPOOL_META = "txpool_meta";
const char* const LMDB_TXPOOL_BLOB = "txpool_blob";
@ -358,6 +358,11 @@ typedef struct outtx {
uint64_t local_index;
} outtx;
typedef struct layer_val {
uint64_t child_chunk_idx;
std::array<uint8_t, 32UL> child_chunk_hash;
} layer_val;
std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0};
std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT;
@ -1295,6 +1300,135 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image)
}
}
template<typename C>
void BlockchainLMDB::grow_layer(const std::vector<fcmp::curve_trees::LayerExtension<C>> &layer_extensions,
const std::size_t ext_idx,
const std::size_t layer_idx,
const fcmp::curve_trees::LastChunkData<C> *last_chunk_ptr)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
mdb_txn_cursors *m_cursors = &m_wcursors;
CURSOR(layers)
CHECK_AND_ASSERT_THROW_MES(ext_idx < layer_extensions.size(), "unexpected layer extension");
const auto &ext = layer_extensions[ext_idx];
CHECK_AND_ASSERT_THROW_MES(!ext.hashes.empty(), "empty layer extension");
// TODO: make sure last_chunk_ptr->parent_layer_size is correct
// Check if we started at 0 if fresh layer, or if started 1 after the last element in the layer
const bool started_after_tip = (ext.start_idx == 0 && last_chunk_ptr == nullptr)
|| (last_chunk_ptr != nullptr && ext.start_idx == last_chunk_ptr->parent_layer_size);
// Check if we updated the last element in the layer
const bool started_at_tip = (last_chunk_ptr != nullptr
&& (ext.start_idx + 1) == last_chunk_ptr->parent_layer_size);
CHECK_AND_ASSERT_THROW_MES(started_after_tip || started_at_tip, "unexpected layer start");
MDB_val_copy<uint64_t> k(layer_idx);
if (started_at_tip)
{
// We updated the last hash, so update it
layer_val lv;
lv.child_chunk_idx = ext.start_idx;
lv.child_chunk_hash = std::array<uint8_t, 32UL>(); // ext.hashes.front(); // TODO
MDB_val_set(v, lv);
// We expect to overwrite the existing hash
// TODO: make sure the hash already exists
int result = mdb_cursor_put(m_cur_layers, &k, &v, 0);
if (result != MDB_SUCCESS)
throw0(DB_ERROR(lmdb_error("Failed to update chunk hash: ", result).c_str()));
}
// Now add all the new hashes found in the extension
for (std::size_t i = started_at_tip ? 1 : 0; i < ext.hashes.size(); ++i)
{
layer_val lv;
lv.child_chunk_idx = i + ext.start_idx;
lv.child_chunk_hash = std::array<uint8_t, 32UL>(); // ext.hashes[i]; // TODO
MDB_val_set(v, lv);
// TODO: according to the docs, MDB_APPENDDUP isn't supposed to perform any key comparisons to maximize efficiency.
// Adding MDB_NODUPDATA I assume re-introduces a key comparison. Benchmark MDB_NODUPDATA here
// MDB_NODUPDATA makes sure key/data pair doesn't already exist
int result = mdb_cursor_put(m_cur_layers, &k, &v, MDB_APPENDDUP | MDB_NODUPDATA);
if (result != MDB_SUCCESS)
throw0(DB_ERROR(lmdb_error("Failed to add hash: ", result).c_str()));
}
}
void BlockchainLMDB::grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees,
const std::vector<fcmp::curve_trees::CurveTreesV1::LeafTuple> &new_leaves)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
mdb_txn_cursors *m_cursors = &m_wcursors;
CURSOR(leaves)
// TODO: read every layer's last chunks
const auto last_chunks = fcmp::curve_trees::CurveTreesV1::LastChunks{};
const auto tree_extension = curve_trees.get_tree_extension(last_chunks, new_leaves);
// Insert the leaves
const auto &leaves = tree_extension.leaves;
for (std::size_t i = 0; i < leaves.tuples.size(); ++i)
{
MDB_val_copy<uint64_t> k(i + leaves.start_idx);
MDB_val_set(v, leaves.tuples[i]);
// TODO: according to the docs, MDB_APPENDDUP isn't supposed to perform any key comparisons to maximize efficiency.
// Adding MDB_NOOVERWRITE I assume re-introduces a key comparison. Benchmark NOOVERWRITE here
// MDB_NOOVERWRITE makes sure key doesn't already exist
int result = mdb_cursor_put(m_cur_leaves, &k, &v, MDB_APPENDDUP | MDB_NOOVERWRITE);
if (result != MDB_SUCCESS)
throw0(DB_ERROR(lmdb_error("Failed to add leaf: ", result).c_str()));
}
// Grow the layers
const auto &c2_extensions = tree_extension.c2_layer_extensions;
const auto &c1_extensions = tree_extension.c1_layer_extensions;
CHECK_AND_ASSERT_THROW_MES(!c2_extensions.empty(), "empty c2 extensions");
bool use_c2 = true;
std::size_t c2_idx = 0;
std::size_t c1_idx = 0;
for (std::size_t i = 0; i < (c2_extensions.size() + c1_extensions.size()); ++i)
{
const std::size_t layer_idx = c2_idx + c1_idx;
if (use_c2)
{
const auto *c2_last_chunk_ptr = (c2_idx >= last_chunks.c2_last_chunks.size())
? nullptr
: &last_chunks.c2_last_chunks[c2_idx];
this->grow_layer<fcmp::curve_trees::Selene>(c2_extensions, c2_idx, layer_idx, c2_last_chunk_ptr);
++c2_idx;
}
else
{
const auto *c1_last_chunk_ptr = (c1_idx >= last_chunks.c1_last_chunks.size())
? nullptr
: &last_chunks.c1_last_chunks[c1_idx];
this->grow_layer<fcmp::curve_trees::Helios>(c1_extensions, c1_idx, layer_idx, c1_last_chunk_ptr);
++c1_idx;
}
use_c2 = !use_c2;
}
}
BlockchainLMDB::~BlockchainLMDB()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -1445,7 +1579,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags)
lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys");
lmdb_db_open(txn, LMDB_LEAVES, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_leaves, "Failed to open db handle for m_leaves");
lmdb_db_open(txn, LMDB_BRANCHES, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_branches, "Failed to open db handle for m_branches");
lmdb_db_open(txn, LMDB_LAYERS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_layers, "Failed to open db handle for m_layers");
lmdb_db_open(txn, LMDB_TXPOOL_META, MDB_CREATE, m_txpool_meta, "Failed to open db handle for m_txpool_meta");
lmdb_db_open(txn, LMDB_TXPOOL_BLOB, MDB_CREATE, m_txpool_blob, "Failed to open db handle for m_txpool_blob");
@ -1467,7 +1601,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags)
mdb_set_dupsort(txn, m_tx_indices, compare_hash32);
mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
mdb_set_dupsort(txn, m_leaves, compare_uint64);
mdb_set_dupsort(txn, m_branches, compare_uint64);
mdb_set_dupsort(txn, m_layers, compare_uint64);
mdb_set_dupsort(txn, m_output_txs, compare_uint64);
mdb_set_dupsort(txn, m_block_info, compare_uint64);
if (!(mdb_flags & MDB_RDONLY))
@ -1647,8 +1781,8 @@ void BlockchainLMDB::reset()
throw0(DB_ERROR(lmdb_error("Failed to drop m_spent_keys: ", result).c_str()));
if (auto result = mdb_drop(txn, m_leaves, 0))
throw0(DB_ERROR(lmdb_error("Failed to drop m_leaves: ", result).c_str()));
if (auto result = mdb_drop(txn, m_branches, 0))
throw0(DB_ERROR(lmdb_error("Failed to drop m_branches: ", result).c_str()));
if (auto result = mdb_drop(txn, m_layers, 0))
throw0(DB_ERROR(lmdb_error("Failed to drop m_layers: ", result).c_str()));
(void)mdb_drop(txn, m_hf_starting_heights, 0); // this one is dropped in new code
if (auto result = mdb_drop(txn, m_hf_versions, 0))
throw0(DB_ERROR(lmdb_error("Failed to drop m_hf_versions: ", result).c_str()));

18
src/blockchain_db/lmdb/db_lmdb.h
View File

@ -65,7 +65,7 @@ typedef struct mdb_txn_cursors
MDB_cursor *m_txc_spent_keys;
MDB_cursor *m_txc_leaves;
MDB_cursor *m_txc_branches;
MDB_cursor *m_txc_layers;
MDB_cursor *m_txc_txpool_meta;
MDB_cursor *m_txc_txpool_blob;
@ -91,7 +91,7 @@ typedef struct mdb_txn_cursors
#define m_cur_tx_outputs m_cursors->m_txc_tx_outputs
#define m_cur_spent_keys m_cursors->m_txc_spent_keys
#define m_cur_leaves m_cursors->m_txc_leaves
#define m_cur_branches m_cursors->m_txc_branches
#define m_cur_layers m_cursors->m_txc_layers
#define m_cur_txpool_meta m_cursors->m_txc_txpool_meta
#define m_cur_txpool_blob m_cursors->m_txc_txpool_blob
#define m_cur_alt_blocks m_cursors->m_txc_alt_blocks
@ -115,7 +115,7 @@ typedef struct mdb_rflags
bool m_rf_tx_outputs;
bool m_rf_spent_keys;
bool m_rf_leaves;
bool m_rf_branches;
bool m_rf_layers;
bool m_rf_txpool_meta;
bool m_rf_txpool_blob;
bool m_rf_alt_blocks;
@ -363,6 +363,10 @@ public:
static int compare_hash32(const MDB_val *a, const MDB_val *b);
static int compare_string(const MDB_val *a, const MDB_val *b);
// make private
virtual void grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees,
const std::vector<fcmp::curve_trees::CurveTreesV1::LeafTuple> &new_leaves);
private:
void do_resize(uint64_t size_increase=0);
@ -406,6 +410,12 @@ private:
virtual void remove_spent_key(const crypto::key_image& k_image);
template<typename C>
void grow_layer(const std::vector<fcmp::curve_trees::LayerExtension<C>> &layer_extensions,
const std::size_t c_idx,
const std::size_t layer_idx,
const fcmp::curve_trees::LastChunkData<C> *last_chunk_data);
uint64_t num_outputs() const;
// Hard fork
@ -471,7 +481,7 @@ private:
MDB_dbi m_spent_keys;
MDB_dbi m_leaves;
MDB_dbi m_branches;
MDB_dbi m_layers;
MDB_dbi m_txpool_meta;
MDB_dbi m_txpool_blob;

2
src/blockchain_db/testdb.h
View File

@ -116,6 +116,8 @@ public:
virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {}
virtual void add_spent_key(const crypto::key_image& k_image) override {}
virtual void remove_spent_key(const crypto::key_image& k_image) override {}
virtual void grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees,
const std::vector<fcmp::curve_trees::CurveTreesV1::LeafTuple> &new_leaves) override {};
virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const override { return true; }
virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const override { return true; }

2
src/fcmp/curve_trees.cpp
View File

@ -243,7 +243,7 @@ CurveTrees<Helios, Selene>::LeafTuple CurveTrees<Helios, Selene>::output_to_leaf
template<typename C1, typename C2>
typename CurveTrees<C1, C2>::TreeExtension CurveTrees<C1, C2>::get_tree_extension(
const LastChunks &existing_last_chunks,
const std::vector<LeafTuple> &new_leaf_tuples)
const std::vector<LeafTuple> &new_leaf_tuples) const
{
TreeExtension tree_extension;

2
src/fcmp/curve_trees.h
View File

@ -143,7 +143,7 @@ public:
// Take in the existing last chunks of each layer in the tree, as well as new leaves to add to the tree,
// and return a tree extension struct that can be used to extend a global tree
TreeExtension get_tree_extension(const LastChunks &existing_last_chunks,
const std::vector<LeafTuple> &new_leaf_tuples);
const std::vector<LeafTuple> &new_leaf_tuples) const;
//private member functions
private:

34
tests/unit_tests/blockchain_db.cpp
View File

@ -39,6 +39,7 @@
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "curve_trees.h"
using namespace cryptonote;
using epee::string_tools::pod_to_hex;
@ -341,4 +342,37 @@ TYPED_TEST(BlockchainDBTest, RetrieveBlockData)
ASSERT_HASH_EQ(get_block_hash(this->m_blocks[1].first), hashes[1]);
}
TYPED_TEST(BlockchainDBTest, GrowCurveTrees)
{
boost::filesystem::path tempPath = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path();
std::string dirPath = tempPath.string();
this->set_prefix(dirPath);
// make sure open does not throw
ASSERT_NO_THROW(this->m_db->open(dirPath));
this->get_filenames();
this->init_hard_fork();
db_wtxn_guard guard(this->m_db);
CHECK_AND_ASSERT_THROW_MES(HELIOS_GENERATORS_LEN >= HELIOS_CHUNK_WIDTH, "helios generators < chunk width");
CHECK_AND_ASSERT_THROW_MES(SELENE_GENERATORS_LEN >= (SELENE_CHUNK_WIDTH * CurveTreesV1::LEAF_TUPLE_SIZE),
"selene generators < max chunk width");
Helios helios(HELIOS_GENERATORS, HELIOS_HASH_INIT_POINT);
Selene selene(SELENE_GENERATORS, SELENE_HASH_INIT_POINT);
auto curve_trees = CurveTreesV1(
helios,
selene,
HELIOS_CHUNK_WIDTH,
SELENE_CHUNK_WIDTH);
// Grow tree by 1 leaf
ASSERT_NO_THROW(this->m_db->grow_tree(curve_trees, generate_random_leaves(curve_trees, 1)));
// TODO: Validate the tree
}
} // anonymous namespace

17
tests/unit_tests/curve_trees.cpp
View File

@ -521,7 +521,7 @@ void CurveTreesUnitTest::log_tree(const CurveTreesUnitTest::Tree &tree)
//----------------------------------------------------------------------------------------------------------------------
// Test helpers
//----------------------------------------------------------------------------------------------------------------------
static const std::vector<CurveTreesV1::LeafTuple> generate_random_leaves(const CurveTreesV1 &curve_trees,
const std::vector<CurveTreesV1::LeafTuple> generate_random_leaves(const CurveTreesV1 &curve_trees,
const std::size_t num_leaves)
{
std::vector<CurveTreesV1::LeafTuple> tuples;
@ -653,25 +653,10 @@ static void grow_tree_test(Helios &helios,
//----------------------------------------------------------------------------------------------------------------------
TEST(curve_trees, grow_tree)
{
// TODO: use static constant generators and hash init points
const std::size_t HELIOS_GENERATORS_LEN = 128;
const std::size_t SELENE_GENERATORS_LEN = 256;
// https://github.com/kayabaNerve/fcmp-plus-plus/blob
// /b2742e86f3d18155fd34dd1ed69cb8f79b900fce/crypto/fcmps/src/tests.rs#L81-L82
const std::size_t HELIOS_CHUNK_WIDTH = 38;
const std::size_t SELENE_CHUNK_WIDTH = 18;
CHECK_AND_ASSERT_THROW_MES(HELIOS_GENERATORS_LEN >= HELIOS_CHUNK_WIDTH, "helios generators < chunk width");
CHECK_AND_ASSERT_THROW_MES(SELENE_GENERATORS_LEN >= (SELENE_CHUNK_WIDTH * CurveTreesV1::LEAF_TUPLE_SIZE),
"selene generators < max chunk width");
const Helios::Generators HELIOS_GENERATORS = fcmp::tower_cycle::random_helios_generators(HELIOS_GENERATORS_LEN);
const Selene::Generators SELENE_GENERATORS = fcmp::tower_cycle::random_selene_generators(SELENE_GENERATORS_LEN);
const Helios::Point HELIOS_HASH_INIT_POINT = fcmp::tower_cycle::random_helios_hash_init_point();
const Selene::Point SELENE_HASH_INIT_POINT = fcmp::tower_cycle::random_selene_hash_init_point();
Helios helios(HELIOS_GENERATORS, HELIOS_HASH_INIT_POINT);
Selene selene(SELENE_GENERATORS, SELENE_HASH_INIT_POINT);

18
tests/unit_tests/curve_trees.h
View File

@ -74,3 +74,21 @@ public:
private:
CurveTreesV1 &m_curve_trees;
};
const std::vector<CurveTreesV1::LeafTuple> generate_random_leaves(const CurveTreesV1 &curve_trees,
const std::size_t num_leaves);
// TODO: use static constant generators and hash init points
const std::size_t HELIOS_GENERATORS_LEN = 128;
const std::size_t SELENE_GENERATORS_LEN = 256;
// https://github.com/kayabaNerve/fcmp-plus-plus/blob
// /b2742e86f3d18155fd34dd1ed69cb8f79b900fce/crypto/fcmps/src/tests.rs#L81-L82
const std::size_t HELIOS_CHUNK_WIDTH = 38;
const std::size_t SELENE_CHUNK_WIDTH = 18;
const Helios::Generators HELIOS_GENERATORS = fcmp::tower_cycle::random_helios_generators(HELIOS_GENERATORS_LEN);
const Selene::Generators SELENE_GENERATORS = fcmp::tower_cycle::random_selene_generators(SELENE_GENERATORS_LEN);
const Helios::Point HELIOS_HASH_INIT_POINT = fcmp::tower_cycle::random_helios_hash_init_point();
const Selene::Point SELENE_HASH_INIT_POINT = fcmp::tower_cycle::random_selene_hash_init_point();