Mirrors
/
wownero
mirror of https://git.wownero.com/wownero/wownero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6358 

8958b4e blockchain_db: faster fetching of consecutive txes (moneromooo-monero)
This commit is contained in:
luigi1111 2020-04-10 16:01:37 -05:00
parent 53800b3b63 8958b4e7aa
commit 7954f8cb18
No known key found for this signature in database
GPG Key ID: F4ACA0183641E010
5 changed files with 121 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/blockchain_db/blockchain_db.h
View File

@ -1290,6 +1290,25 @@ public:
*/
virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const = 0;
/**
* @brief fetches a variable number of blocks and transactions from the given height, in canonical blockchain order
*
* The subclass should return the blocks and transactions stored from the one with the given
* height. The number of blocks returned is variable, based on the max_size passed.
*
* @param start_height the height of the first block
* @param min_count the minimum number of blocks to return, if they exist
* @param max_count the maximum number of blocks to return
* @param max_size the maximum size of block/transaction data to return (will be exceeded by one blocks's worth at most, if min_count is met)
* @param blocks the returned block/transaction data
* @param pruned whether to return full or pruned tx data
* @param skip_coinbase whether to return or skip coinbase transactions (they're in blocks regardless)
* @param get_miner_tx_hash whether to calculate and return the miner (coinbase) tx hash
*
* @return true iff the blocks and transactions were found
*/
virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0;
/**
* @brief fetches the prunable transaction blob with the given hash
*

98
src/blockchain_db/lmdb/db_lmdb.cpp
View File

@ -3108,6 +3108,104 @@ bool BlockchainLMDB::get_pruned_tx_blobs_from(const crypto::hash& h, size_t coun
return true;
}
bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
TXN_PREFIX_RDONLY();
RCURSOR(blocks);
RCURSOR(tx_indices);
RCURSOR(txs_pruned);
if (!pruned)
{
RCURSOR(txs_prunable);
}
blocks.reserve(std::min<size_t>(max_count, 10000)); // guard against very large max count if only checking bytes
const uint64_t blockchain_height = height();
uint64_t size = 0;
MDB_val_copy<uint64_t> key(start_height);
MDB_val k, v, val_tx_id;
uint64_t tx_id = ~0;
MDB_cursor_op op = MDB_SET;
for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_count && (size < max_size || blocks.size() < min_count); ++h)
{
MDB_cursor_op op = h == start_height ? MDB_SET : MDB_NEXT;
int result = mdb_cursor_get(m_cur_blocks, &key, &v, op);
if (result == MDB_NOTFOUND)
throw0(BLOCK_DNE(std::string("Attempt to get block from height ").append(boost::lexical_cast<std::string>(h)).append(" failed -- block not in db").c_str()));
else if (result)
throw0(DB_ERROR(lmdb_error("Error attempting to retrieve a block from the db", result).c_str()));
blocks.resize(blocks.size() + 1);
auto &current_block = blocks.back();
current_block.first.first.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
size += v.mv_size;
cryptonote::block b;
if (!parse_and_validate_block_from_blob(current_block.first.first, b))
throw0(DB_ERROR("Invalid block"));
current_block.first.second = get_miner_tx_hash ? cryptonote::get_transaction_hash(b.miner_tx) : crypto::null_hash;
// get the tx_id for the first tx (the first block's coinbase tx)
if (h == start_height)
{
crypto::hash hash = cryptonote::get_transaction_hash(b.miner_tx);
MDB_val_set(v, hash);
result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
if (result)
throw0(DB_ERROR(lmdb_error("Error attempting to retrieve block coinbase transaction from the db: ", result).c_str()));
const txindex *tip = (const txindex *)v.mv_data;
tx_id = tip->data.tx_id;
val_tx_id.mv_data = &tx_id;
val_tx_id.mv_size = sizeof(tx_id);
}
if (skip_coinbase)
{
result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &v, op);
if (result)
throw0(DB_ERROR(lmdb_error("Error attempting to retrieve transaction data from the db: ", result).c_str()));
if (!pruned)
{
result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &v, op);
if (result)
throw0(DB_ERROR(lmdb_error("Error attempting to retrieve transaction data from the db: ", result).c_str()));
}
}
op = MDB_NEXT;
current_block.second.reserve(b.tx_hashes.size());
for (const auto &tx_hash: b.tx_hashes)
{
// get pruned data
cryptonote::blobdata tx_blob;
result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &v, op);
if (result)
throw0(DB_ERROR(lmdb_error("Error attempting to retrieve transaction data from the db: ", result).c_str()));
tx_blob.assign((const char*)v.mv_data, v.mv_size);
if (!pruned)
{
result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &v, op);
if (result)
throw0(DB_ERROR(lmdb_error("Error attempting to retrieve transaction data from the db: ", result).c_str()));
tx_blob.append(reinterpret_cast<const char*>(v.mv_data), v.mv_size);
}
current_block.second.push_back(std::make_pair(tx_hash, std::move(tx_blob)));
size += current_block.second.back().second.size();
}
}
TXN_POSTFIX_RDONLY();
return true;
}
bool BlockchainLMDB::get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);

1
src/blockchain_db/lmdb/db_lmdb.h
View File

@ -255,6 +255,7 @@ public:
virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const;
virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const;
virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const;

1
src/blockchain_db/testdb.h
View File

@ -70,6 +70,7 @@ public:
virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const { return false; }
virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const { return false; }
virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; }
virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; }

32
src/cryptonote_core/blockchain.cpp
View File

@ -2493,38 +2493,10 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons
}
db_rtxn_guard rtxn_guard(m_db);
total_height = get_current_blockchain_height();
size_t count = 0, size = 0;
blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height)));
for(uint64_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++)
{
blocks.resize(blocks.size()+1);
blocks.back().first.first = m_db->get_block_blob_from_height(i);
block b;
CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first.first, b), false, "internal error, invalid block");
blocks.back().first.second = get_miner_tx_hash ? cryptonote::get_transaction_hash(b.miner_tx) : crypto::null_hash;
std::vector<cryptonote::blobdata> txs;
if (pruned)
{
CHECK_AND_ASSERT_MES(m_db->get_pruned_tx_blobs_from(b.tx_hashes.front(), b.tx_hashes.size(), txs), false, "Failed to retrieve all transactions needed");
}
else
{
std::vector<crypto::hash> mis;
get_transactions_blobs(b.tx_hashes, txs, mis, pruned);
CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found");
}
size += blocks.back().first.first.size();
for (const auto &t: txs)
size += t.size();
CHECK_AND_ASSERT_MES(m_db->get_blocks_from(start_height, 3, max_count, FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE, blocks, pruned, true, get_miner_tx_hash),
false, "Error getting blocks");
CHECK_AND_ASSERT_MES(txs.size() == b.tx_hashes.size(), false, "mismatched sizes of b.tx_hashes and txs");
blocks.back().second.reserve(txs.size());
for (size_t i = 0; i < txs.size(); ++i)
{
blocks.back().second.push_back(std::make_pair(b.tx_hashes[i], std::move(txs[i])));
}
}
return true;
}
//------------------------------------------------------------------