blockchain_db: speedup tx output gathering

We know all the data we'll want for getblocks.bin is contiguous
This commit is contained in:
moneromooo-monero 2018-12-16 13:28:49 +00:00
parent 6bc0c7e685
commit 008647d7eb
No known key found for this signature in database
GPG Key ID: 686F07454D6CEFC3
9 changed files with 75 additions and 48 deletions

View File

@ -1329,10 +1329,11 @@ public:
* If an output cannot be found, the subclass should throw OUTPUT_DNE. * If an output cannot be found, the subclass should throw OUTPUT_DNE.
* *
* @param tx_id a transaction ID * @param tx_id a transaction ID
* @param n_txes how many txes to get data for, starting with tx_id
* *
* @return a list of amount-specific output indices * @return a list of amount-specific output indices
*/ */
virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const = 0; virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes = 1) const = 0;
/** /**
* @brief check if a key image is stored as spent * @brief check if a key image is stored as spent

View File

@ -1019,7 +1019,8 @@ void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction&
{ {
LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L3("BlockchainLMDB::" << __func__);
std::vector<uint64_t> amount_output_indices = get_tx_amount_output_indices(tx_id); std::vector<std::vector<uint64_t>> amount_output_indices_set = get_tx_amount_output_indices(tx_id, 1);
const std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.front();
if (amount_output_indices.empty()) if (amount_output_indices.empty())
{ {
@ -2628,7 +2629,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con
return indices[0]; return indices[0];
} }
std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_id) const std::vector<std::vector<uint64_t>> BlockchainLMDB::get_tx_amount_output_indices(uint64_t tx_id, size_t n_txes) const
{ {
LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -2637,35 +2638,40 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_
TXN_PREFIX_RDONLY(); TXN_PREFIX_RDONLY();
RCURSOR(tx_outputs); RCURSOR(tx_outputs);
int result = 0;
MDB_val_set(k_tx_id, tx_id); MDB_val_set(k_tx_id, tx_id);
MDB_val v; MDB_val v;
std::vector<uint64_t> amount_output_indices; std::vector<std::vector<uint64_t>> amount_output_indices_set;
amount_output_indices_set.reserve(n_txes);
result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, MDB_SET); MDB_cursor_op op = MDB_SET;
if (result == MDB_NOTFOUND) while (n_txes-- > 0)
LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in "
"tx_outputs, but it should have an empty entry even if it's a tx without "
"outputs");
else if (result)
throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str()));
const uint64_t* indices = (const uint64_t*)v.mv_data;
int num_outputs = v.mv_size / sizeof(uint64_t);
amount_output_indices.reserve(num_outputs);
for (int i = 0; i < num_outputs; ++i)
{ {
// LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << " global output index: " << paired_indices[2*i+1]); int result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, op);
amount_output_indices.push_back(indices[i]); if (result == MDB_NOTFOUND)
LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in "
"tx_outputs, but it should have an empty entry even if it's a tx without "
"outputs");
else if (result)
throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str()));
op = MDB_NEXT;
const uint64_t* indices = (const uint64_t*)v.mv_data;
size_t num_outputs = v.mv_size / sizeof(uint64_t);
amount_output_indices_set.resize(amount_output_indices_set.size() + 1);
std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.back();
amount_output_indices.reserve(num_outputs);
for (size_t i = 0; i < num_outputs; ++i)
{
amount_output_indices.push_back(indices[i]);
}
} }
indices = nullptr;
TXN_POSTFIX_RDONLY(); TXN_POSTFIX_RDONLY();
return amount_output_indices; return amount_output_indices_set;
} }
bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
{ {
LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L3("BlockchainLMDB::" << __func__);

View File

@ -252,7 +252,7 @@ public:
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const;
virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const; virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const;
virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const; virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes) const;
virtual bool has_key_image(const crypto::key_image& img) const; virtual bool has_key_image(const crypto::key_image& img) const;

View File

@ -2297,6 +2297,22 @@ bool Blockchain::check_for_double_spend(const transaction& tx, key_images_contai
return true; return true;
} }
//------------------------------------------------------------------ //------------------------------------------------------------------
bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const
{
LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock);
uint64_t tx_index;
if (!m_db->tx_exists(tx_id, tx_index))
{
MERROR_VER("get_tx_outputs_gindexs failed to find transaction with id = " << tx_id);
return false;
}
indexs = m_db->get_tx_amount_output_indices(tx_index, n_txes);
CHECK_AND_ASSERT_MES(n_txes == indexs.size(), false, "Wrong indexs size");
return true;
}
//------------------------------------------------------------------
bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const
{ {
LOG_PRINT_L3("Blockchain::" << __func__); LOG_PRINT_L3("Blockchain::" << __func__);
@ -2307,16 +2323,9 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u
MERROR_VER("get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); MERROR_VER("get_tx_outputs_gindexs failed to find transaction with id = " << tx_id);
return false; return false;
} }
std::vector<std::vector<uint64_t>> indices = m_db->get_tx_amount_output_indices(tx_index, 1);
// get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts CHECK_AND_ASSERT_MES(indices.size() == 1, false, "Wrong indices size");
indexs = m_db->get_tx_amount_output_indices(tx_index); indexs = indices.front();
if (indexs.empty())
{
// empty indexs is only valid if the vout is empty, which is legal but rare
cryptonote::transaction tx = m_db->get_tx(tx_id);
CHECK_AND_ASSERT_MES(tx.vout.empty(), false, "internal error: global indexes for transaction " << tx_id << " is empty, and tx vout is not");
}
return true; return true;
} }
//------------------------------------------------------------------ //------------------------------------------------------------------

View File

@ -516,10 +516,12 @@ namespace cryptonote
* *
* @param tx_id the hash of the transaction to fetch indices for * @param tx_id the hash of the transaction to fetch indices for
* @param indexs return-by-reference the global indices for the transaction's outputs * @param indexs return-by-reference the global indices for the transaction's outputs
* @param n_txes how many txes in a row to get results for
* *
* @return false if the transaction does not exist, or if no indices are found, otherwise true * @return false if the transaction does not exist, or if no indices are found, otherwise true
*/ */
bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const;
bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const;
/** /**
* @brief stores the blockchain * @brief stores the blockchain

View File

@ -1220,6 +1220,11 @@ namespace cryptonote
return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, indexs); return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, indexs);
} }
//----------------------------------------------------------------------------------------------- //-----------------------------------------------------------------------------------------------
bool core::get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const
{
return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, n_txes, indexs);
}
//-----------------------------------------------------------------------------------------------
void core::pause_mine() void core::pause_mine()
{ {
m_miner.pause(); m_miner.pause();

View File

@ -534,6 +534,7 @@ namespace cryptonote
* @note see Blockchain::get_tx_outputs_gindexs * @note see Blockchain::get_tx_outputs_gindexs
*/ */
bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const;
bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const;
/** /**
* @copydoc Blockchain::get_tail_id * @copydoc Blockchain::get_tail_id

View File

@ -252,19 +252,11 @@ namespace cryptonote
pruned_size += bd.first.first.size(); pruned_size += bd.first.first.size();
unpruned_size += bd.first.first.size(); unpruned_size += bd.first.first.size();
res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices());
res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
if (!req.no_miner_tx)
{
bool r = m_core.get_tx_outputs_gindexs(bd.first.second, res.output_indices.back().indices.back().indices);
if (!r)
{
res.status = "Failed";
return false;
}
}
ntxes += bd.second.size(); ntxes += bd.second.size();
res.output_indices.back().indices.reserve(1 + bd.second.size());
if (req.no_miner_tx)
res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
res.blocks.back().txs.reserve(bd.second.size()); res.blocks.back().txs.reserve(bd.second.size());
res.output_indices.back().indices.reserve(bd.second.size());
for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
{ {
unpruned_size += i->second.size(); unpruned_size += i->second.size();
@ -272,14 +264,25 @@ namespace cryptonote
i->second.clear(); i->second.clear();
i->second.shrink_to_fit(); i->second.shrink_to_fit();
pruned_size += res.blocks.back().txs.back().size(); pruned_size += res.blocks.back().txs.back().size();
}
res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1);
bool r = m_core.get_tx_outputs_gindexs(i->first, res.output_indices.back().indices.back().indices); if (n_txes_to_lookup > 0)
{
std::vector<std::vector<uint64_t>> indices;
bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices);
if (!r) if (!r)
{ {
res.status = "Failed"; res.status = "Failed";
return false; return false;
} }
if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0))
{
res.status = "Failed";
return false;
}
for (size_t i = 0; i < indices.size(); ++i)
res.output_indices.back().indices.push_back({std::move(indices[i])});
} }
} }

View File

@ -96,7 +96,7 @@ public:
virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<cryptonote::output_data_t> &outputs, bool allow_partial = false) {} virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<cryptonote::output_data_t> &outputs, bool allow_partial = false) {}
virtual bool can_thread_bulk_indices() const { return false; } virtual bool can_thread_bulk_indices() const { return false; }
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); } virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); }
virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector<uint64_t>(); } virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_index, size_t n_txes) const { return std::vector<std::vector<uint64_t>>(); }
virtual bool has_key_image(const crypto::key_image& img) const { return false; } virtual bool has_key_image(const crypto::key_image& img) const { return false; }
virtual void remove_block() { } virtual void remove_block() { }
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const cryptonote::transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) {return 0;} virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const cryptonote::transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) {return 0;}