blockchain_db: allow implementor to be tx format agnostic

Removes cryptonote::transaction specific things from BlockchainLMDB.
This refactor should be useful during the Seraphis migration.
This commit shouldn't change any observable behavior about the database.
This commit is contained in:
jeffro256 2024-02-15 19:20:48 -06:00
parent 059028a30a
commit 5b85d01356
No known key found for this signature in database
GPG Key ID: 6F79797A6E392442
5 changed files with 125 additions and 129 deletions

View File

@ -184,7 +184,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair
const transaction &tx = txp.first;
bool miner_tx = false;
crypto::hash tx_hash, tx_prunable_hash;
crypto::hash tx_hash, tx_prunable_hash = crypto::null_hash;
if (!tx_hash_ptr)
{
// should only need to compute hash for miner transactions
@ -221,7 +221,21 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair
}
}
uint64_t tx_id = add_transaction_data(blk_hash, txp, tx_hash, tx_prunable_hash);
// Calculate tx unprunable size if not already cached
size_t unprunable_size = txp.first.unprunable_size;
if (0 == unprunable_size)
{
std::stringstream ss;
binary_archive<true> ba(ss);
bool r = const_cast<cryptonote::transaction&>(txp.first).serialize_base(ba);
if (!r)
throw DB_ERROR("Failed to serialize pruned tx");
unprunable_size = ss.str().size();
}
// Add transaction blobs and related hashes to DB
const uint64_t tx_id = add_transaction_data(blk_hash, txp.second, txp.first.unlock_time,
unprunable_size, tx_hash, tx_prunable_hash);
std::vector<uint64_t> amount_output_indices(tx.vout.size());
@ -336,6 +350,10 @@ bool BlockchainDB::is_open() const
void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
{
uint64_t tx_index;
if (!tx_exists(tx_hash, tx_index))
throw TX_DNE("Cannot remove transaction since it doesn't exist");
transaction tx = get_pruned_tx(tx_hash);
for (const txin_v& tx_input : tx.vin)
@ -346,8 +364,19 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
}
}
// need tx as tx.vout has the tx outputs, and the output amounts are needed
remove_transaction_data(tx_hash, tx);
// Remove tx output information in backwards vout order
const auto amount_output_indices = get_tx_amount_output_indices(tx_index, 1).front();
if (amount_output_indices.size() != tx.vout.size())
throw DB_ERROR("tx has outputs, but fetched amount indices had different size");
const bool is_pseudo_rct = tx.version >= 2 && tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen);
for (size_t i = tx.vout.size(); i-- > 0;)
{
const uint64_t indexable_amount = is_pseudo_rct ? 0 : tx.vout[i].amount;
remove_output(indexable_amount, amount_output_indices[i]);
}
// Remove tx blobs, related hashes, and other misc tx output info
remove_transaction_data(tx_hash);
}
block BlockchainDB::get_block_from_height(const uint64_t& height) const
@ -408,6 +437,20 @@ transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const
return tx;
}
bool BlockchainDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f, bool pruned) const
{
const auto blob_ref_handler = [&f, pruned](const crypto::hash& txid, blobdata_ref tx_blob_ref) -> bool {
cryptonote::transaction tx;
if (pruned)
CHECK_AND_ASSERT_MES(parse_and_validate_tx_base_from_blob(tx_blob_ref, tx), false, "failed to parse tx base");
else
CHECK_AND_ASSERT_MES(parse_and_validate_tx_from_blob(tx_blob_ref, tx), false, "failed to parse tx");
return f(txid, tx);
};
return for_all_transactions(blob_ref_handler, pruned);
}
void BlockchainDB::reset_stats()
{
num_calls = 0;

View File

@ -435,30 +435,30 @@ private:
* subclass of DB_EXCEPTION
*
* @param blk_hash the hash of the block containing the transaction
* @param tx the transaction to be added
* @param tx_blob the transaction blob to be added
* @param tx_unlock_time the unlock time of the transaction
* @param unprunable_size the length of the unprunable part of the transaction blob
* @param tx_hash the hash of the transaction
* @param tx_prunable_hash the hash of the prunable part of the transaction
* @param tx_prunable_hash the hash of prunable part of the tx (if is null, don't add prunable data to store)
* @return the transaction ID
*/
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0;
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const blobdata_ref& tx_blob,
const uint64_t tx_unlock_time, const size_t unprunable_size, const crypto::hash& tx_hash,
const crypto::hash& tx_prunable_hash) = 0;
/**
* @brief remove data about a transaction
*
* The subclass implementing this will remove the transaction data
* for the passed transaction. The data to be removed was added in
* add_transaction_data(). Additionally, current subclasses have behavior
* which requires the transaction itself as a parameter here. Future
* implementations should note that this parameter is subject to be removed
* at a later time.
* add_transaction_data() and add_tx_amount_output_indices().
*
* If any of this cannot be done, the subclass should throw the corresponding
* subclass of DB_EXCEPTION
*
* @param tx_hash the hash of the transaction to be removed
* @param tx the transaction
*/
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0;
virtual void remove_transaction_data(const crypto::hash& tx_hash) = 0;
/**
* @brief store an output
@ -488,6 +488,21 @@ private:
*/
virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) = 0;
/**
* @brief remove data about an output
*
* The subclass implementing this will remove the output data
* for the passed transaction. The data to be removed was added in
* add_output().
*
* If any of this cannot be done, the subclass should throw the corresponding
* subclass of DB_EXCEPTION
*
* @param indexable_amount the amount of the set that the output is apart of
* @param out_index global output index returned from add_output()
*/
virtual void remove_output(const uint64_t indexable_amount, const uint64_t& out_index) = 0;
/**
* @brief store amount output indices for a tx's outputs
*
@ -1380,23 +1395,6 @@ public:
*/
virtual uint64_t get_tx_count() const = 0;
/**
* @brief fetches a list of transactions based on their hashes
*
* The subclass should attempt to fetch each transaction referred to by
* the hashes passed.
*
* Currently, if any of the transactions is not in BlockchainDB, the call
* to get_tx in the implementation will throw TX_DNE.
*
* <!-- TODO: decide if this behavior is correct for missing transactions -->
*
* @param hlist a list of hashes
*
* @return the list of transactions
*/
virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const = 0;
// returns height of block that contains transaction with hash <h>
/**
* @brief fetches the height of a transaction's block
@ -1428,13 +1426,6 @@ public:
*/
virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0;
/**
* @brief return index of the first element (should be hidden, but isn't)
*
* @return the index
*/
virtual uint64_t get_indexing_base() const { return 0; }
/**
* @brief get some of an output's data
*
@ -1759,7 +1750,26 @@ public:
*
* @return false if the function returns false for any transaction, otherwise true
*/
virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const = 0;
virtual bool for_all_transactions(std::function<bool(const crypto::hash&, blobdata_ref)>, bool pruned) const = 0;
/**
* @brief runs a function over all transactions stored
*
* The subclass should run the passed function for each transaction it has
* stored, passing (transaction_hash, transaction) as its parameters.
*
* If any call to the function returns false, the subclass should return
* false. Otherwise, the subclass returns true.
*
* The subclass should throw DB_ERROR if any of the expected values are
* not found. Current implementations simply return false.
*
* @param std::function fn the function to run
* @param bool pruned whether to only get pruned tx data, or the whole
*
* @return false if the function returns false for any transaction, otherwise true
*/
virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const;
/**
* @brief runs a function over all outputs stored

View File

@ -883,7 +883,9 @@ void BlockchainLMDB::remove_block()
throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str()));
}
uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash)
uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const blobdata_ref& tx_blob,
const uint64_t tx_unlock_time, const size_t unprunable_size, const crypto::hash& tx_hash,
const crypto::hash& tx_prunable_hash)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@ -909,11 +911,10 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str()));
}
const cryptonote::transaction &tx = txp.first;
txindex ti;
ti.key = tx_hash;
ti.data.tx_id = tx_id;
ti.data.unlock_time = tx.unlock_time;
ti.data.unlock_time = tx_unlock_time;
ti.data.block_id = m_height; // we don't need blk_hash since we know m_height
val_h.mv_size = sizeof(ti);
@ -923,28 +924,15 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str()));
const cryptonote::blobdata_ref &blob = txp.second;
unsigned int unprunable_size = tx.unprunable_size;
if (unprunable_size == 0)
{
std::stringstream ss;
binary_archive<true> ba(ss);
bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba);
if (!r)
throw0(DB_ERROR("Failed to serialize pruned tx"));
unprunable_size = ss.str().size();
}
if (unprunable_size > blob.size())
if (unprunable_size > tx_blob.size())
throw0(DB_ERROR("pruned tx size is larger than tx size"));
MDB_val pruned_blob = {unprunable_size, (void*)blob.data()};
MDB_val pruned_blob = {unprunable_size, (void*)tx_blob.data()};
result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str()));
MDB_val prunable_blob = {blob.size() - unprunable_size, (void*)(blob.data() + unprunable_size)};
MDB_val prunable_blob = {tx_blob.size() - unprunable_size, (void*)(tx_blob.data() + unprunable_size)};
result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str()));
@ -957,7 +945,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
throw0(DB_ERROR(lmdb_error("Failed to add prunable tx id to db transaction: ", result).c_str()));
}
if (tx.version > 1)
if (tx_prunable_hash != crypto::null_hash)
{
MDB_val_set(val_prunable_hash, tx_prunable_hash);
result = mdb_cursor_put(m_cur_txs_prunable_hash, &val_tx_id, &val_prunable_hash, MDB_APPEND);
@ -970,7 +958,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
// TODO: compare pros and cons of looking up the tx hash's tx index once and
// passing it in to functions like this
void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx)
void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash)
{
int result;
@ -1018,16 +1006,14 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
throw1(DB_ERROR(lmdb_error("Error adding removal of tx id to db transaction", result).c_str()));
}
if (tx.version > 1)
{
if ((result = mdb_cursor_get(m_cur_txs_prunable_hash, &val_tx_id, NULL, MDB_SET)))
throw1(DB_ERROR(lmdb_error("Failed to locate prunable hash tx for removal: ", result).c_str()));
result = mdb_cursor_del(m_cur_txs_prunable_hash, 0);
if (result)
throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable hash tx to db transaction: ", result).c_str()));
}
remove_tx_outputs(tip->data.tx_id, tx);
// Remove prunable hash
result = mdb_cursor_get(m_cur_txs_prunable_hash, &val_tx_id, NULL, MDB_SET);
if (result == MDB_NOTFOUND)
LOG_PRINT_L1("tx has no prunable hash to remove: " << tx_hash);
else if (result)
throw1(DB_ERROR(lmdb_error("Failed to locate prunable hash tx for removal: ", result).c_str()));
else if ((result = mdb_cursor_del(m_cur_txs_prunable_hash, 0)))
throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable hash tx to db transaction: ", result).c_str()));
result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_id, NULL, MDB_SET);
if (result == MDB_NOTFOUND)
@ -1136,29 +1122,6 @@ void BlockchainLMDB::add_tx_amount_output_indices(const uint64_t tx_id,
throw0(DB_ERROR(std::string("Failed to add <tx hash, amount output index array> to db transaction: ").append(mdb_strerror(result)).c_str()));
}
void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction& tx)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
std::vector<std::vector<uint64_t>> amount_output_indices_set = get_tx_amount_output_indices(tx_id, 1);
const std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.front();
if (amount_output_indices.empty())
{
if (tx.vout.empty())
LOG_PRINT_L2("tx has no outputs, so no output indices");
else
throw0(DB_ERROR("tx has outputs, but no output indices found"));
}
bool is_pseudo_rct = tx.version >= 2 && tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen);
for (size_t i = tx.vout.size(); i-- > 0;)
{
uint64_t amount = is_pseudo_rct ? 0 : tx.vout[i].amount;
remove_output(amount, amount_output_indices[i]);
}
}
void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_index)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -3375,20 +3338,6 @@ uint64_t BlockchainLMDB::get_tx_count() const
return db_stats.ms_entries;
}
std::vector<transaction> BlockchainLMDB::get_tx_list(const std::vector<crypto::hash>& hlist) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
std::vector<transaction> v;
for (auto& h : hlist)
{
v.push_back(get_tx(h));
}
return v;
}
uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -3649,7 +3598,7 @@ bool BlockchainLMDB::for_blocks_range(const uint64_t& h1, const uint64_t& h2, st
return fret;
}
bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f, bool pruned) const
bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&, blobdata_ref)> f, bool pruned) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@ -3683,25 +3632,22 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&
break;
if (ret)
throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str()));
transaction tx;
blobdata owned_txblob;
blobdata_ref bdref;
if (pruned)
{
blobdata_ref bd{reinterpret_cast<char*>(v.mv_data), v.mv_size};
if (!parse_and_validate_tx_base_from_blob(bd, tx))
throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
bdref = {reinterpret_cast<const char*>(v.mv_data), v.mv_size};
}
else
{
blobdata bd;
bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
owned_txblob.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
ret = mdb_cursor_get(m_cur_txs_prunable, &k, &v, MDB_SET);
if (ret)
throw0(DB_ERROR(lmdb_error("Failed to get prunable tx data the db: ", ret).c_str()));
bd.append(reinterpret_cast<char*>(v.mv_data), v.mv_size);
if (!parse_and_validate_tx_from_blob(bd, tx))
throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
owned_txblob.append(reinterpret_cast<char*>(v.mv_data), v.mv_size);
bdref = owned_txblob;
}
if (!f(hash, tx)) {
if (!f(hash, bdref)) {
fret = false;
break;
}

View File

@ -266,8 +266,6 @@ public:
virtual uint64_t get_tx_count() const;
virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const;
virtual uint64_t get_tx_block_height(const crypto::hash& h) const;
virtual uint64_t get_num_outputs(const uint64_t& amount) const;
@ -309,7 +307,7 @@ public:
virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const;
virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const;
virtual bool for_all_transactions(std::function<bool(const crypto::hash&, blobdata_ref)>, bool pruned) const override;
virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const;
virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const;
virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const;
@ -378,9 +376,11 @@ private:
virtual void remove_block();
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash);
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const blobdata_ref& tx_blob,
const uint64_t tx_unlock_time, const size_t unprunable_size, const crypto::hash& tx_hash,
const crypto::hash& tx_prunable_hash);
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
virtual void remove_transaction_data(const crypto::hash& tx_hash);
virtual uint64_t add_output(const crypto::hash& tx_hash,
const tx_out& tx_output,
@ -393,8 +393,6 @@ private:
const std::vector<uint64_t>& amount_output_indices
);
void remove_tx_outputs(const uint64_t tx_id, const transaction& tx);
void remove_output(const uint64_t amount, const uint64_t& out_index);
virtual void prune_outputs(uint64_t amount);

View File

@ -99,10 +99,8 @@ public:
virtual cryptonote::transaction get_tx(const crypto::hash& h) const override { return cryptonote::transaction(); }
virtual bool get_tx(const crypto::hash& h, cryptonote::transaction &tx) const override { return false; }
virtual uint64_t get_tx_count() const override { return 0; }
virtual std::vector<cryptonote::transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const override { return std::vector<cryptonote::transaction>(); }
virtual uint64_t get_tx_block_height(const crypto::hash& h) const override { return 0; }
virtual uint64_t get_num_outputs(const uint64_t& amount) const override { return 1; }
virtual uint64_t get_indexing_base() const override { return 0; }
virtual cryptonote::output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const override { return cryptonote::output_data_t(); }
virtual cryptonote::tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const override { return cryptonote::tx_out_index(); }
virtual cryptonote::tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const override { return cryptonote::tx_out_index(); }
@ -112,16 +110,17 @@ public:
virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_index, size_t n_txes) const override { return std::vector<std::vector<uint64_t>>(); }
virtual bool has_key_image(const crypto::key_image& img) const override { return false; }
virtual void remove_block() override { }
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<cryptonote::transaction, cryptonote::blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override {return 0;}
virtual void remove_transaction_data(const crypto::hash& tx_hash, const cryptonote::transaction& tx) override {}
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const blobdata_ref& tx_blob, const uint64_t tx_unlock_time, const size_t unprunable_size, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override { return 0; }
virtual void remove_transaction_data(const crypto::hash& tx_hash) override {}
virtual uint64_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) override {return 0;}
virtual void remove_output(const uint64_t indexable_amount, const uint64_t& out_index) override {};
virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {}
virtual void add_spent_key(const crypto::key_image& k_image) override {}
virtual void remove_spent_key(const crypto::key_image& k_image) override {}
virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const override { return true; }
virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const override { return true; }
virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const override { return true; }
virtual bool for_all_transactions(std::function<bool(const crypto::hash&, blobdata_ref)>, bool pruned) const override { return true; }
virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const override { return true; }
virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const override { return true; }
virtual bool is_read_only() const override { return false; }