Mirrors
/
wownero
mirror of https://git.wownero.com/wownero/wownero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'performance' of https://github.com/LMDB/bitmonero

This commit is contained in:
Riccardo Spagni 2016-04-29 21:41:43 +02:00
parent 8b0d22a2aa 2b0fa05f0d
commit 30dce45643
7 changed files with 1250 additions and 790 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
external/db_drivers/liblmdb/mdb.c vendored
View File

@ -6771,8 +6771,8 @@ set1:
if (op == MDB_GET_BOTH || rc > 0)
return MDB_NOTFOUND;
rc = 0;
*data = olddata;
}
*data = olddata;
} else {
if (mc->mc_xcursor)
@ -10501,8 +10501,11 @@ mdb_drop0(MDB_cursor *mc, int subs)
/* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves.
* This also avoids any P_LEAF2 pages, which have no nodes.
* Also if the DB doesn't have sub-DBs and has no overflow
* pages, omit scanning leaves.
*/
if (mc->mc_flags & C_SUB)
if ((mc->mc_flags & C_SUB) ||
(!subs && !mc->mc_db->md_overflow_pages))
mdb_cursor_pop(mc);
mdb_cursor_copy(mc, &mx);
@ -10529,6 +10532,9 @@ mdb_drop0(MDB_cursor *mc, int subs)
pg, omp->mp_pages);
if (rc)
goto done;
mc->mc_db->md_overflow_pages -= omp->mp_pages;
if (!mc->mc_db->md_overflow_pages && !subs)
break;
} else if (subs && (ni->mn_flags & F_SUBDATA)) {
mdb_xcursor_init1(mc, ni);
rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0);
@ -10536,6 +10542,8 @@ mdb_drop0(MDB_cursor *mc, int subs)
goto done;
}
}
if (!subs && !mc->mc_db->md_overflow_pages)
goto pop;
} else {
if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0)
goto done;
@ -10557,6 +10565,7 @@ mdb_drop0(MDB_cursor *mc, int subs)
/* no more siblings, go back to beginning
* of previous level.
*/
pop:
mdb_cursor_pop(mc);
mc->mc_ki[0] = 0;
for (i=1; i<mc->mc_snum; i++) {

7
src/blockchain_db/blockchain_db.cpp
View File

@ -82,14 +82,17 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
}
}
add_transaction_data(blk_hash, tx, tx_hash);
uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash);
std::vector<uint64_t> amount_output_indices;
// iterate tx.vout using indices instead of C++11 foreach syntax because
// we need the index
for (uint64_t i = 0; i < tx.vout.size(); ++i)
{
add_output(tx_hash, tx.vout[i], i, tx.unlock_time);
amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time));
}
add_tx_amount_output_indices(tx_id, amount_output_indices);
}
uint64_t BlockchainDB::add_block( const block& blk

118
src/blockchain_db/blockchain_db.h
View File

@ -59,6 +59,39 @@
* Unspent transaction outputs are duplicated to quickly gather random
* outputs to use for mixins
*
* Indices and Identifiers:
* The word "index" is used ambiguously throughout this code. It is
* particularly confusing when talking about the output or transaction
* tables since their indexing can refer to themselves or each other.
* I have attempted to clarify these usages here:
*
* Blocks, transactions, and outputs are all identified by a hash.
* For storage efficiency, a 64-bit integer ID is used instead of the hash
* inside the DB. Tables exist to map between hash and ID. A block ID is
* also referred to as its "height". Transactions and outputs generally are
* not referred to by ID outside of this module, but the tx ID is returned
* by tx_exists() and used by get_tx_amount_output_indices(). Like their
* corresponding hashes, IDs are globally unique.
*
* The remaining uses of the word "index" refer to local offsets, and are
* not globally unique. An "amount output index" N refers to the Nth output
* of a specific amount. An "output local index" N refers to the Nth output
* of a specific tx.
*
* Exceptions:
* DB_ERROR -- generic
* DB_OPEN_FAILURE
* DB_CREATE_FAILURE
* DB_SYNC_FAILURE
* BLOCK_DNE
* BLOCK_PARENT_DNE
* BLOCK_EXISTS
* BLOCK_INVALID -- considering making this multiple errors
* TX_DNE
* TX_EXISTS
* OUTPUT_DNE
* OUTPUT_EXISTS
* KEY_IMAGE_EXISTS
*/
namespace cryptonote
@ -80,6 +113,15 @@ struct output_data_t
};
#pragma pack(pop)
#pragma pack(push, 1)
struct tx_data_t
{
uint64_t tx_id;
uint64_t unlock_time;
uint64_t block_id;
};
#pragma pack(pop)
/***********************************
* Exception Definitions
***********************************/
@ -311,14 +353,18 @@ private:
* and the other data passed here, not the separate outputs of the
* transaction.
*
* It returns a tx ID, which is a mapping from the tx_hash. The tx ID
* is used in #add_tx_amount_output_indices().
*
* If any of this cannot be done, the subclass should throw the corresponding
* subclass of DB_EXCEPTION
*
* @param blk_hash the hash of the block containing the transaction
* @param tx the transaction to be added
* @param tx_hash the hash of the transaction
* @return the transaction ID
*/
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0;
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0;
/**
* @brief remove data about a transaction
@ -348,6 +394,9 @@ private:
* future, this tracking (of the number, at least) should be moved to
* this class, as it is necessary and the same among all BlockchainDB.
*
* It returns an amount output index, which is the index of the output
* for its specified amount.
*
* This data should be stored in such a manner that the only thing needed to
* reverse the process is the tx_out.
*
@ -358,25 +407,24 @@ private:
* @param tx_output the output
* @param local_index index of the output in its transaction
* @param unlock_time unlock time/height of the output
* @return amount output index
*/
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0;
virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0;
/**
* @brief remove an output
* @brief store amount output indices for a tx's outputs
*
* The subclass implementing this will remove all output data it stored
* in add_output().
*
* In addition, the subclass is responsible for correctly decrementing
* its global output counter (this may be automatic for some, such as using
* a database backend "count" feature).
* The subclass implementing this will add the amount output indices to its
* backing store in a suitable manner. The tx_id will be the same one that
* was returned from #add_output().
*
* If any of this cannot be done, the subclass should throw the corresponding
* subclass of DB_EXCEPTION
*
* @param tx_output the output to be removed
* @param tx_id ID of the transaction containing these outputs
* @param amount_output_indices the amount output indices of the transaction
*/
virtual void remove_output(const tx_out& tx_output) = 0;
virtual void add_tx_amount_output_indices(const uint64_t tx_id, const std::vector<uint64_t>& amount_output_indices) = 0;
/**
* @brief store a spent key
@ -414,18 +462,6 @@ private:
*/
void pop_block();
/**
* @brief helper function for add_transactions, to add each individual transaction
*
* This function is called by add_transactions() for each transaction to be
* added.
*
* @param blk_hash hash of the block which has the transaction
* @param tx the transaction to add
* @param tx_hash_ptr the hash of the transaction, if already calculated
*/
void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL);
// helper function to remove transaction from blockchain
/**
* @brief helper function to remove transaction from the blockchain
@ -444,6 +480,18 @@ private:
protected:
/**
* @brief helper function for add_transactions, to add each individual transaction
*
* This function is called by add_transactions() for each transaction to be
* added.
*
* @param blk_hash hash of the block which has the transaction
* @param tx the transaction to add
* @param tx_hash_ptr the hash of the transaction, if already calculated
*/
void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL);
mutable uint64_t time_tx_exists = 0; //!< a performance metric
uint64_t time_commit1 = 0; //!< a performance metric
bool m_auto_remove_logs = true; //!< whether or not to automatically remove old logs
@ -930,10 +978,12 @@ public:
* given hash and return true if so, false otherwise.
*
* @param h the hash to check against
* @param tx_id (optional) returns the tx_id for the tx hash
*
* @return true if the transaction exists, otherwise false
*/
virtual bool tx_exists(const crypto::hash& h) const = 0;
virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_id) const = 0;
// return unlock time of tx with hash <h>
/**
@ -1124,37 +1174,21 @@ public:
*/
virtual bool can_thread_bulk_indices() const = 0;
/**
* @brief gets output indices (global) for a transaction's outputs
*
* The subclass should fetch the global output indices for each output
* in the transaction with the given hash.
*
* If the transaction does not exist, the subclass should throw TX_DNE.
*
* If an output cannot be found, the subclass should throw OUTPUT_DNE.
*
* @param h a transaction hash
*
* @return a list of global output indices
*/
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const = 0;
/**
* @brief gets output indices (amount-specific) for a transaction's outputs
*
* The subclass should fetch the amount-specific output indices for each
* output in the transaction with the given hash.
* output in the transaction with the given ID.
*
* If the transaction does not exist, the subclass should throw TX_DNE.
*
* If an output cannot be found, the subclass should throw OUTPUT_DNE.
*
* @param h a transaction hash
* @param tx_id a transaction ID
*
* @return a list of amount-specific output indices
*/
virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const = 0;
virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const = 0;
/**
* @brief check if a key image is stored as spent

1805
src/blockchain_db/lmdb/db_lmdb.cpp
View File

File diff suppressed because it is too large Load Diff

83
src/blockchain_db/lmdb/db_lmdb.h
View File

@ -43,20 +43,13 @@ typedef struct mdb_txn_cursors
{
MDB_cursor *m_txc_blocks;
MDB_cursor *m_txc_block_heights;
MDB_cursor *m_txc_block_hashes;
MDB_cursor *m_txc_block_timestamps;
MDB_cursor *m_txc_block_sizes;
MDB_cursor *m_txc_block_diffs;
MDB_cursor *m_txc_block_coins;
MDB_cursor *m_txc_block_info;
MDB_cursor *m_txc_output_txs;
MDB_cursor *m_txc_output_indices;
MDB_cursor *m_txc_output_amounts;
MDB_cursor *m_txc_output_keys;
MDB_cursor *m_txc_txs;
MDB_cursor *m_txc_tx_heights;
MDB_cursor *m_txc_tx_unlocks;
MDB_cursor *m_txc_tx_indices;
MDB_cursor *m_txc_tx_outputs;
MDB_cursor *m_txc_spent_keys;
@ -66,18 +59,11 @@ typedef struct mdb_txn_cursors
#define m_cur_blocks m_cursors->m_txc_blocks
#define m_cur_block_heights m_cursors->m_txc_block_heights
#define m_cur_block_hashes m_cursors->m_txc_block_hashes
#define m_cur_block_timestamps m_cursors->m_txc_block_timestamps
#define m_cur_block_sizes m_cursors->m_txc_block_sizes
#define m_cur_block_diffs m_cursors->m_txc_block_diffs
#define m_cur_block_coins m_cursors->m_txc_block_coins
#define m_cur_block_info m_cursors->m_txc_block_info
#define m_cur_output_txs m_cursors->m_txc_output_txs
#define m_cur_output_indices m_cursors->m_txc_output_indices
#define m_cur_output_amounts m_cursors->m_txc_output_amounts
#define m_cur_output_keys m_cursors->m_txc_output_keys
#define m_cur_txs m_cursors->m_txc_txs
#define m_cur_tx_heights m_cursors->m_txc_tx_heights
#define m_cur_tx_unlocks m_cursors->m_txc_tx_unlocks
#define m_cur_tx_indices m_cursors->m_txc_tx_indices
#define m_cur_tx_outputs m_cursors->m_txc_tx_outputs
#define m_cur_spent_keys m_cursors->m_txc_spent_keys
#define m_cur_hf_versions m_cursors->m_txc_hf_versions
@ -87,18 +73,11 @@ typedef struct mdb_rflags
bool m_rf_txn;
bool m_rf_blocks;
bool m_rf_block_heights;
bool m_rf_block_hashes;
bool m_rf_block_timestamps;
bool m_rf_block_sizes;
bool m_rf_block_diffs;
bool m_rf_block_coins;
bool m_rf_block_info;
bool m_rf_output_txs;
bool m_rf_output_indices;
bool m_rf_output_amounts;
bool m_rf_output_keys;
bool m_rf_txs;
bool m_rf_tx_heights;
bool m_rf_tx_unlocks;
bool m_rf_tx_indices;
bool m_rf_tx_outputs;
bool m_rf_spent_keys;
bool m_rf_hf_versions;
@ -223,6 +202,7 @@ public:
virtual uint64_t height() const;
virtual bool tx_exists(const crypto::hash& h) const;
virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const;
virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const;
@ -246,10 +226,8 @@ public:
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index);
virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices);
virtual void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &indices);
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const;
virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const;
virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const;
virtual bool has_key_image(const crypto::key_image& img) const;
@ -306,18 +284,23 @@ private:
virtual void remove_block();
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time);
virtual uint64_t add_output(const crypto::hash& tx_hash,
const tx_out& tx_output,
const uint64_t& local_index,
const uint64_t unlock_time
);
virtual void remove_output(const tx_out& tx_output);
virtual void add_tx_amount_output_indices(const uint64_t tx_id,
const std::vector<uint64_t>& amount_output_indices
);
void remove_tx_outputs(const MDB_val *tx_hash, const transaction& tx);
void remove_tx_outputs(const uint64_t tx_id, const transaction& tx);
void remove_output(const MDB_val *out_index, const uint64_t amount);
void remove_amount_output_index(const uint64_t amount, const MDB_val *global_output_index);
void remove_output(const uint64_t amount, const uint64_t& out_index);
virtual void add_spent_key(const crypto::key_image& k_image);
@ -349,16 +332,6 @@ private:
*/
tx_out output_from_blob(const blobdata& blob) const;
/**
* @brief get the global index of the index-th output of the given amount
*
* @param amount the output amount
* @param index the index into the set of outputs of that amount
*
* @return the global index of the desired output
*/
uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index);
void check_open() const;
virtual bool is_read_only() const;
@ -366,25 +339,24 @@ private:
// fix up anything that may be wrong due to past bugs
virtual void fixup();
// migrate from older DB version to current
void migrate(const uint32_t oldversion);
// migrate from DB version 0 to 1
void migrate_0_1();
MDB_env* m_env;
MDB_dbi m_blocks;
MDB_dbi m_block_heights;
MDB_dbi m_block_hashes;
MDB_dbi m_block_timestamps;
MDB_dbi m_block_sizes;
MDB_dbi m_block_diffs;
MDB_dbi m_block_coins;
MDB_dbi m_block_info;
MDB_dbi m_txs;
MDB_dbi m_tx_unlocks;
MDB_dbi m_tx_heights;
MDB_dbi m_tx_indices;
MDB_dbi m_tx_outputs;
MDB_dbi m_output_txs;
MDB_dbi m_output_indices;
MDB_dbi m_output_amounts;
MDB_dbi m_output_keys;
MDB_dbi m_spent_keys;
@ -394,6 +366,7 @@ private:
MDB_dbi m_properties;
uint64_t m_height;
uint64_t m_num_txs;
uint64_t m_num_outputs;
mutable uint64_t m_cum_size; // used in batch size estimation
mutable int m_cum_count;

5
src/cryptonote_core/blockchain.cpp
View File

@ -1971,14 +1971,15 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u
{
LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock);
if (!m_db->tx_exists(tx_id))
uint64_t tx_index;
if (!m_db->tx_exists(tx_id, tx_index))
{
LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id);
return false;
}
// get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts
indexs = m_db->get_tx_amount_output_indices(tx_id);
indexs = m_db->get_tx_amount_output_indices(tx_index);
CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty");
return true;

9
tests/unit_tests/hardfork.cpp
View File

@ -78,6 +78,7 @@ public:
virtual block get_top_block() const { return block(); }
virtual uint64_t height() const { return blocks.size(); }
virtual bool tx_exists(const crypto::hash& h) const { return false; }
virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const { return false; }
virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const { return 0; }
virtual transaction get_tx(const crypto::hash& h) const { return transaction(); }
virtual uint64_t get_tx_count() const { return 0; }
@ -93,13 +94,13 @@ public:
virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) {}
virtual bool can_thread_bulk_indices() const { return false; }
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); }
virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); }
virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector<uint64_t>(); }
virtual bool has_key_image(const crypto::key_image& img) const { return false; }
virtual void remove_block() { blocks.pop_back(); }
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {}
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {return 0;}
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) {}
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {}
virtual void remove_output(const tx_out& tx_output) {}
virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {return 0;}
virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) {}
virtual void add_spent_key(const crypto::key_image& k_image) {}
virtual void remove_spent_key(const crypto::key_image& k_image) {}