From a2b557fe5c3853be03f33bdabd88eb2c0ffd8136 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 15 Apr 2018 19:16:13 +0100 Subject: [PATCH 01/17] wallet2: use std::move on containers where appropriate --- src/wallet/wallet2.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index bb0953689..4d0f31fde 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1695,8 +1695,8 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, boost::lexical_cast(res.output_indices.size()) + ") sizes from daemon"); blocks_start_height = res.start_height; - blocks = res.blocks; - o_indices = res.output_indices; + blocks = std::move(res.blocks); + o_indices = std::move(res.output_indices); } //---------------------------------------------------------------------------------------------------- void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list &short_chain_history, std::list &hashes) @@ -1714,7 +1714,7 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, res.status); blocks_start_height = res.start_height; - hashes = res.m_block_ids; + hashes = std::move(res.m_block_ids); } //---------------------------------------------------------------------------------------------------- void wallet2::process_blocks(uint64_t start_height, const std::list &blocks, const std::vector &o_indices, uint64_t& blocks_added) @@ -2297,7 +2297,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // switch to the new blocks from the daemon blocks_start_height = next_blocks_start_height; - blocks = next_blocks; + blocks = std::move(next_blocks); o_indices = next_o_indices; // handle error from async fetching thread From 6795bd0d4eb680f378ea779a130cefad618e11d2 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 15 Apr 2018 21:58:09 +0100 Subject: [PATCH 02/17] rpc: add option to skip coinbase info in get_blocks.bin --- src/rpc/core_rpc_server.cpp | 11 +++++++---- src/rpc/core_rpc_server_commands_defs.h | 2 ++ src/wallet/wallet2.cpp | 1 + 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index dc7b6b30f..4c14e5c64 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -249,11 +249,14 @@ namespace cryptonote res.status = "Invalid block"; return false; } - bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(b.miner_tx), res.output_indices.back().indices.back().indices); - if (!r) + if (!req.no_miner_tx) { - res.status = "Failed"; - return false; + bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(b.miner_tx), res.output_indices.back().indices.back().indices); + if (!r) + { + res.status = "Failed"; + return false; + } } size_t txidx = 0; ntxes += bd.second.size(); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 1e624da1b..abebad27b 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -83,10 +83,12 @@ namespace cryptonote std::list block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ uint64_t start_height; bool prune; + bool no_miner_tx; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) KV_SERIALIZE(start_height) KV_SERIALIZE(prune) + KV_SERIALIZE_OPT(no_miner_tx, false) END_KV_SERIALIZE_MAP() }; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 4d0f31fde..98bd6c318 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1684,6 +1684,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, } req.start_height = start_height; + req.no_miner_tx = m_refresh_type == RefreshNoCoinbase; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); From 209ec963b5ed0a3ad98f159b9f500c28bb3e7d78 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 15 Apr 2018 22:17:58 +0100 Subject: [PATCH 03/17] rpc: sanity check on number of txes in a block --- src/rpc/core_rpc_server.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 4c14e5c64..83cf5ba3c 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -258,6 +258,12 @@ namespace cryptonote return false; } } + if(b.tx_hashes.size() != bd.second.size()) + { + MERROR("block " << get_block_hash(b) << ": tx_hashes.size() " << b.tx_hashes.size() << ", bd.second.size() " << bd.second.size()); + res.status = "Failed"; + return false; + } size_t txidx = 0; ntxes += bd.second.size(); for (std::list::iterator i = bd.second.begin(); i != bd.second.end(); ++i) From ed2c81ed95be71bace897e62a0c241068cfde7be Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 16 Apr 2018 00:16:02 +0100 Subject: [PATCH 04/17] replace std::list with std::vector on some hot paths also use reserve where appropriate --- .../blockchain_import.cpp | 8 +-- src/cryptonote_basic/connection_context.h | 2 +- src/cryptonote_core/blockchain.cpp | 66 ++++++++++++------ src/cryptonote_core/blockchain.h | 16 ++--- src/cryptonote_core/cryptonote_core.cpp | 42 ++++++------ src/cryptonote_core/cryptonote_core.h | 40 +++++------ src/cryptonote_core/tx_pool.cpp | 14 +++- src/cryptonote_core/tx_pool.h | 6 +- src/cryptonote_protocol/block_queue.cpp | 18 ++--- src/cryptonote_protocol/block_queue.h | 20 +++--- .../cryptonote_protocol_defs.h | 16 ++--- .../cryptonote_protocol_handler.inl | 67 ++++++++++--------- src/rpc/core_rpc_server.cpp | 37 +++++----- src/rpc/core_rpc_server_commands_defs.h | 18 ++--- src/rpc/daemon_handler.cpp | 30 ++------- src/rpc/daemon_messages.h | 2 +- src/wallet/wallet2.cpp | 22 +++--- src/wallet/wallet2.h | 8 +-- tests/core_proxy/core_proxy.cpp | 2 +- tests/core_proxy/core_proxy.h | 10 +-- tests/core_tests/chain_switch_1.cpp | 10 +-- tests/core_tests/chain_switch_1.h | 4 +- tests/core_tests/chaingen.h | 2 +- tests/core_tests/chaingen001.cpp | 2 +- tests/core_tests/double_spend.cpp | 2 +- tests/core_tests/double_spend.inl | 4 +- tests/core_tests/ring_signature_1.cpp | 12 ++-- tests/unit_tests/ban.cpp | 10 +-- 28 files changed, 257 insertions(+), 233 deletions(-) diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index caa549c13..3078ec31b 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -164,7 +164,7 @@ int pop_blocks(cryptonote::core& core, int num_blocks) return num_blocks; } -int check_flush(cryptonote::core &core, std::list &blocks, bool force) +int check_flush(cryptonote::core &core, std::vector &blocks, bool force) { if (blocks.empty()) return 0; @@ -176,7 +176,7 @@ int check_flush(cryptonote::core &core, std::list &blocks, if (!force && new_height % HASH_OF_HASHES_STEP) return 0; - std::list hashes; + std::vector hashes; for (const auto &b: blocks) { cryptonote::block block; @@ -312,7 +312,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path MINFO("Reading blockchain from bootstrap file..."); std::cout << ENDL; - std::list blocks; + std::vector blocks; // Skip to start_height before we start adding. { @@ -437,7 +437,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { cryptonote::blobdata block; cryptonote::block_to_blob(bp.block, block); - std::list txs; + std::vector txs; for (const auto &tx: bp.txs) { txs.push_back(cryptonote::blobdata()); diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index 3f4651565..eb73ab0ea 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -52,7 +52,7 @@ namespace cryptonote }; state m_state; - std::list m_needed_objects; + std::vector m_needed_objects; std::unordered_set m_requested_objects; uint64_t m_remote_blockchain_height; uint64_t m_last_response_height; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index ad604deef..7518cc4e1 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -242,6 +242,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke MDEBUG("Additional outputs needed: " << absolute_offsets.size() - outputs.size()); std::vector < uint64_t > add_offsets; std::vector add_outputs; + add_outputs.reserve(absolute_offsets.size() - outputs.size()); for (size_t i = outputs.size(); i < absolute_offsets.size(); i++) add_offsets.push_back(absolute_offsets[i]); try @@ -850,6 +851,11 @@ difficulty_type Blockchain::get_difficulty_for_next_block() timestamps.clear(); difficulties.clear(); + if (height > offset) + { + timestamps.reserve(height - offset); + difficulties.reserve(height - offset); + } for (; offset < height; offset++) { timestamps.push_back(m_db->get_block_timestamp(offset)); @@ -1170,6 +1176,7 @@ void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) m_db->block_txn_start(true); // add size of last blocks to vector (or less, if blockchain size < count) size_t start_offset = h - std::min(h, count); + sz.reserve(sz.size() + h - start_offset); for(size_t i = start_offset; i < h; i++) { sz.push_back(m_db->get_block_size(i)); @@ -1367,6 +1374,7 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); CHECK_AND_ASSERT_MES(start_top_height < m_db->height(), false, "internal error: passed start_height not < " << " m_db->height() -- " << start_top_height << " >= " << m_db->height()); size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; + timestamps.reserve(timestamps.size() + start_top_height - stop_offset); while (start_top_height != stop_offset) { timestamps.push_back(m_db->get_block_timestamp(start_top_height)); @@ -1566,7 +1574,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return true; } //------------------------------------------------------------------ -bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks, std::vector& txs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1580,7 +1588,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list missed_ids; + std::vector missed_ids; get_transactions_blobs(blk.second.tx_hashes, txs, missed_ids); CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "has missed transactions in own block in main blockchain"); } @@ -1588,14 +1596,16 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list>& blocks) const +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - if(start_offset >= m_db->height()) + const uint64_t height = m_db->height(); + if(start_offset >= height) return false; - for(size_t i = start_offset; i < start_offset + count && i < m_db->height();i++) + blocks.reserve(blocks.size() + height - start_offset); + for(size_t i = start_offset; i < start_offset + count && i < height;i++) { blocks.push_back(std::make_pair(m_db->get_block_blob_from_height(i), block())); if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) @@ -1620,13 +1630,13 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO CRITICAL_REGION_LOCAL(m_blockchain_lock); m_db->block_txn_start(true); rsp.current_blockchain_height = get_current_blockchain_height(); - std::list> blocks; + std::vector> blocks; get_blocks(arg.blocks, blocks, rsp.missed_ids); for (const auto& bl: blocks) { - std::list missed_tx_ids; - std::list txs; + std::vector missed_tx_ids; + std::vector txs; // FIXME: s/rsp.missed_ids/missed_tx_id/ ? Seems like rsp.missed_ids // is for missed blocks, not missed transactions as well. @@ -1642,8 +1652,8 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO // append missed transaction hashes to response missed_ids field, // as done below if any standalone transactions were requested // and missed. - rsp.missed_ids.splice(rsp.missed_ids.end(), missed_tx_ids); - m_db->block_txn_stop(); + rsp.missed_ids.insert(rsp.missed_ids.end(), missed_tx_ids.begin(), missed_tx_ids.end()); + m_db->block_txn_stop(); return false; } @@ -1656,7 +1666,7 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO e.txs.push_back(tx); } //get another transactions, if need - std::list txs; + std::vector txs; get_transactions_blobs(arg.txs, txs, rsp.missed_ids); //pack aside transactions for (const auto& tx: txs) @@ -1666,11 +1676,12 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO return true; } //------------------------------------------------------------------ -bool Blockchain::get_alternative_blocks(std::list& blocks) const +bool Blockchain::get_alternative_blocks(std::vector& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + blocks.reserve(m_alternative_chains.size()); for (const auto& alt_bl: m_alternative_chains) { blocks.push_back(alt_bl.second.bl); @@ -2090,6 +2101,9 @@ uint64_t Blockchain::block_difficulty(uint64_t i) const return 0; } //------------------------------------------------------------------ +template void reserve_container(std::vector &v, size_t N) { v.reserve(N); } +template void reserve_container(std::list &v, size_t N) { } +//------------------------------------------------------------------ //TODO: return type should be void, throw on exception // alternatively, return true only if no blocks missed template @@ -2098,6 +2112,7 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(blocks, block_ids.size()); for (const auto& block_hash : block_ids) { try @@ -2132,6 +2147,7 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(txs, txs_ids.size()); for (const auto& tx_hash : txs_ids) { try @@ -2158,6 +2174,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(txs, txs_ids.size()); for (const auto& tx_hash : txs_ids) { try @@ -2186,7 +2203,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container // Find the split point between us and foreign blockchain and return // (by reference) the most recent common block hash along with up to // BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. -bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, std::list& hashes, uint64_t& start_height, uint64_t& current_height) const +bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, std::vector& hashes, uint64_t& start_height, uint64_t& current_height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2200,6 +2217,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc m_db->block_txn_start(true); current_height = get_current_blockchain_height(); size_t count = 0; + hashes.reserve(std::max((size_t)(current_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); for(size_t i = start_height; i < current_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { hashes.push_back(m_db->get_block_hash_from_height(i)); @@ -2224,7 +2242,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc // find split point between ours and foreign blockchain (or start at // blockchain height ), and return up to max_count FULL // blocks by reference. -bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2250,13 +2268,14 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons m_db->block_txn_start(true); total_height = get_current_blockchain_height(); size_t count = 0, size = 0; + blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height))); for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) { blocks.resize(blocks.size()+1); blocks.back().first = m_db->get_block_blob_from_height(i); block b; CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first, b), false, "internal error, invalid block"); - std::list mis; + std::vector mis; get_transactions_blobs(b.tx_hashes, blocks.back().second, mis, pruned); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); size += blocks.back().first.size(); @@ -2991,6 +3010,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const crypto::key_image &key_image, const std::vector &pubkeys, const std::vector& sig, uint64_t &result) { std::vector p_output_keys; + p_output_keys.reserve(pubkeys.size()); for (auto &key : pubkeys) { // rct::key and crypto::public_key have the same structure, avoid object ctor/memcpy @@ -3087,6 +3107,7 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons const uint64_t min_block_size = get_min_block_size(version); std::vector sz; get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); + sz.reserve(grace_blocks); for (size_t i = 0; i < grace_blocks; ++i) sz.push_back(min_block_size); @@ -3244,6 +3265,7 @@ bool Blockchain::check_block_timestamp(const block& b, uint64_t& median_ts) cons // need most recent 60 blocks, get index of first of those size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW; + timestamps.reserve(h - offset); for(;offset < h; ++offset) { timestamps.push_back(m_db->get_block_timestamp(offset)); @@ -3270,7 +3292,7 @@ void Blockchain::return_tx_to_pool(std::vector &txs) } } //------------------------------------------------------------------ -bool Blockchain::flush_txes_from_pool(const std::list &txids) +bool Blockchain::flush_txes_from_pool(const std::vector &txids) { CRITICAL_REGION_LOCAL(m_tx_pool); @@ -3460,6 +3482,7 @@ leave: // Iterate over the block's transaction hashes, grabbing each // from the tx_pool and validating them. Each is then added // to txs. Keys spent in each are added to by the double spend check. + txs.reserve(bl.tx_hashes.size()); for (const crypto::hash& tx_id : bl.tx_hashes) { transaction tx; @@ -3873,7 +3896,7 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector &hashes) +uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector &hashes) { // new: . . . . . X X X X X . . . . . . // pre: A A A A B B B B C C C C D D D D @@ -3976,7 +3999,7 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::list &blocks_entry) +bool Blockchain::prepare_handle_incoming_blocks(const std::vector &blocks_entry) { MTRACE("Blockchain::" << __func__); TIME_MEASURE_START(prepare); @@ -4042,6 +4065,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list txs; + std::vector txs; m_tx_pool.get_transactions(txs); size_t blob_size; @@ -4568,6 +4592,6 @@ bool Blockchain::for_all_outputs(uint64_t amount, std::function&, std::list&, std::list&) const; -template bool Blockchain::get_transactions_blobs(const std::vector&, std::list&, std::list&, bool) const; +template bool Blockchain::get_transactions(const std::vector&, std::vector&, std::vector&) const; +template bool Blockchain::get_transactions_blobs(const std::vector&, std::vector&, std::vector&, bool) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index ef736d1e7..bd46be0af 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -157,7 +157,7 @@ namespace cryptonote * * @return false if start_offset > blockchain height, else true */ - bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks, std::vector& txs) const; /** * @brief get blocks from blocks based on start height and count @@ -168,7 +168,7 @@ namespace cryptonote * * @return false if start_offset > blockchain height, else true */ - bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks) const; /** * @brief compiles a list of all blocks stored as alternative chains @@ -177,7 +177,7 @@ namespace cryptonote * * @return true */ - bool get_alternative_blocks(std::list& blocks) const; + bool get_alternative_blocks(std::vector& blocks) const; /** * @brief returns the number of alternative blocks stored @@ -213,7 +213,7 @@ namespace cryptonote * * @return false on erroneous blocks, else true */ - bool prepare_handle_incoming_blocks(const std::list &blocks); + bool prepare_handle_incoming_blocks(const std::vector &blocks); /** * @brief incoming blocks post-processing, cleanup, and disk sync @@ -373,7 +373,7 @@ namespace cryptonote * * @return true if a block found in common, else false */ - bool find_blockchain_supplement(const std::list& qblock_ids, std::list& hashes, uint64_t& start_height, uint64_t& current_height) const; + bool find_blockchain_supplement(const std::list& qblock_ids, std::vector& hashes, uint64_t& start_height, uint64_t& current_height) const; /** * @brief get recent block hashes for a foreign chain @@ -420,7 +420,7 @@ namespace cryptonote * * @return true if a block found in common or req_start_block specified, else false */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; /** * @brief retrieves a set of blocks and their transactions, and possibly other transactions @@ -829,7 +829,7 @@ namespace cryptonote * * @return false if any removals fail, otherwise true */ - bool flush_txes_from_pool(const std::list &txids); + bool flush_txes_from_pool(const std::vector &txids); /** * @brief return a histogram of outputs on the blockchain @@ -952,7 +952,7 @@ namespace cryptonote bool is_within_compiled_block_hash_area(uint64_t height) const; bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } - uint64_t prevalidate_block_hashes(uint64_t height, const std::list &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector &hashes); void lock(); void unlock(); diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 7fc81a87d..8b24af43c 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -323,19 +323,19 @@ namespace cryptonote top_id = m_blockchain_storage.get_tail_id(height); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks, std::vector& txs) const { return m_blockchain_storage.get_blocks(start_offset, count, blocks, txs); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list>& blocks) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks) const { return m_blockchain_storage.get_blocks(start_offset, count, blocks); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector& blocks) const { - std::list> bs; + std::vector> bs; if (!m_blockchain_storage.get_blocks(start_offset, count, bs)) return false; for (const auto &b: bs) @@ -343,7 +343,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const + bool core::get_transactions(const std::vector& txs_ids, std::vector& txs, std::vector& missed_txs) const { return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); } @@ -354,12 +354,12 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const + bool core::get_transactions(const std::vector& txs_ids, std::vector& txs, std::vector& missed_txs) const { return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- - bool core::get_alternative_blocks(std::list& blocks) const + bool core::get_alternative_blocks(std::vector& blocks) const { return m_blockchain_storage.get_alternative_blocks(blocks); } @@ -672,7 +672,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_txs(const std::list& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_txs(const std::vector& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { TRY_ENTRY(); CRITICAL_REGION_LOCAL(m_incoming_tx_lock); @@ -683,7 +683,7 @@ namespace cryptonote tvc.resize(tx_blobs.size()); tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; - std::list::const_iterator it = tx_blobs.begin(); + std::vector::const_iterator it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { tpool.submit(&waiter, [&, i, it] { try @@ -752,7 +752,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { - std::list tx_blobs; + std::vector tx_blobs; tx_blobs.push_back(tx_blob); std::vector tvcv(1); bool r = handle_incoming_txs(tx_blobs, tvcv, keeped_by_block, relayed, do_not_relay); @@ -918,8 +918,8 @@ namespace cryptonote const uint64_t end = start_offset + count - 1; m_blockchain_storage.for_blocks_range(start_offset, end, [this, &emission_amount, &total_fee_amount](uint64_t, const crypto::hash& hash, const block& b){ - std::list txs; - std::list missed_txs; + std::vector txs; + std::vector missed_txs; uint64_t coinbase_amount = get_outs_money_amount(b.miner_tx); this->get_transactions(b.tx_hashes, txs, missed_txs); uint64_t tx_fee_amount = 0; @@ -1015,7 +1015,7 @@ namespace cryptonote bool core::relay_txpool_transactions() { // we attempt to relay txes that should be relayed, but were not - std::list> txs; + std::vector> txs; if (m_mempool.get_relayable_transactions(txs) && !txs.empty()) { cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); @@ -1033,7 +1033,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- void core::on_transaction_relayed(const cryptonote::blobdata& tx_blob) { - std::list> txs; + std::vector> txs; cryptonote::transaction tx; crypto::hash tx_hash, tx_prefix_hash; if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) @@ -1055,7 +1055,7 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const + bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const { return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, max_count); } @@ -1112,7 +1112,7 @@ namespace cryptonote { block_verification_context bvc = boost::value_initialized(); m_miner.pause(); - std::list blocks; + std::vector blocks; try { blocks.push_back(get_block_complete_entry(b, m_mempool)); @@ -1136,8 +1136,8 @@ namespace cryptonote cryptonote_connection_context exclude_context = boost::value_initialized(); NOTIFY_NEW_BLOCK::request arg = AUTO_VAL_INIT(arg); arg.current_blockchain_height = m_blockchain_storage.get_current_blockchain_height(); - std::list missed_txs; - std::list txs; + std::vector missed_txs; + std::vector txs; m_blockchain_storage.get_transactions_blobs(b.tx_hashes, txs, missed_txs); if(missed_txs.size() && m_blockchain_storage.get_block_id_by_height(get_block_height(b)) != get_block_hash(b)) { @@ -1173,7 +1173,7 @@ namespace cryptonote } //----------------------------------------------------------------------------------------------- - bool core::prepare_handle_incoming_blocks(const std::list &blocks) + bool core::prepare_handle_incoming_blocks(const std::vector &blocks) { m_incoming_tx_lock.lock(); m_blockchain_storage.prepare_handle_incoming_blocks(blocks); @@ -1266,7 +1266,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_pool_transactions(std::list& txs, bool include_sensitive_data) const + bool core::get_pool_transactions(std::vector& txs, bool include_sensitive_data) const { m_mempool.get_transactions(txs, include_sensitive_data); return true; @@ -1554,7 +1554,7 @@ namespace cryptonote return m_target_blockchain_height; } //----------------------------------------------------------------------------------------------- - uint64_t core::prevalidate_block_hashes(uint64_t height, const std::list &hashes) + uint64_t core::prevalidate_block_hashes(uint64_t height, const std::vector &hashes) { return get_blockchain_storage().prevalidate_block_hashes(height, hashes); } diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 91bd50729..443290c82 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -133,7 +133,7 @@ namespace cryptonote * * @return true if the transactions made it to the transaction pool, otherwise false */ - bool handle_incoming_txs(const std::list& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_txs(const std::vector& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief handles an incoming block @@ -156,7 +156,7 @@ namespace cryptonote * * @note see Blockchain::prepare_handle_incoming_blocks */ - bool prepare_handle_incoming_blocks(const std::list &blocks); + bool prepare_handle_incoming_blocks(const std::vector &blocks); /** * @copydoc Blockchain::cleanup_handle_incoming_blocks @@ -308,25 +308,25 @@ namespace cryptonote void get_blockchain_top(uint64_t& height, crypto::hash& top_id) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list>&, std::list&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector>&, std::vector&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list>&, std::list&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector>&, std::vector&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks, std::vector& txs) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector& blocks) const; /** * @copydoc Blockchain::get_blocks(const t_ids_container&, t_blocks_container&, t_missed_container&) const @@ -351,14 +351,14 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const; + bool get_transactions(const std::vector& txs_ids, std::vector& txs, std::vector& missed_txs) const; /** * @copydoc Blockchain::get_transactions * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const; + bool get_transactions(const std::vector& txs_ids, std::vector& txs, std::vector& missed_txs) const; /** * @copydoc Blockchain::get_block_by_hash @@ -370,9 +370,9 @@ namespace cryptonote /** * @copydoc Blockchain::get_alternative_blocks * - * @note see Blockchain::get_alternative_blocks(std::list&) const + * @note see Blockchain::get_alternative_blocks(std::vector&) const */ - bool get_alternative_blocks(std::list& blocks) const; + bool get_alternative_blocks(std::vector& blocks) const; /** * @copydoc Blockchain::get_alternative_blocks_count @@ -429,7 +429,7 @@ namespace cryptonote * * @note see tx_memory_pool::get_transactions */ - bool get_pool_transactions(std::list& txs, bool include_unrelayed_txes = true) const; + bool get_pool_transactions(std::vector& txs, bool include_unrelayed_txes = true) const; /** * @copydoc tx_memory_pool::get_txpool_backlog @@ -512,11 +512,11 @@ namespace cryptonote bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; /** - * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list&, std::list > >&, uint64_t&, uint64_t&, size_t) const + * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list&, std::vector > >&, uint64_t&, uint64_t&, size_t) const * - * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list&, std::list > >&, uint64_t&, uint64_t&, size_t) const + * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list&, std::vector > >&, uint64_t&, uint64_t&, size_t) const */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; /** * @brief gets some stats about the daemon @@ -763,7 +763,7 @@ namespace cryptonote * * @return number of usable blocks */ - uint64_t prevalidate_block_hashes(uint64_t height, const std::list &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector &hashes); /** * @brief get free disk space on the blockchain partition diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 164530b3e..2e8dd197d 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -556,11 +556,12 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::get_relayable_transactions(std::list> &txs) const + bool tx_memory_pool::get_relayable_transactions(std::vector> &txs) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + txs.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){ // 0 fee transactions are never relayed if(meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time)) @@ -588,7 +589,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_relayed(const std::list> &txs) + void tx_memory_pool::set_relayed(const std::vector> &txs) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -621,10 +622,11 @@ namespace cryptonote return m_blockchain.get_txpool_tx_count(include_unrelayed_txes); } //--------------------------------------------------------------------------------- - void tx_memory_pool::get_transactions(std::list& txs, bool include_unrelayed_txes) const + void tx_memory_pool::get_transactions(std::vector& txs, bool include_unrelayed_txes) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ transaction tx; if (!parse_and_validate_tx_from_blob(*bd, tx)) @@ -642,6 +644,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ txs.push_back(txid); return true; @@ -653,6 +656,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + backlog.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now}); return true; @@ -741,6 +745,8 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + tx_infos.reserve(m_blockchain.get_txpool_tx_count()); + key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos, include_sensitive_data](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ tx_info txi; txi.id_hash = epee::string_tools::pod_to_hex(txid); @@ -811,6 +817,8 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + tx_infos.reserve(m_blockchain.get_txpool_tx_count()); + key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ cryptonote::rpc::tx_in_pool txi; txi.tx_hash = txid; diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 4ce2f085d..5ccb71196 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -237,7 +237,7 @@ namespace cryptonote * @param include_unrelayed_txes include unrelayed txes in the result * */ - void get_transactions(std::list& txs, bool include_unrelayed_txes = true) const; + void get_transactions(std::vector& txs, bool include_unrelayed_txes = true) const; /** * @brief get a list of all transaction hashes in the pool @@ -324,14 +324,14 @@ namespace cryptonote * * @return true */ - bool get_relayable_transactions(std::list>& txs) const; + bool get_relayable_transactions(std::vector>& txs) const; /** * @brief tell the pool that certain transactions were just relayed * * @param txs the list of transactions (and their hashes) */ - void set_relayed(const std::list>& txs); + void set_relayed(const std::vector>& txs); /** * @brief get the total number of transactions in the pool diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 9ae33d540..c39d67ceb 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -50,10 +50,10 @@ namespace std { namespace cryptonote { -void block_queue::add_blocks(uint64_t height, std::list bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) +void block_queue::add_blocks(uint64_t height, std::vector bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) { boost::unique_lock lock(mutex); - std::list hashes; + std::vector hashes; bool has_hashes = remove_span(height, &hashes); blocks.insert(span(height, std::move(bcel), connection_id, rate, size)); if (has_hashes) @@ -97,7 +97,7 @@ void block_queue::flush_stale_spans(const std::set &live_con } } -bool block_queue::remove_span(uint64_t start_block_height, std::list *hashes) +bool block_queue::remove_span(uint64_t start_block_height, std::vector *hashes) { boost::unique_lock lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) @@ -172,7 +172,7 @@ bool block_queue::requested(const crypto::hash &hash) const return false; } -std::pair block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::list &block_hashes, boost::posix_time::ptime time) +std::pair block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector &block_hashes, boost::posix_time::ptime time) { boost::unique_lock lock(mutex); @@ -183,14 +183,14 @@ std::pair block_queue::reserve_span(uint64_t first_block_hei } uint64_t span_start_height = last_block_height - block_hashes.size() + 1; - std::list::const_iterator i = block_hashes.begin(); + std::vector::const_iterator i = block_hashes.begin(); while (i != block_hashes.end() && requested(*i)) { ++i; ++span_start_height; } uint64_t span_length = 0; - std::list hashes; + std::vector hashes; while (i != block_hashes.end() && span_length < max_blocks) { hashes.push_back(*i); @@ -230,7 +230,7 @@ std::pair block_queue::get_start_gap_span() const return std::make_pair(current_height + 1, first_span_height - current_height - 1); } -std::pair block_queue::get_next_span_if_scheduled(std::list &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const +std::pair block_queue::get_next_span_if_scheduled(std::vector &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const { boost::unique_lock lock(mutex); if (blocks.empty()) @@ -248,7 +248,7 @@ std::pair block_queue::get_next_span_if_scheduled(std::list< return std::make_pair(i->start_block_height, i->nblocks); } -void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list hashes) +void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector hashes) { boost::unique_lock lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) @@ -264,7 +264,7 @@ void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uui } } -bool block_queue::get_next_span(uint64_t &height, std::list &bcel, boost::uuids::uuid &connection_id, bool filled) const +bool block_queue::get_next_span(uint64_t &height, std::vector &bcel, boost::uuids::uuid &connection_id, bool filled) const { boost::unique_lock lock(mutex); if (blocks.empty()) diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 69ddaa435..9059e89ac 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -31,7 +31,7 @@ #pragma once #include -#include +#include #include #include #include @@ -49,15 +49,15 @@ namespace cryptonote struct span { uint64_t start_block_height; - std::list hashes; - std::list blocks; + std::vector hashes; + std::vector blocks; boost::uuids::uuid connection_id; uint64_t nblocks; float rate; size_t size; boost::posix_time::ptime time; - span(uint64_t start_block_height, std::list blocks, const boost::uuids::uuid &connection_id, float rate, size_t size): + span(uint64_t start_block_height, std::vector blocks, const boost::uuids::uuid &connection_id, float rate, size_t size): start_block_height(start_block_height), blocks(std::move(blocks)), connection_id(connection_id), nblocks(this->blocks.size()), rate(rate), size(size), time() {} span(uint64_t start_block_height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time): start_block_height(start_block_height), connection_id(connection_id), nblocks(nblocks), rate(0.0f), size(0), time(time) {} @@ -67,21 +67,21 @@ namespace cryptonote typedef std::set block_map; public: - void add_blocks(uint64_t height, std::list bcel, const boost::uuids::uuid &connection_id, float rate, size_t size); + void add_blocks(uint64_t height, std::vector bcel, const boost::uuids::uuid &connection_id, float rate, size_t size); void add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time = boost::date_time::min_date_time); void flush_spans(const boost::uuids::uuid &connection_id, bool all = false); void flush_stale_spans(const std::set &live_connections); - bool remove_span(uint64_t start_block_height, std::list *hashes = NULL); + bool remove_span(uint64_t start_block_height, std::vector *hashes = NULL); void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height); uint64_t get_max_block_height() const; void print() const; std::string get_overview() const; - std::pair reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::list &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); + std::pair reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); bool is_blockchain_placeholder(const span &span) const; std::pair get_start_gap_span() const; - std::pair get_next_span_if_scheduled(std::list &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; - void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list hashes); - bool get_next_span(uint64_t &height, std::list &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; + std::pair get_next_span_if_scheduled(std::vector &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; + void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector hashes); + bool get_next_span(uint64_t &height, std::vector &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const; size_t get_data_size() const; size_t get_num_filled_spans_prefix() const; diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index cf0043287..db159f0f4 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -109,7 +109,7 @@ namespace cryptonote struct block_complete_entry { blobdata block; - std::list txs; + std::vector txs; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block) KV_SERIALIZE(txs) @@ -145,7 +145,7 @@ namespace cryptonote struct request { - std::list txs; + std::vector txs; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs) @@ -161,8 +161,8 @@ namespace cryptonote struct request { - std::list txs; - std::list blocks; + std::vector txs; + std::vector blocks; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(txs) @@ -177,9 +177,9 @@ namespace cryptonote struct request { - std::list txs; - std::list blocks; - std::list missed_ids; + std::vector txs; + std::vector blocks; + std::vector missed_ids; uint64_t current_blockchain_height; BEGIN_KV_SERIALIZE_MAP() @@ -230,7 +230,7 @@ namespace cryptonote uint64_t start_height; uint64_t total_height; uint64_t cumulative_difficulty; - std::list m_block_ids; + std::vector m_block_ids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(start_height) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 2e1df8078..b4786b903 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -351,7 +351,7 @@ namespace cryptonote return 1; } m_core.pause_mine(); - std::list blocks; + std::vector blocks; blocks.push_back(arg.b); m_core.prepare_handle_incoming_blocks(blocks); for(auto tx_blob_it = arg.b.txs.begin(); tx_blob_it!=arg.b.txs.end();tx_blob_it++) @@ -438,7 +438,7 @@ namespace cryptonote } } - std::list have_tx; + std::vector have_tx; // Instead of requesting missing transactions by hash like BTC, // we do it by index (thanks to a suggestion from moneromooo) because @@ -578,8 +578,8 @@ namespace cryptonote else { std::vector tx_ids; - std::list txes; - std::list missing; + std::vector txes; + std::vector missing; tx_ids.push_back(tx_hash); if (m_core.get_transactions(tx_ids, txes, missing) && missing.empty()) { @@ -626,7 +626,7 @@ namespace cryptonote b.block = arg.b.block; b.txs = have_tx; - std::list blocks; + std::vector blocks; blocks.push_back(b); m_core.prepare_handle_incoming_blocks(blocks); @@ -687,8 +687,8 @@ namespace cryptonote { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_FLUFFY_MISSING_TX (" << arg.missing_tx_indices.size() << " txes), block hash " << arg.block_hash); - std::list> local_blocks; - std::list local_txs; + std::vector> local_blocks; + std::vector local_txs; block b; if (!m_core.get_block_by_hash(arg.block_hash, b)) @@ -725,8 +725,8 @@ namespace cryptonote } } - std::list txs; - std::list missed; + std::vector txs; + std::vector missed; if (!m_core.get_transactions(txids, txs, missed)) { LOG_ERROR_CCONTEXT("Failed to handle request NOTIFY_REQUEST_FLUFFY_MISSING_TX, " @@ -774,10 +774,12 @@ namespace cryptonote return 1; } - for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end();) + std::vector newtxs; + newtxs.reserve(arg.txs.size()); + for (size_t i = 0; i < arg.txs.size(); ++i) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(*tx_blob_it, tvc, false, true, false); + m_core.handle_incoming_tx(arg.txs[i], tvc, false, true, false); if(tvc.m_verifivation_failed) { LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection"); @@ -785,10 +787,9 @@ namespace cryptonote return 1; } if(tvc.m_should_be_relayed) - ++tx_blob_it; - else - arg.txs.erase(tx_blob_it++); + newtxs.push_back(std::move(arg.txs[i])); } + arg.txs = std::move(newtxs); if(arg.txs.size()) { @@ -996,7 +997,7 @@ skip: { const uint64_t previous_height = m_core.get_current_blockchain_height(); uint64_t start_height; - std::list blocks; + std::vector blocks; boost::uuids::uuid span_connection_id; if (!m_block_queue.get_next_span(start_height, blocks, span_connection_id)) { @@ -1070,7 +1071,7 @@ skip: LOG_ERROR_CCONTEXT("Internal error: tvc.size() != block_entry.txs.size()"); return 1; } - std::list::const_iterator it = block_entry.txs.begin(); + std::vector::const_iterator it = block_entry.txs.begin(); for (size_t i = 0; i < tvc.size(); ++i, ++it) { if(tvc[i].m_verifivation_failed) @@ -1248,7 +1249,7 @@ skip: template bool t_cryptonote_protocol_handler::should_download_next_span(cryptonote_connection_context& context) const { - std::list hashes; + std::vector hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; std::pair span; @@ -1267,7 +1268,7 @@ skip: // we might be in a weird case where there is a filled next span, // but it starts higher than the current height uint64_t height; - std::list bcel; + std::vector bcel; if (!m_block_queue.get_next_span(height, bcel, span_connection_id, true)) return false; if (height > m_core.get_current_blockchain_height()) @@ -1415,7 +1416,7 @@ skip: { if (span.second == 0) { - std::list hashes; + std::vector hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); @@ -1441,14 +1442,18 @@ skip: goto skip; } // take out blocks we already have - while (!context.m_needed_objects.empty() && m_core.have_block(context.m_needed_objects.front())) + size_t skip = 0; + while (skip < context.m_needed_objects.size() && m_core.have_block(context.m_needed_objects[skip])) { // if we're popping the last hash, record it so we can ask again from that hash, // this prevents never being able to progress on peers we get old hash lists from - if (context.m_needed_objects.size() == 1) - context.m_last_known_hash = context.m_needed_objects.front(); - context.m_needed_objects.pop_front(); + if (skip + 1 == context.m_needed_objects.size()) + context.m_last_known_hash = context.m_needed_objects[skip]; + ++skip; } + if (skip > 0) + context.m_needed_objects = std::vector(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_needed_objects); MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second); @@ -1456,7 +1461,7 @@ skip: if (span.second == 0 && !force_next_span) { MDEBUG(context << " still no span reserved, we may be in the corner case of next span scheduled and everything else scheduled/filled"); - std::list hashes; + std::vector hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); @@ -1487,23 +1492,21 @@ skip: MERROR("ERROR: skip " << skip << ", m_needed_objects " << context.m_needed_objects.size() << ", first_context_block_height" << first_context_block_height); return false; } - while (skip--) - context.m_needed_objects.pop_front(); + if (skip > 0) + context.m_needed_objects = std::vector(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); if (context.m_needed_objects.size() < span.second) { MERROR("ERROR: span " << span.first << "/" << span.second << ", m_needed_objects " << context.m_needed_objects.size()); return false; } - auto it = context.m_needed_objects.begin(); for (size_t n = 0; n < span.second; ++n) { - req.blocks.push_back(*it); + req.blocks.push_back(context.m_needed_objects[n]); ++count; - context.m_requested_objects.insert(*it); - auto j = it++; - context.m_needed_objects.erase(j); + context.m_requested_objects.insert(context.m_needed_objects[n]); } + context.m_needed_objects = std::vector(context.m_needed_objects.begin() + span.second, context.m_needed_objects.end()); } context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); @@ -1664,7 +1667,7 @@ skip: { NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_arg = AUTO_VAL_INIT(fluffy_arg); fluffy_arg.current_blockchain_height = arg.current_blockchain_height; - std::list fluffy_txs; + std::vector fluffy_txs; fluffy_arg.b = arg.b; fluffy_arg.b.txs = fluffy_txs; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 83cf5ba3c..b921e1438 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -226,7 +226,7 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary(invoke_http_mode::BIN, "/getblocks.bin", req, res, r)) return r; - std::list > > bs; + std::vector > > bs; if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { @@ -235,6 +235,8 @@ namespace cryptonote } size_t pruned_size = 0, unpruned_size = 0, ntxes = 0; + res.blocks.reserve(bs.size()); + res.output_indices.reserve(bs.size()); for(auto& bd: bs) { res.blocks.resize(res.blocks.size()+1); @@ -266,7 +268,9 @@ namespace cryptonote } size_t txidx = 0; ntxes += bd.second.size(); - for (std::list::iterator i = bd.second.begin(); i != bd.second.end(); ++i) + res.blocks.back().txs.reserve(bd.second.size()); + res.output_indices.back().indices.reserve(bd.second.size()); + for (std::vector::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { unpruned_size += i->size(); res.blocks.back().txs.push_back(std::move(*i)); @@ -295,7 +299,7 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/get_alt_blocks_hashes", req, res, r)) return r; - std::list blks; + std::vector blks; if(!m_core.get_alternative_blocks(blks)) { @@ -337,8 +341,8 @@ namespace cryptonote res.status = "Error retrieving block at height " + std::to_string(height); return true; } - std::list txs; - std::list missed_txs; + std::vector txs; + std::vector missed_txs; m_core.get_transactions(blk.tx_hashes, txs, missed_txs); res.blocks.resize(res.blocks.size() + 1); res.blocks.back().block = block_to_blob(blk); @@ -553,8 +557,8 @@ namespace cryptonote } vh.push_back(*reinterpret_cast(b.data())); } - std::list missed_txs; - std::list txs; + std::vector missed_txs; + std::vector txs; bool r = m_core.get_transactions(vh, txs, missed_txs); if(!r) { @@ -575,25 +579,26 @@ namespace cryptonote if(r) { // sort to match original request - std::list sorted_txs; + std::vector sorted_txs; std::vector::const_iterator i; + unsigned txs_processed = 0; for (const crypto::hash &h: vh) { if (std::find(missed_txs.begin(), missed_txs.end(), h) == missed_txs.end()) { - if (txs.empty()) + if (txs.size() == txs_processed) { res.status = "Failed: internal error - txs is empty"; return true; } // core returns the ones it finds in the right order - if (get_transaction_hash(txs.front()) != h) + if (get_transaction_hash(txs[txs_processed]) != h) { res.status = "Failed: tx hash mismatch"; return true; } - sorted_txs.push_back(std::move(txs.front())); - txs.pop_front(); + sorted_txs.push_back(std::move(txs[txs_processed])); + ++txs_processed; } else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end()) { @@ -604,7 +609,7 @@ namespace cryptonote return true; } sorted_txs.push_back(tx); - missed_txs.remove(h); + missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); const std::string hash_string = epee::string_tools::pod_to_hex(h); for (const auto &ti: pool_tx_info) @@ -623,7 +628,7 @@ namespace cryptonote LOG_PRINT_L2("Found " << found_in_pool << "/" << vh.size() << " transactions in the pool"); } - std::list::const_iterator txhi = req.txs_hashes.begin(); + std::vector::const_iterator txhi = req.txs_hashes.begin(); std::vector::const_iterator vhi = vh.begin(); for(auto& tx: txs) { @@ -1664,10 +1669,10 @@ namespace cryptonote PERF_TIMER(on_flush_txpool); bool failed = false; - std::list txids; + std::vector txids; if (req.txids.empty()) { - std::list pool_txs; + std::vector pool_txs; bool r = m_core.get_pool_transactions(pool_txs); if (!r) { diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index abebad27b..49b730149 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -112,7 +112,7 @@ namespace cryptonote struct response { - std::list blocks; + std::vector blocks; uint64_t start_height; uint64_t current_height; std::string status; @@ -190,7 +190,7 @@ namespace cryptonote struct response { - std::list m_block_ids; + std::vector m_block_ids; uint64_t start_height; uint64_t current_height; std::string status; @@ -275,7 +275,7 @@ namespace cryptonote uint64_t total_received; uint64_t total_received_unlocked = 0; // OpenMonero only uint64_t scanned_height; - std::list transactions; + std::vector transactions; uint64_t blockchain_height; uint64_t scanned_block_height; std::string status; @@ -563,7 +563,7 @@ namespace cryptonote { struct request { - std::list txs_hashes; + std::vector txs_hashes; bool decode_as_json; bool prune; @@ -600,11 +600,11 @@ namespace cryptonote struct response { // older compatibility stuff - std::list txs_as_hex; //transactions blobs as hex (old compat) - std::list txs_as_json; //transactions decoded as json (old compat) + std::vector txs_as_hex; //transactions blobs as hex (old compat) + std::vector txs_as_json; //transactions decoded as json (old compat) // in both old and new - std::list missed_tx; //not found transactions + std::vector missed_tx; //not found transactions // new style std::vector txs; @@ -1933,7 +1933,7 @@ namespace cryptonote { struct request { - std::list txids; + std::vector txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txids) @@ -2150,7 +2150,7 @@ namespace cryptonote { struct request { - std::list txids; + std::vector txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txids) diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 39f169cdf..582a3440b 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -50,7 +50,7 @@ namespace rpc void DaemonHandler::handle(const GetBlocksFast::Request& req, GetBlocksFast::Response& res) { - std::list > > blocks; + std::vector > > blocks; if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { @@ -62,9 +62,6 @@ namespace rpc res.blocks.resize(blocks.size()); res.output_indices.resize(blocks.size()); - //TODO: really need to switch uses of std::list to std::vector unless - // it's a huge performance concern - auto it = blocks.begin(); uint64_t block_count = 0; @@ -89,7 +86,7 @@ namespace rpc res.error_details = "incorrect number of transactions retrieved for block"; return; } - std::list txs; + std::vector txs; for (const auto& blob : it->second) { txs.resize(txs.size() + 1); @@ -163,10 +160,10 @@ namespace rpc void DaemonHandler::handle(const GetTransactions::Request& req, GetTransactions::Response& res) { - std::list found_txs; - std::list missed_hashes; + std::vector found_txs_vec; + std::vector missed_vec; - bool r = m_core.get_transactions(req.tx_hashes, found_txs, missed_hashes); + bool r = m_core.get_transactions(req.tx_hashes, found_txs_vec, missed_vec); // TODO: consider fixing core::get_transactions to not hide exceptions if (!r) @@ -176,20 +173,7 @@ namespace rpc return; } - size_t num_found = found_txs.size(); - - // std::list is annoying - std::vector found_txs_vec - { - std::make_move_iterator(std::begin(found_txs)), - std::make_move_iterator(std::end(found_txs)) - }; - - std::vector missed_vec - { - std::make_move_iterator(std::begin(missed_hashes)), - std::make_move_iterator(std::end(missed_hashes)) - }; + size_t num_found = found_txs_vec.size(); std::vector heights(num_found); std::vector in_pool(num_found, false); @@ -204,7 +188,7 @@ namespace rpc // if any missing from blockchain, check in tx pool if (!missed_vec.empty()) { - std::list pool_txs; + std::vector pool_txs; m_core.get_pool_transactions(pool_txs); diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index 1495c845f..8fff369df 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -106,7 +106,7 @@ BEGIN_RPC_MESSAGE_CLASS(GetHashesFast); RPC_MESSAGE_MEMBER(uint64_t, start_height); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(std::list, hashes); + RPC_MESSAGE_MEMBER(std::vector, hashes); RPC_MESSAGE_MEMBER(uint64_t, start_height); RPC_MESSAGE_MEMBER(uint64_t, current_height); END_RPC_MESSAGE_RESPONSE; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 98bd6c318..56a2a2adf 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1649,7 +1649,7 @@ void wallet2::parse_block_round(const cryptonote::blobdata &blob, cryptonote::bl bl_id = get_block_hash(bl); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const std::list &short_chain_history, std::list &blocks, std::vector &o_indices) +void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const std::list &short_chain_history, std::vector &blocks, std::vector &o_indices) { cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::request req = AUTO_VAL_INIT(req); cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); @@ -1700,7 +1700,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, o_indices = std::move(res.output_indices); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list &short_chain_history, std::list &hashes) +void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list &short_chain_history, std::vector &hashes) { cryptonote::COMMAND_RPC_GET_HASHES_FAST::request req = AUTO_VAL_INIT(req); cryptonote::COMMAND_RPC_GET_HASHES_FAST::response res = AUTO_VAL_INIT(res); @@ -1718,7 +1718,7 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, hashes = std::move(res.m_block_ids); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_blocks(uint64_t start_height, const std::list &blocks, const std::vector &o_indices, uint64_t& blocks_added) +void wallet2::process_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &o_indices, uint64_t& blocks_added) { size_t current_index = start_height; blocks_added = 0; @@ -1735,13 +1735,13 @@ void wallet2::process_blocks(uint64_t start_height, const std::list round_blocks(threads); std::deque error(threads); size_t blocks_size = blocks.size(); - std::list::const_iterator blocki = blocks.begin(); + std::vector::const_iterator blocki = blocks.begin(); for (size_t b = 0; b < blocks_size; b += threads) { size_t round_size = std::min((size_t)threads, blocks_size - b); tools::threadpool::waiter waiter; - std::list::const_iterator tmpblocki = blocki; + std::vector::const_iterator tmpblocki = blocki; for (size_t i = 0; i < round_size; ++i) { tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(tmpblocki->block), @@ -1833,7 +1833,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched) refresh(start_height, blocks_fetched, received_money); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::list &prev_blocks, std::list &blocks, std::vector &o_indices, bool &error) +void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::vector &prev_blocks, std::vector &blocks, std::vector &o_indices, bool &error) { error = false; @@ -1843,7 +1843,7 @@ void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_hei // prepend the last 3 blocks, should be enough to guard against a block or two's reorg cryptonote::block bl; - std::list::const_reverse_iterator i = prev_blocks.rbegin(); + std::vector::const_reverse_iterator i = prev_blocks.rbegin(); for (size_t n = 0; n < std::min((size_t)3, prev_blocks.size()); ++n) { bool ok = cryptonote::parse_and_validate_block_from_blob(i->block, bl); @@ -2110,7 +2110,7 @@ void wallet2::update_pool_state(bool refreshed) //---------------------------------------------------------------------------------------------------- void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list &short_chain_history) { - std::list hashes; + std::vector hashes; const uint64_t checkpoint_height = m_checkpoints.get_max_height(); if (stop_height > checkpoint_height && m_blockchain.size()-1 < checkpoint_height) @@ -2138,7 +2138,7 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, } if (hashes.size() + current_index < stop_height) { drop_from_short_history(short_chain_history, 3); - std::list::iterator right = hashes.end(); + std::vector::iterator right = hashes.end(); // prepend 3 more for (int i = 0; i<3; i++) { right--; @@ -2243,7 +2243,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; uint64_t blocks_start_height; - std::list blocks; + std::vector blocks; std::vector o_indices; bool refreshed = false; @@ -2276,7 +2276,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re { // pull the next set of blocks while we're processing the current one uint64_t next_blocks_start_height; - std::list next_blocks; + std::vector next_blocks; std::vector next_o_indices; bool error = false; if (blocks.empty()) diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index a10cef561..3a1f555d7 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1123,11 +1123,11 @@ namespace tools void get_short_chain_history(std::list& ids) const; bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; bool clear(); - void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::list &blocks, std::vector &o_indices); - void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::list &hashes); + void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::vector &blocks, std::vector &o_indices); + void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::vector &hashes); void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list &short_chain_history); - void pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::list &prev_blocks, std::list &blocks, std::vector &o_indices, bool &error); - void process_blocks(uint64_t start_height, const std::list &blocks, const std::vector &o_indices, uint64_t& blocks_added); + void pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::vector &prev_blocks, std::vector &blocks, std::vector &o_indices, bool &error); + void process_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &o_indices, uint64_t& blocks_added); uint64_t select_transfers(uint64_t needed_money, std::vector unused_transfers_indices, std::vector& selected_transfers, bool trusted_daemon) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); diff --git a/tests/core_proxy/core_proxy.cpp b/tests/core_proxy/core_proxy.cpp index f0a1eb5ce..17e552714 100644 --- a/tests/core_proxy/core_proxy.cpp +++ b/tests/core_proxy/core_proxy.cpp @@ -184,7 +184,7 @@ bool tests::proxy_core::handle_incoming_tx(const cryptonote::blobdata& tx_blob, return true; } -bool tests::proxy_core::handle_incoming_txs(const std::list& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) +bool tests::proxy_core::handle_incoming_txs(const std::vector& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { tvc.resize(tx_blobs.size()); size_t i = 0; diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index 8b7ac4291..7d36a0f68 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -76,7 +76,7 @@ namespace tests bool have_block(const crypto::hash& id); void get_blockchain_top(uint64_t& height, crypto::hash& top_id); bool handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); - bool handle_incoming_txs(const std::list& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_txs(const std::vector& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); bool handle_incoming_block(const cryptonote::blobdata& block_blob, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true); void pause_mine(){} void resume_mine(){} @@ -86,7 +86,7 @@ namespace tests cryptonote::Blockchain &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class proxy_core."); } bool get_test_drop_download() {return true;} bool get_test_drop_download_height() {return true;} - bool prepare_handle_incoming_blocks(const std::list &blocks) { return true; } + bool prepare_handle_incoming_blocks(const std::vector &blocks) { return true; } bool cleanup_handle_incoming_blocks(bool force_sync = false) { return true; } uint64_t get_target_blockchain_height() const { return 1; } size_t get_block_sync_size(uint64_t height) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; } @@ -94,8 +94,8 @@ namespace tests cryptonote::network_type get_nettype() const { return cryptonote::MAINNET; } bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; } bool pool_has_tx(const crypto::hash &txid) const { return false; } - bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const { return false; } - bool get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const { return false; } + bool get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks, std::vector& txs) const { return false; } + bool get_transactions(const std::vector& txs_ids, std::vector& txs, std::vector& missed_txs) const { return false; } bool get_block_by_hash(const crypto::hash &h, cryptonote::block &blk, bool *orphan = NULL) const { return false; } uint8_t get_ideal_hard_fork_version() const { return 0; } uint8_t get_ideal_hard_fork_version(uint64_t height) const { return 0; } @@ -103,6 +103,6 @@ namespace tests uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return 0; } cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; } bool fluffy_blocks_enabled() const { return false; } - uint64_t prevalidate_block_hashes(uint64_t height, const std::list &hashes) { return 0; } + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector &hashes) { return 0; } }; } diff --git a/tests/core_tests/chain_switch_1.cpp b/tests/core_tests/chain_switch_1.cpp index c5017a0df..18a813b19 100644 --- a/tests/core_tests/chain_switch_1.cpp +++ b/tests/core_tests/chain_switch_1.cpp @@ -128,7 +128,7 @@ bool gen_chain_switch_1::check_split_not_switched(cryptonote::core& c, size_t ev m_recipient_account_3 = boost::get(events[3]); m_recipient_account_4 = boost::get(events[4]); - std::list blocks; + std::vector blocks; bool r = c.get_blocks(0, 10000, blocks); CHECK_TEST_CONDITION(r); CHECK_EQ(5 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks.size()); @@ -145,7 +145,7 @@ bool gen_chain_switch_1::check_split_not_switched(cryptonote::core& c, size_t ev CHECK_EQ(MK_COINS(14), get_balance(m_recipient_account_3, chain, mtx)); CHECK_EQ(MK_COINS(3), get_balance(m_recipient_account_4, chain, mtx)); - std::list tx_pool; + std::vector tx_pool; r = c.get_pool_transactions(tx_pool); CHECK_TEST_CONDITION(r); CHECK_EQ(1, tx_pool.size()); @@ -166,7 +166,7 @@ bool gen_chain_switch_1::check_split_switched(cryptonote::core& c, size_t ev_ind { DEFINE_TESTS_ERROR_CONTEXT("gen_chain_switch_1::check_split_switched"); - std::list blocks; + std::vector blocks; bool r = c.get_blocks(0, 10000, blocks); CHECK_TEST_CONDITION(r); CHECK_EQ(6 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks.size()); @@ -175,7 +175,7 @@ bool gen_chain_switch_1::check_split_switched(cryptonote::core& c, size_t ev_ind CHECK_TEST_CONDITION(std::equal(blocks.begin(), it, m_chain_1.begin())); CHECK_TEST_CONDITION(blocks.back() == boost::get(events[24 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW])); // blk_7 - std::list alt_blocks; + std::vector alt_blocks; r = c.get_alternative_blocks(alt_blocks); CHECK_TEST_CONDITION(r); CHECK_EQ(2, c.get_alternative_blocks_count()); @@ -195,7 +195,7 @@ bool gen_chain_switch_1::check_split_switched(cryptonote::core& c, size_t ev_ind CHECK_EQ(MK_COINS(14), get_balance(m_recipient_account_3, chain, mtx)); CHECK_EQ(MK_COINS(16), get_balance(m_recipient_account_4, chain, mtx)); - std::list tx_pool; + std::vector tx_pool; r = c.get_pool_transactions(tx_pool); CHECK_TEST_CONDITION(r); CHECK_EQ(1, tx_pool.size()); diff --git a/tests/core_tests/chain_switch_1.h b/tests/core_tests/chain_switch_1.h index 5a035bf06..989b6df11 100644 --- a/tests/core_tests/chain_switch_1.h +++ b/tests/core_tests/chain_switch_1.h @@ -45,12 +45,12 @@ public: bool check_split_switched(cryptonote::core& c, size_t ev_index, const std::vector& events); private: - std::list m_chain_1; + std::vector m_chain_1; cryptonote::account_base m_recipient_account_1; cryptonote::account_base m_recipient_account_2; cryptonote::account_base m_recipient_account_3; cryptonote::account_base m_recipient_account_4; - std::list m_tx_pool; + std::vector m_tx_pool; }; diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index 6a723d56f..201da4fa0 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -481,7 +481,7 @@ inline bool do_replay_events(std::vector& events) MERROR("Failed to flush txpool"); return false; } - c.get_blockchain_storage().flush_txes_from_pool(std::list(pool_txs.begin(), pool_txs.end())); + c.get_blockchain_storage().flush_txes_from_pool(pool_txs); t_test_class validator; bool ret = replay_events_through_core(c, events, validator); diff --git a/tests/core_tests/chaingen001.cpp b/tests/core_tests/chaingen001.cpp index e84bfb924..a76cf1592 100644 --- a/tests/core_tests/chaingen001.cpp +++ b/tests/core_tests/chaingen001.cpp @@ -78,7 +78,7 @@ bool one_block::verify_1(cryptonote::core& c, size_t ev_index, const std::vector //CHECK_TEST_CONDITION(get_block_reward(0) == get_balance(alice, events, chain, mtx)); // check height - std::list blocks; + std::vector blocks; std::list outs; bool r = c.get_blocks(0, 100, blocks); //c.get_outs(100, outs); diff --git a/tests/core_tests/double_spend.cpp b/tests/core_tests/double_spend.cpp index 7ed62cf6d..c60ea885e 100644 --- a/tests/core_tests/double_spend.cpp +++ b/tests/core_tests/double_spend.cpp @@ -73,7 +73,7 @@ bool gen_double_spend_in_different_chains::check_double_spend(cryptonote::core& { DEFINE_TESTS_ERROR_CONTEXT("gen_double_spend_in_different_chains::check_double_spend"); - std::list block_list; + std::vector block_list; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, block_list); CHECK_TEST_CONDITION(r); diff --git a/tests/core_tests/double_spend.inl b/tests/core_tests/double_spend.inl index 0c58fb018..d02147065 100644 --- a/tests/core_tests/double_spend.inl +++ b/tests/core_tests/double_spend.inl @@ -64,7 +64,7 @@ bool gen_double_spend_base::check_block_verification_context(cons template bool gen_double_spend_base::mark_last_valid_block(cryptonote::core& c, size_t /*ev_index*/, const std::vector& /*events*/) { - std::list block_list; + std::vector block_list; bool r = c.get_blocks(c.get_current_blockchain_height() - 1, 1, block_list); CHECK_AND_ASSERT_MES(r, false, "core::get_blocks failed"); m_last_valid_block = block_list.back(); @@ -96,7 +96,7 @@ bool gen_double_spend_base::check_double_spend(cryptonote::core& } CHECK_NOT_EQ(invalid_index_value, m_invalid_block_index); - std::list block_list; + std::vector block_list; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, block_list); CHECK_TEST_CONDITION(r); CHECK_TEST_CONDITION(m_last_valid_block == block_list.back()); diff --git a/tests/core_tests/ring_signature_1.cpp b/tests/core_tests/ring_signature_1.cpp index 38eb1cf9b..8b2b943cc 100644 --- a/tests/core_tests/ring_signature_1.cpp +++ b/tests/core_tests/ring_signature_1.cpp @@ -101,7 +101,7 @@ bool gen_ring_signature_1::check_balances_1(cryptonote::core& c, size_t ev_index m_bob_account = boost::get(events[3]); m_alice_account = boost::get(events[4]); - std::list blocks; + std::vector blocks; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); @@ -119,7 +119,7 @@ bool gen_ring_signature_1::check_balances_2(cryptonote::core& c, size_t ev_index { DEFINE_TESTS_ERROR_CONTEXT("gen_ring_signature_1::check_balances_2"); - std::list blocks; + std::vector blocks; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); @@ -182,7 +182,7 @@ bool gen_ring_signature_2::check_balances_1(cryptonote::core& c, size_t ev_index m_bob_account = boost::get(events[1]); m_alice_account = boost::get(events[2]); - std::list blocks; + std::vector blocks; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); @@ -200,7 +200,7 @@ bool gen_ring_signature_2::check_balances_2(cryptonote::core& c, size_t ev_index { DEFINE_TESTS_ERROR_CONTEXT("gen_ring_signature_2::check_balances_2"); - std::list blocks; + std::vector blocks; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); @@ -292,7 +292,7 @@ bool gen_ring_signature_big::check_balances_1(cryptonote::core& c, size_t ev_ind m_bob_account = boost::get(events[1]); m_alice_account = boost::get(events[1 + m_test_size]); - std::list blocks; + std::vector blocks; bool r = c.get_blocks(0, 2 * m_test_size + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); @@ -317,7 +317,7 @@ bool gen_ring_signature_big::check_balances_2(cryptonote::core& c, size_t ev_ind { DEFINE_TESTS_ERROR_CONTEXT("gen_ring_signature_big::check_balances_2"); - std::list blocks; + std::vector blocks; bool r = c.get_blocks(0, 2 * m_test_size + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); diff --git a/tests/unit_tests/ban.cpp b/tests/unit_tests/ban.cpp index 15bc0bce3..e3dbdaef1 100644 --- a/tests/unit_tests/ban.cpp +++ b/tests/unit_tests/ban.cpp @@ -55,7 +55,7 @@ public: bool have_block(const crypto::hash& id) const {return true;} void get_blockchain_top(uint64_t& height, crypto::hash& top_id)const{height=0;top_id=crypto::null_hash;} bool handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } - bool handle_incoming_txs(const std::list& tx_blob, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } + bool handle_incoming_txs(const std::vector& tx_blob, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } bool handle_incoming_block(const cryptonote::blobdata& block_blob, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true) { return true; } void pause_mine(){} void resume_mine(){} @@ -65,7 +65,7 @@ public: cryptonote::blockchain_storage &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class test_core."); } bool get_test_drop_download() const {return true;} bool get_test_drop_download_height() const {return true;} - bool prepare_handle_incoming_blocks(const std::list &blocks) { return true; } + bool prepare_handle_incoming_blocks(const std::vector &blocks) { return true; } bool cleanup_handle_incoming_blocks(bool force_sync = false) { return true; } uint64_t get_target_blockchain_height() const { return 1; } size_t get_block_sync_size(uint64_t height) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; } @@ -73,8 +73,8 @@ public: cryptonote::network_type get_nettype() const { return cryptonote::MAINNET; } bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; } bool pool_has_tx(const crypto::hash &txid) const { return false; } - bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const { return false; } - bool get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const { return false; } + bool get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks, std::vector& txs) const { return false; } + bool get_transactions(const std::vector& txs_ids, std::vector& txs, std::vector& missed_txs) const { return false; } bool get_block_by_hash(const crypto::hash &h, cryptonote::block &blk, bool *orphan = NULL) const { return false; } uint8_t get_ideal_hard_fork_version() const { return 0; } uint8_t get_ideal_hard_fork_version(uint64_t height) const { return 0; } @@ -82,7 +82,7 @@ public: uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return 0; } cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; } bool fluffy_blocks_enabled() const { return false; } - uint64_t prevalidate_block_hashes(uint64_t height, const std::list &hashes) { return 0; } + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector &hashes) { return 0; } void stop() {} }; From a830db25775f7b708210e736154caf147ddf0ef3 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 16 Apr 2018 20:16:00 +0100 Subject: [PATCH 05/17] rpc: rework to avoid repeated calculations in get_blocks.bin --- src/cryptonote_core/blockchain.cpp | 21 ++++++++++---- src/cryptonote_core/blockchain.h | 2 +- src/cryptonote_core/cryptonote_core.cpp | 4 +-- src/cryptonote_core/cryptonote_core.h | 2 +- src/rpc/core_rpc_server.cpp | 37 ++++++++----------------- src/rpc/daemon_handler.cpp | 10 +++---- 6 files changed, 36 insertions(+), 40 deletions(-) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 7518cc4e1..59563380d 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2242,7 +2242,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc // find split point between ours and foreign blockchain (or start at // blockchain height ), and return up to max_count FULL // blocks by reference. -bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2272,15 +2272,24 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) { blocks.resize(blocks.size()+1); - blocks.back().first = m_db->get_block_blob_from_height(i); + blocks.back().first.first = m_db->get_block_blob_from_height(i); block b; - CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first, b), false, "internal error, invalid block"); + CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first.first, b), false, "internal error, invalid block"); + blocks.back().first.second = get_miner_tx_hash ? cryptonote::get_transaction_hash(b.miner_tx) : crypto::null_hash; std::vector mis; - get_transactions_blobs(b.tx_hashes, blocks.back().second, mis, pruned); + std::vector txs; + get_transactions_blobs(b.tx_hashes, txs, mis, pruned); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); - size += blocks.back().first.size(); - for (const auto &t: blocks.back().second) + size += blocks.back().first.first.size(); + for (const auto &t: txs) size += t.size(); + + CHECK_AND_ASSERT_MES(txs.size() == b.tx_hashes.size(), false, "mismatched sizes of b.tx_hashes and txs"); + blocks.back().second.reserve(txs.size()); + for (size_t i = 0; i < txs.size(); ++i) + { + blocks.back().second.push_back(std::make_pair(b.tx_hashes[i], std::move(txs[i]))); + } } m_db->block_txn_stop(); return true; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index bd46be0af..36d6b8609 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -420,7 +420,7 @@ namespace cryptonote * * @return true if a block found in common or req_start_block specified, else false */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; /** * @brief retrieves a set of blocks and their transactions, and possibly other transactions diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 8b24af43c..dc0b0d297 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1055,9 +1055,9 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const + bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const { - return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, max_count); + return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count); } //----------------------------------------------------------------------------------------------- bool core::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 443290c82..03000383e 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -516,7 +516,7 @@ namespace cryptonote * * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list&, std::vector > >&, uint64_t&, uint64_t&, size_t) const */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; /** * @brief gets some stats about the daemon diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index b921e1438..b55b1994b 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -226,9 +226,9 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary(invoke_http_mode::BIN, "/getblocks.bin", req, res, r)) return r; - std::vector > > bs; + std::vector, std::vector > > > bs; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = "Failed"; return false; @@ -240,46 +240,33 @@ namespace cryptonote for(auto& bd: bs) { res.blocks.resize(res.blocks.size()+1); - res.blocks.back().block = bd.first; - pruned_size += bd.first.size(); - unpruned_size += bd.first.size(); + res.blocks.back().block = bd.first.first; + pruned_size += bd.first.first.size(); + unpruned_size += bd.first.first.size(); res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - block b; - if (!parse_and_validate_block_from_blob(bd.first, b)) - { - res.status = "Invalid block"; - return false; - } if (!req.no_miner_tx) { - bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(b.miner_tx), res.output_indices.back().indices.back().indices); + bool r = m_core.get_tx_outputs_gindexs(bd.first.second, res.output_indices.back().indices.back().indices); if (!r) { res.status = "Failed"; return false; } } - if(b.tx_hashes.size() != bd.second.size()) - { - MERROR("block " << get_block_hash(b) << ": tx_hashes.size() " << b.tx_hashes.size() << ", bd.second.size() " << bd.second.size()); - res.status = "Failed"; - return false; - } - size_t txidx = 0; ntxes += bd.second.size(); res.blocks.back().txs.reserve(bd.second.size()); res.output_indices.back().indices.reserve(bd.second.size()); - for (std::vector::iterator i = bd.second.begin(); i != bd.second.end(); ++i) + for (std::vector>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { - unpruned_size += i->size(); - res.blocks.back().txs.push_back(std::move(*i)); - i->clear(); - i->shrink_to_fit(); + unpruned_size += i->second.size(); + res.blocks.back().txs.push_back(std::move(i->second)); + i->second.clear(); + i->second.shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - bool r = m_core.get_tx_outputs_gindexs(b.tx_hashes[txidx++], res.output_indices.back().indices.back().indices); + bool r = m_core.get_tx_outputs_gindexs(i->first, res.output_indices.back().indices.back().indices); if (!r) { res.status = "Failed"; diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 582a3440b..55858cc2a 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -50,9 +50,9 @@ namespace rpc void DaemonHandler::handle(const GetBlocksFast::Request& req, GetBlocksFast::Response& res) { - std::vector > > blocks; + std::vector, std::vector > > > blocks; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = Message::STATUS_FAILED; res.error_details = "core::find_blockchain_supplement() returned false"; @@ -69,7 +69,7 @@ namespace rpc { cryptonote::rpc::block_with_transactions& bwt = res.blocks[block_count]; - if (!parse_and_validate_block_from_blob(it->first, bwt.block)) + if (!parse_and_validate_block_from_blob(it->first.first, bwt.block)) { res.blocks.clear(); res.output_indices.clear(); @@ -87,10 +87,10 @@ namespace rpc return; } std::vector txs; - for (const auto& blob : it->second) + for (const auto& p : it->second) { txs.resize(txs.size() + 1); - if (!parse_and_validate_tx_from_blob(blob, txs.back())) + if (!parse_and_validate_tx_from_blob(p.second, txs.back())) { res.blocks.clear(); res.output_indices.clear(); From 57ea90285cb403416b9b59d0b62393fb2b6998a0 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 17 Apr 2018 11:44:17 +0100 Subject: [PATCH 06/17] wallet: don't process miner tx if we're refreshing with no-coinbase --- src/wallet/wallet2.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 56a2a2adf..61b488624 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1582,7 +1582,9 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry if(b.timestamp + 60*60*24 > m_account.get_createtime() && height >= m_refresh_from_block_height) { TIME_MEASURE_START(miner_tx_handle_time); - process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, o_indices.indices[txidx++].indices, height, b.timestamp, true, false, false); + if (m_refresh_type != RefreshNoCoinbase) + process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, o_indices.indices[txidx].indices, height, b.timestamp, true, false, false); + ++txidx; TIME_MEASURE_FINISH(miner_tx_handle_time); TIME_MEASURE_START(txs_handle_time); From 31a895e876bca91be52247f79b0e34b42d0d0ddc Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 17 Apr 2018 11:45:54 +0100 Subject: [PATCH 07/17] wallet2: simplify/speedup process_blocks --- src/wallet/wallet2.cpp | 83 +++++++++--------------------------------- 1 file changed, 17 insertions(+), 66 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 61b488624..9fdcde93f 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1729,76 +1729,30 @@ void wallet2::process_blocks(uint64_t start_height, const std::vector round_block_hashes(blocks.size()); + std::vector round_blocks(blocks.size()); + std::deque error(blocks.size()); + tools::threadpool& tpool = tools::threadpool::getInstance(); - int threads = tpool.get_max_concurrency(); - if (threads > 1) + tools::threadpool::waiter waiter; + for (size_t i = 0; i < blocks.size(); ++i) { - std::vector round_block_hashes(threads); - std::vector round_blocks(threads); - std::deque error(threads); - size_t blocks_size = blocks.size(); - std::vector::const_iterator blocki = blocks.begin(); - for (size_t b = 0; b < blocks_size; b += threads) - { - size_t round_size = std::min((size_t)threads, blocks_size - b); - tools::threadpool::waiter waiter; - - std::vector::const_iterator tmpblocki = blocki; - for (size_t i = 0; i < round_size; ++i) - { - tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(tmpblocki->block), - std::ref(round_blocks[i]), std::ref(round_block_hashes[i]), std::ref(error[i]))); - ++tmpblocki; - } - waiter.wait(); - tmpblocki = blocki; - for (size_t i = 0; i < round_size; ++i) - { - THROW_WALLET_EXCEPTION_IF(error[i], error::block_parse_error, tmpblocki->block); - ++tmpblocki; - } - for (size_t i = 0; i < round_size; ++i) - { - const crypto::hash &bl_id = round_block_hashes[i]; - cryptonote::block &bl = round_blocks[i]; - - if(current_index >= m_blockchain.size()) - { - process_new_blockchain_entry(bl, *blocki, bl_id, current_index, o_indices[b+i]); - ++blocks_added; - } - else if(bl_id != m_blockchain[current_index]) - { - //split detected here !!! - THROW_WALLET_EXCEPTION_IF(current_index == start_height, error::wallet_internal_error, - "wrong daemon response: split starts from the first block in response " + string_tools::pod_to_hex(bl_id) + - " (height " + std::to_string(start_height) + "), local block id at this height: " + - string_tools::pod_to_hex(m_blockchain[current_index])); - - detach_blockchain(current_index); - process_new_blockchain_entry(bl, *blocki, bl_id, current_index, o_indices[b+i]); - } - else - { - LOG_PRINT_L2("Block is already in blockchain: " << string_tools::pod_to_hex(bl_id)); - } - ++current_index; - ++blocki; - } - } + tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(blocks[i].block), + std::ref(round_blocks[i]), std::ref(round_block_hashes[i]), std::ref(error[i]))); } - else + waiter.wait(); + for (size_t i = 0; i < blocks.size(); ++i) { - for(auto& bl_entry: blocks) + THROW_WALLET_EXCEPTION_IF(error[i], error::block_parse_error, blocks[i].block); + } + for (size_t i = 0; i < blocks.size(); ++i) { - cryptonote::block bl; - bool r = cryptonote::parse_and_validate_block_from_blob(bl_entry.block, bl); - THROW_WALLET_EXCEPTION_IF(!r, error::block_parse_error, bl_entry.block); + const crypto::hash &bl_id = round_block_hashes[i]; + cryptonote::block &bl = round_blocks[i]; - crypto::hash bl_id = get_block_hash(bl); if(current_index >= m_blockchain.size()) { - process_new_blockchain_entry(bl, bl_entry, bl_id, current_index, o_indices[tx_o_indices_idx]); + process_new_blockchain_entry(bl, blocks[i], bl_id, current_index, o_indices[i]); ++blocks_added; } else if(bl_id != m_blockchain[current_index]) @@ -1810,16 +1764,13 @@ void wallet2::process_blocks(uint64_t start_height, const std::vector Date: Tue, 17 Apr 2018 18:16:19 +0100 Subject: [PATCH 08/17] wallet2: parse blocks in the RPC thread, not the processing thread Processing typically is the bottleneck --- src/wallet/wallet2.cpp | 73 ++++++++++++++++++++++++------------------ src/wallet/wallet2.h | 12 +++++-- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 9fdcde93f..57f75e3ec 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1720,39 +1720,23 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, hashes = std::move(res.m_block_ids); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &o_indices, uint64_t& blocks_added) +void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &parsed_blocks, uint64_t& blocks_added) { size_t current_index = start_height; blocks_added = 0; size_t tx_o_indices_idx = 0; - THROW_WALLET_EXCEPTION_IF(blocks.size() != o_indices.size(), error::wallet_internal_error, "size mismatch"); + THROW_WALLET_EXCEPTION_IF(blocks.size() != parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); THROW_WALLET_EXCEPTION_IF(!m_blockchain.is_in_bounds(current_index), error::wallet_internal_error, "Index out of bounds of hashchain"); - std::vector round_block_hashes(blocks.size()); - std::vector round_blocks(blocks.size()); - std::deque error(blocks.size()); - - tools::threadpool& tpool = tools::threadpool::getInstance(); - tools::threadpool::waiter waiter; for (size_t i = 0; i < blocks.size(); ++i) { - tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(blocks[i].block), - std::ref(round_blocks[i]), std::ref(round_block_hashes[i]), std::ref(error[i]))); - } - waiter.wait(); - for (size_t i = 0; i < blocks.size(); ++i) - { - THROW_WALLET_EXCEPTION_IF(error[i], error::block_parse_error, blocks[i].block); - } - for (size_t i = 0; i < blocks.size(); ++i) - { - const crypto::hash &bl_id = round_block_hashes[i]; - cryptonote::block &bl = round_blocks[i]; + const crypto::hash &bl_id = parsed_blocks[i].hash; + const cryptonote::block &bl = parsed_blocks[i].block; if(current_index >= m_blockchain.size()) { - process_new_blockchain_entry(bl, blocks[i], bl_id, current_index, o_indices[i]); + process_new_blockchain_entry(bl, blocks[i], bl_id, current_index, parsed_blocks[i].o_indices); ++blocks_added; } else if(bl_id != m_blockchain[current_index]) @@ -1764,7 +1748,7 @@ void wallet2::process_blocks(uint64_t start_height, const std::vector &short_chain_history, const std::vector &prev_blocks, std::vector &blocks, std::vector &o_indices, bool &error) +void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::vector &prev_blocks, std::vector &blocks, std::vector &parsed_blocks, bool &error) { error = false; @@ -1806,7 +1790,28 @@ void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_hei } // pull the new blocks + std::vector o_indices; pull_blocks(start_height, blocks_start_height, short_chain_history, blocks, o_indices); + THROW_WALLET_EXCEPTION_IF(blocks.size() != o_indices.size(), error::wallet_internal_error, "Mismatched sizes of blocks and o_indices"); + + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + parsed_blocks.resize(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) + { + tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(blocks[i].block), + std::ref(parsed_blocks[i].block), std::ref(parsed_blocks[i].hash), std::ref(parsed_blocks[i].error))); + } + waiter.wait(); + for (size_t i = 0; i < blocks.size(); ++i) + { + if (parsed_blocks[i].error) + { + error = true; + break; + } + parsed_blocks[i].o_indices = std::move(o_indices[i]); + } } catch(...) { @@ -2197,7 +2202,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re tools::threadpool::waiter waiter; uint64_t blocks_start_height; std::vector blocks; - std::vector o_indices; + std::vector parsed_blocks; bool refreshed = false; // pull the first set of blocks @@ -2218,11 +2223,11 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // If stop() is called during fast refresh we don't need to continue if(!m_run.load(std::memory_order_relaxed)) return; - pull_blocks(start_height, blocks_start_height, short_chain_history, blocks, o_indices); // always reset start_height to 0 to force short_chain_ history to be used on // subsequent pulls in this refresh. start_height = 0; + bool first = true; while(m_run.load(std::memory_order_relaxed)) { try @@ -2230,19 +2235,22 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // pull the next set of blocks while we're processing the current one uint64_t next_blocks_start_height; std::vector next_blocks; - std::vector next_o_indices; + std::vector next_parsed_blocks; bool error = false; - if (blocks.empty()) + if (!first && blocks.empty()) { refreshed = false; break; } - tpool.submit(&waiter, [&]{pull_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, next_blocks, next_o_indices, error);}); + tpool.submit(&waiter, [&]{pull_and_parse_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, next_blocks, next_parsed_blocks, error);}); - process_blocks(blocks_start_height, blocks, o_indices, added_blocks); - blocks_fetched += added_blocks; + if (!first) + { + process_parsed_blocks(blocks_start_height, blocks, parsed_blocks, added_blocks); + blocks_fetched += added_blocks; + } waiter.wait(); - if(blocks_start_height == next_blocks_start_height) + if(!first && blocks_start_height == next_blocks_start_height) { m_node_rpc_proxy.set_height(m_blockchain.size()); refreshed = true; @@ -2252,7 +2260,8 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // switch to the new blocks from the daemon blocks_start_height = next_blocks_start_height; blocks = std::move(next_blocks); - o_indices = next_o_indices; + parsed_blocks = std::move(next_parsed_blocks); + first = false; // handle error from async fetching thread if (error) diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 3a1f555d7..8f036a707 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -452,6 +452,14 @@ namespace tools typedef std::tuple get_outs_entry; + struct parsed_block + { + crypto::hash hash; + cryptonote::block block; + cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices o_indices; + bool error; + }; + /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -1126,8 +1134,8 @@ namespace tools void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::vector &blocks, std::vector &o_indices); void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::vector &hashes); void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list &short_chain_history); - void pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::vector &prev_blocks, std::vector &blocks, std::vector &o_indices, bool &error); - void process_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &o_indices, uint64_t& blocks_added); + void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::vector &prev_blocks, std::vector &blocks, std::vector &parsed_blocks, bool &error); + void process_parsed_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &parsed_blocks, uint64_t& blocks_added); uint64_t select_transfers(uint64_t needed_money, std::vector unused_transfers_indices, std::vector& selected_transfers, bool trusted_daemon) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); From f7f1917ed4708a73e54d9f983e6919df764beb06 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 18 Apr 2018 09:31:11 +0100 Subject: [PATCH 09/17] wallet2: remove useless device locking --- src/wallet/wallet2.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 57f75e3ec..e220cc058 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1093,9 +1093,8 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote //ensure device is let in NONE mode in any case hw::device &hwdev = m_account.get_device(); - boost::unique_lock hwdev_lock (hwdev); + boost::unique_lock hwdev_lock (hwdev, boost::defer_lock); hw::reset_mode rst(hwdev); - hwdev_lock.unlock(); // In this function, tx (probably) only contains the base information // (that is, the prunable stuff may or may not be included) From 41be339655018d17c3169d3add8d5ca81f960c6a Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 18 Apr 2018 09:33:46 +0100 Subject: [PATCH 10/17] wallet2: avoid duplicate parsing of tx extra --- src/wallet/wallet2.cpp | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index e220cc058..7ebc648bf 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1146,15 +1146,18 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote } // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses - std::vector additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(tx); + tx_extra_additional_pub_keys additional_tx_pub_keys; std::vector additional_derivations; - for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) + if (find_tx_extra_field_by_type(tx_extra_fields, additional_tx_pub_keys)) { - additional_derivations.push_back({}); - if (!hwdev.generate_key_derivation(additional_tx_pub_keys[i], keys.m_view_secret_key, additional_derivations.back())) + for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) { - MWARNING("Failed to generate key derivation from tx pubkey, skipping"); - additional_derivations.pop_back(); + additional_derivations.push_back({}); + if (!hwdev.generate_key_derivation(additional_tx_pub_keys.data[i], keys.m_view_secret_key, additional_derivations.back())) + { + MWARNING("Failed to generate key derivation from tx pubkey, skipping"); + additional_derivations.pop_back(); + } } } hwdev_lock.unlock(); @@ -1187,7 +1190,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); } } @@ -1210,7 +1213,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); } } @@ -1226,7 +1229,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote { hwdev_lock.lock(); hwdev.set_mode(hw::device::NONE); - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); hwdev_lock.unlock(); } From f025ae9760c4cc5c9438a6884b07975e9e3014b1 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 18 Apr 2018 10:28:35 +0100 Subject: [PATCH 11/17] wallet2: avoid re-parsing blocks when refreshing in a loop --- src/wallet/wallet2.cpp | 11 ++++------- src/wallet/wallet2.h | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 7ebc648bf..e61d9ca2a 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1772,7 +1772,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched) refresh(start_height, blocks_fetched, received_money); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::vector &prev_blocks, std::vector &blocks, std::vector &parsed_blocks, bool &error) +void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::vector &prev_blocks, const std::vector &prev_parsed_blocks, std::vector &blocks, std::vector &parsed_blocks, bool &error) { error = false; @@ -1781,13 +1781,10 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks drop_from_short_history(short_chain_history, 3); // prepend the last 3 blocks, should be enough to guard against a block or two's reorg - cryptonote::block bl; - std::vector::const_reverse_iterator i = prev_blocks.rbegin(); + std::vector::const_reverse_iterator i = prev_parsed_blocks.rbegin(); for (size_t n = 0; n < std::min((size_t)3, prev_blocks.size()); ++n) { - bool ok = cryptonote::parse_and_validate_block_from_blob(i->block, bl); - THROW_WALLET_EXCEPTION_IF(!ok, error::block_parse_error, i->block); - short_chain_history.push_front(cryptonote::get_block_hash(bl)); + short_chain_history.push_front(i->hash); ++i; } @@ -2244,7 +2241,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re refreshed = false; break; } - tpool.submit(&waiter, [&]{pull_and_parse_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, next_blocks, next_parsed_blocks, error);}); + tpool.submit(&waiter, [&]{pull_and_parse_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, parsed_blocks, next_blocks, next_parsed_blocks, error);}); if (!first) { diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 8f036a707..ee7049231 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1134,7 +1134,7 @@ namespace tools void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::vector &blocks, std::vector &o_indices); void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::vector &hashes); void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list &short_chain_history); - void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::vector &prev_blocks, std::vector &blocks, std::vector &parsed_blocks, bool &error); + void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::vector &prev_blocks, const std::vector &prev_parsed_blocks, std::vector &blocks, std::vector &parsed_blocks, bool &error); void process_parsed_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &parsed_blocks, uint64_t& blocks_added); uint64_t select_transfers(uint64_t needed_money, std::vector unused_transfers_indices, std::vector& selected_transfers, bool trusted_daemon) const; bool prepare_file_names(const std::string& file_path); From ef2cb6328788e14e8d593593ea9b2b716d72d6bd Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 18 Apr 2018 11:41:52 +0100 Subject: [PATCH 12/17] blockchain: simplify/speedup handle_get_objects --- src/cryptonote_core/blockchain.cpp | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 59563380d..208ea1efa 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1633,14 +1633,17 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO std::vector> blocks; get_blocks(arg.blocks, blocks, rsp.missed_ids); - for (const auto& bl: blocks) + for (auto& bl: blocks) { std::vector missed_tx_ids; std::vector txs; + rsp.blocks.push_back(block_complete_entry()); + block_complete_entry& e = rsp.blocks.back(); + // FIXME: s/rsp.missed_ids/missed_tx_id/ ? Seems like rsp.missed_ids // is for missed blocks, not missed transactions as well. - get_transactions_blobs(bl.second.tx_hashes, txs, missed_tx_ids); + get_transactions_blobs(bl.second.tx_hashes, e.txs, missed_tx_ids); if (missed_tx_ids.size() != 0) { @@ -1657,20 +1660,12 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO return false; } - rsp.blocks.push_back(block_complete_entry()); - block_complete_entry& e = rsp.blocks.back(); //pack block - e.block = bl.first; - //pack transactions - for (const cryptonote::blobdata& tx: txs) - e.txs.push_back(tx); + e.block = std::move(bl.first); } - //get another transactions, if need + //get and pack other transactions, if needed std::vector txs; - get_transactions_blobs(arg.txs, txs, rsp.missed_ids); - //pack aside transactions - for (const auto& tx: txs) - rsp.txs.push_back(tx); + get_transactions_blobs(arg.txs, rsp.txs, rsp.missed_ids); m_db->block_txn_stop(); return true; From dcfd299239cae50caad7a2f115489f86014b5114 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 21 Apr 2018 12:33:02 +0100 Subject: [PATCH 13/17] wallet2: speedup refresh key derivation and checking for incoming outputs are threaded in batch before adding blocks to the local blockchain. Other minor bits and bobs are also cached. --- src/wallet/wallet2.cpp | 295 +++++++++++++++++++++++++++++++++-------- src/wallet/wallet2.h | 22 ++- 2 files changed, 258 insertions(+), 59 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index e61d9ca2a..ff35ebd9e 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1035,6 +1035,23 @@ void wallet2::check_acc_out_precomp(const tx_out &o, const crypto::key_derivatio tx_scan_info.error = false; } //---------------------------------------------------------------------------------------------------- +void wallet2::check_acc_out_precomp(const tx_out &o, const crypto::key_derivation &derivation, const std::vector &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const +{ + if (!is_out_data || i >= is_out_data->received.size()) + return check_acc_out_precomp(o, derivation, additional_derivations, i, tx_scan_info); + + tx_scan_info.received = is_out_data->received[i]; + if(tx_scan_info.received) + { + tx_scan_info.money_transfered = o.amount; // may be 0 for ringct outputs + } + else + { + tx_scan_info.money_transfered = 0; + } + tx_scan_info.error = false; +} +//---------------------------------------------------------------------------------------------------- static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation &derivation, unsigned int i, rct::key & mask, hw::device &hwdev) { crypto::secret_key scalar1; @@ -1088,15 +1105,48 @@ void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::publi ++num_vouts_received; } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen) +void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const { - //ensure device is let in NONE mode in any case - hw::device &hwdev = m_account.get_device(); - - boost::unique_lock hwdev_lock (hwdev, boost::defer_lock); - hw::reset_mode rst(hwdev); + const cryptonote::account_keys& keys = m_account.get_keys(); - // In this function, tx (probably) only contains the base information + if(!parse_tx_extra(tx.extra, tx_cache_data.tx_extra_fields)) + { + // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key + LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + tx_cache_data.tx_extra_fields.clear(); + return; + } + + // Don't try to extract tx public key if tx has no ouputs + const bool is_miner = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!is_miner || m_refresh_type != RefreshType::RefreshNoCoinbase) + { + const size_t rec_size = is_miner && m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : tx.vout.size(); + if (!tx.vout.empty()) + { + // if tx.vout is not empty, we loop through all tx pubkeys + const std::vector> rec(rec_size, boost::none); + + tx_extra_pub_key pub_key_field; + size_t pk_index = 0; + while (find_tx_extra_field_by_type(tx_cache_data.tx_extra_fields, pub_key_field, pk_index++)) + tx_cache_data.primary.push_back({pub_key_field.pub_key, {}, rec}); + + // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses + tx_extra_additional_pub_keys additional_tx_pub_keys; + std::vector additional_derivations; + if (find_tx_extra_field_by_type(tx_cache_data.tx_extra_fields, additional_tx_pub_keys)) + { + for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) + tx_cache_data.additional.push_back({additional_tx_pub_keys.data[i], {}, {}}); + } + } + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data) +{ + // In this function, tx (probably) only contains the base information // (that is, the prunable stuff may or may not be included) if (!miner_tx && !pool) process_unconfirmed(txid, tx, height); @@ -1104,12 +1154,16 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote std::unordered_map tx_money_got_in_outs; // per receiving subaddress index crypto::public_key tx_pub_key = null_pkey; - std::vector tx_extra_fields; - if(!parse_tx_extra(tx.extra, tx_extra_fields)) + std::vector local_tx_extra_fields; + if (tx_cache_data.tx_extra_fields.empty()) { - // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key - LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + if(!parse_tx_extra(tx.extra, local_tx_extra_fields)) + { + // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key + LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + } } + const std::vector &tx_extra_fields = tx_cache_data.tx_extra_fields.empty() ? local_tx_extra_fields : tx_cache_data.tx_extra_fields; // Don't try to extract tx public key if tx has no ouputs size_t pk_index = 0; @@ -1128,6 +1182,11 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote m_callback->on_skip_transaction(height, txid, tx); break; } + if (!tx_cache_data.primary.empty()) + { + THROW_WALLET_EXCEPTION_IF(tx_cache_data.primary.size() < pk_index || pub_key_field.pub_key != tx_cache_data.primary[pk_index - 1].pkey, + error::wallet_internal_error, "tx_cache_data is out of sync"); + } int num_vouts_received = 0; tx_pub_key = pub_key_field.pub_key; @@ -1136,31 +1195,49 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote const cryptonote::account_keys& keys = m_account.get_keys(); crypto::key_derivation derivation; - hwdev_lock.lock(); - hwdev.set_mode(hw::device::TRANSACTION_PARSE); - if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) - { - MWARNING("Failed to generate key derivation from tx pubkey, skipping"); - static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); - memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); - } - - // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses - tx_extra_additional_pub_keys additional_tx_pub_keys; std::vector additional_derivations; - if (find_tx_extra_field_by_type(tx_extra_fields, additional_tx_pub_keys)) + tx_extra_additional_pub_keys additional_tx_pub_keys; + const wallet2::is_out_data *is_out_data_ptr = NULL; + if (tx_cache_data.primary.empty()) { - for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) + hw::device &hwdev = m_account.get_device(); + boost::unique_lock hwdev_lock (hwdev); + hw::reset_mode rst(hwdev); + + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) { - additional_derivations.push_back({}); - if (!hwdev.generate_key_derivation(additional_tx_pub_keys.data[i], keys.m_view_secret_key, additional_derivations.back())) + MWARNING("Failed to generate key derivation from tx pubkey in " << txid << ", skipping"); + static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); + } + + // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses + if (find_tx_extra_field_by_type(tx_extra_fields, additional_tx_pub_keys)) + { + for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) { - MWARNING("Failed to generate key derivation from tx pubkey, skipping"); - additional_derivations.pop_back(); + additional_derivations.push_back({}); + if (!hwdev.generate_key_derivation(additional_tx_pub_keys.data[i], keys.m_view_secret_key, additional_derivations.back())) + { + MWARNING("Failed to generate key derivation from additional tx pubkey in " << txid << ", skipping"); + memcpy(&additional_derivations.back(), rct::identity().bytes, sizeof(crypto::key_derivation)); + } } } } - hwdev_lock.unlock(); + else + { + THROW_WALLET_EXCEPTION_IF(pk_index - 1 >= tx_cache_data.primary.size(), + error::wallet_internal_error, "pk_index out of range of tx_cache_data"); + is_out_data_ptr = &tx_cache_data.primary[pk_index - 1]; + derivation = tx_cache_data.primary[pk_index - 1].derivation; + for (size_t n = 0; n < tx_cache_data.additional.size(); ++n) + { + additional_tx_pub_keys.data.push_back(tx_cache_data.additional[n].pkey); + additional_derivations.push_back(tx_cache_data.additional[n].derivation); + } + } if (miner_tx && m_refresh_type == RefreshNoCoinbase) { @@ -1168,7 +1245,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote } else if (miner_tx && m_refresh_type == RefreshOptimizeCoinbase) { - check_acc_out_precomp(tx.vout[0], derivation, additional_derivations, 0, tx_scan_info[0]); + check_acc_out_precomp(tx.vout[0], derivation, additional_derivations, 0, is_out_data_ptr, tx_scan_info[0]); THROW_WALLET_EXCEPTION_IF(tx_scan_info[0].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); // this assumes that the miner tx pays a single address @@ -1179,11 +1256,12 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote for (size_t i = 1; i < tx.vout.size(); ++i) { tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, - std::ref(tx_scan_info[i]))); + std::cref(is_out_data_ptr), std::ref(tx_scan_info[i]))); } waiter.wait(); // then scan all outputs from 0 - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); for (size_t i = 0; i < tx.vout.size(); ++i) { @@ -1194,19 +1272,19 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); } } - hwdev_lock.unlock(); } } - else if (tx.vout.size() > 1 && tools::threadpool::getInstance().get_max_concurrency() > 1) + else if (tx.vout.size() > 1 && tools::threadpool::getInstance().get_max_concurrency() > 1 && !is_out_data_ptr) { for (size_t i = 0; i < tx.vout.size(); ++i) { tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, - std::ref(tx_scan_info[i]))); + std::cref(is_out_data_ptr), std::ref(tx_scan_info[i]))); } waiter.wait(); - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); for (size_t i = 0; i < tx.vout.size(); ++i) { @@ -1217,21 +1295,20 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); } } - hwdev_lock.unlock(); } else { for (size_t i = 0; i < tx.vout.size(); ++i) { - check_acc_out_precomp(tx.vout[i], derivation, additional_derivations, i, tx_scan_info[i]); + check_acc_out_precomp(tx.vout[i], derivation, additional_derivations, i, is_out_data_ptr, tx_scan_info[i]); THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); - hwdev_lock.unlock(); } } } @@ -1571,12 +1648,11 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans add_rings(tx); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices) +void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector &tx_cache_data, size_t tx_cache_data_offset) { - size_t txidx = 0; - THROW_WALLET_EXCEPTION_IF(bche.txs.size() + 1 != o_indices.indices.size(), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(bche.txs.size() + 1 != parsed_block.o_indices.indices.size(), error::wallet_internal_error, "block transactions=" + std::to_string(bche.txs.size()) + - " not match with daemon response size=" + std::to_string(o_indices.indices.size())); + " not match with daemon response size=" + std::to_string(parsed_block.o_indices.indices.size())); //handle transactions from new block @@ -1585,20 +1661,16 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry { TIME_MEASURE_START(miner_tx_handle_time); if (m_refresh_type != RefreshNoCoinbase) - process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, o_indices.indices[txidx].indices, height, b.timestamp, true, false, false); - ++txidx; + process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, parsed_block.o_indices.indices[0].indices, height, b.timestamp, true, false, false, tx_cache_data[tx_cache_data_offset]); + ++tx_cache_data_offset; TIME_MEASURE_FINISH(miner_tx_handle_time); TIME_MEASURE_START(txs_handle_time); THROW_WALLET_EXCEPTION_IF(bche.txs.size() != b.tx_hashes.size(), error::wallet_internal_error, "Wrong amount of transactions for block"); - size_t idx = 0; - for (const auto& txblob: bche.txs) + THROW_WALLET_EXCEPTION_IF(bche.txs.size() != parsed_block.txes.size(), error::wallet_internal_error, "Wrong amount of transactions for block"); + for (size_t idx = 0; idx < b.tx_hashes.size(); ++idx) { - cryptonote::transaction tx; - bool r = parse_and_validate_tx_base_from_blob(txblob, tx); - THROW_WALLET_EXCEPTION_IF(!r, error::tx_parse_error, txblob); - process_new_transaction(b.tx_hashes[idx], tx, o_indices.indices[txidx++].indices, height, b.timestamp, false, false, false); - ++idx; + process_new_transaction(b.tx_hashes[idx], parsed_block.txes[idx], parsed_block.o_indices.indices[idx+1].indices, height, b.timestamp, false, false, false, tx_cache_data[tx_cache_data_offset++]); } TIME_MEASURE_FINISH(txs_handle_time); LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " << miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms"); @@ -1726,11 +1798,102 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector tx_cache_data; + for (size_t i = 0; i < blocks.size(); ++i) + num_txes += 1 + parsed_blocks[i].txes.size(); + tx_cache_data.resize(num_txes); + size_t txidx = 0; + for (size_t i = 0; i < blocks.size(); ++i) + { + THROW_WALLET_EXCEPTION_IF(parsed_blocks[i].txes.size() != parsed_blocks[i].block.tx_hashes.size(), + error::wallet_internal_error, "Mismatched parsed_blocks[i].txes.size() and parsed_blocks[i].block.tx_hashes.size()"); + if (m_refresh_type != RefreshNoCoinbase) + tpool.submit(&waiter, [&, i, txidx](){ cache_tx_data(parsed_blocks[i].block.miner_tx, get_transaction_hash(parsed_blocks[i].block.miner_tx), tx_cache_data[txidx]); }); + ++txidx; + for (size_t idx = 0; idx < parsed_blocks[i].txes.size(); ++idx) + { + tpool.submit(&waiter, [&, i, idx, txidx](){ cache_tx_data(parsed_blocks[i].txes[idx], parsed_blocks[i].block.tx_hashes[idx], tx_cache_data[txidx]); }); + ++txidx; + } + } + THROW_WALLET_EXCEPTION_IF(txidx != num_txes, error::wallet_internal_error, "txidx does not match tx_cache_data size"); + waiter.wait(); + + hw::device &hwdev = m_account.get_device(); + hw::reset_mode rst(hwdev); + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + const cryptonote::account_keys &keys = m_account.get_keys(); + + auto gender = [&](wallet2::is_out_data &iod) { + boost::unique_lock hwdev_lock(hwdev); + if (!hwdev.generate_key_derivation(iod.pkey, keys.m_view_secret_key, iod.derivation)) + { + MWARNING("Failed to generate key derivation from tx pubkey, skipping"); + static_assert(sizeof(iod.derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&iod.derivation, rct::identity().bytes, sizeof(iod.derivation)); + } + }; + + for (auto &slot: tx_cache_data) + { + for (auto &iod: slot.primary) + tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }); + for (auto &iod: slot.additional) + tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }); + } + waiter.wait(); + + auto geniod = [&](const cryptonote::transaction &tx, size_t n_vouts, size_t txidx) { + for (size_t k = 0; k < n_vouts; ++k) + { + const auto &o = tx.vout[k]; + if (o.target.type() == typeid(cryptonote::txout_to_key)) + { + std::vector additional_derivations; + for (const auto &iod: tx_cache_data[txidx].additional) + additional_derivations.push_back(iod.derivation); + const auto &key = boost::get(o.target).key; + for (size_t l = 0; l < tx_cache_data[txidx].primary.size(); ++l) + { + THROW_WALLET_EXCEPTION_IF(tx_cache_data[txidx].primary[l].received.size() != n_vouts, + error::wallet_internal_error, "Unexpected received array size"); + tx_cache_data[txidx].primary[l].received[k] = is_out_to_acc_precomp(m_subaddresses, key, tx_cache_data[txidx].primary[l].derivation, additional_derivations, k, hwdev); + additional_derivations.clear(); + } + } + } + }; + + txidx = 0; + for (size_t i = 0; i < blocks.size(); ++i) + { + if (m_refresh_type != RefreshType::RefreshNoCoinbase) + { + THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); + const size_t n_vouts = m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : parsed_blocks[i].block.miner_tx.vout.size(); + tpool.submit(&waiter, [&, i, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }); + } + ++txidx; + for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) + { + THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); + tpool.submit(&waiter, [&, i, j, txidx](){ geniod(parsed_blocks[i].txes[j], parsed_blocks[i].txes[j].vout.size(), txidx); }); + ++txidx; + } + } + THROW_WALLET_EXCEPTION_IF(txidx != tx_cache_data.size(), error::wallet_internal_error, "txidx did not reach expected value"); + waiter.wait(); + hwdev.set_mode(hw::device::NONE); + + size_t tx_cache_data_offset = 0; for (size_t i = 0; i < blocks.size(); ++i) { const crypto::hash &bl_id = parsed_blocks[i].hash; @@ -1738,7 +1901,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector= m_blockchain.size()) { - process_new_blockchain_entry(bl, blocks[i], bl_id, current_index, parsed_blocks[i].o_indices); + process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset); ++blocks_added; } else if(bl_id != m_blockchain[current_index]) @@ -1750,13 +1913,14 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector lock(error_lock); + error = true; + } + }); + } + } + waiter.wait(); } catch(...) { @@ -2023,7 +2204,7 @@ void wallet2::update_pool_state(bool refreshed) [tx_hash](const std::pair &e) { return e.first == tx_hash; }); if (i != txids.end()) { - process_new_transaction(tx_hash, tx, std::vector(), 0, time(NULL), false, true, tx_entry.double_spend_seen); + process_new_transaction(tx_hash, tx, std::vector(), 0, time(NULL), false, true, tx_entry.double_spend_seen, {}); m_scanned_pool_txs[0].insert(tx_hash); if (m_scanned_pool_txs[0].size() > 5000) { diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index ee7049231..522a2404a 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -456,10 +456,25 @@ namespace tools { crypto::hash hash; cryptonote::block block; + std::vector txes; cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices o_indices; bool error; }; + struct is_out_data + { + crypto::public_key pkey; + crypto::key_derivation derivation; + std::vector> received; + }; + + struct tx_cache_data + { + std::vector tx_extra_fields; + std::vector primary; + std::vector additional; + }; + /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -1125,8 +1140,8 @@ namespace tools * \param password Password of wallet file */ bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password); - void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen); - void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices); + void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data); + void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector &tx_cache_data, size_t tx_cache_data_offset); void detach_blockchain(uint64_t height); void get_short_chain_history(std::list& ids) const; bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; @@ -1146,6 +1161,7 @@ namespace tools bool generate_chacha_key_from_secret_keys(crypto::chacha_key &key) const; crypto::hash get_payment_id(const pending_tx &ptx) const; void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector &additional_derivations, size_t i, tx_scan_info_t &tx_scan_info) const; + void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const; void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const; uint64_t get_upper_transaction_size_limit() const; std::vector get_unspent_amounts_vector() const; @@ -1175,6 +1191,8 @@ namespace tools uint64_t get_segregation_fork_height() const; + void cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const; + cryptonote::account_base m_account; boost::optional m_daemon_login; std::string m_daemon_address; From 5d3e70229e0f12891817b81705652ecf5d49919a Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 24 Apr 2018 01:03:34 +0100 Subject: [PATCH 14/17] wallet2: remove old blockchain size cache hack --- src/wallet/wallet2.cpp | 16 ++-------------- src/wallet/wallet2.h | 3 +-- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index ff35ebd9e..f911b3a97 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -769,8 +769,6 @@ bool wallet2::init(std::string daemon_address, boost::optionalon_new_block(height, b); @@ -2258,7 +2255,6 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, while (missing_blocks-- > 0) m_blockchain.push_back(crypto::null_hash); // maybe a bit suboptimal, but deque won't do huge reallocs like vector m_blockchain.push_back(m_checkpoints.get_points().at(checkpoint_height)); - m_local_bc_height = m_blockchain.size(); short_chain_history.clear(); get_short_chain_history(short_chain_history); } @@ -2291,7 +2287,6 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, if (!(current_index % 1000)) LOG_PRINT_L2( "Skipped block by height: " << current_index); m_blockchain.push_back(bl_id); - ++m_local_bc_height; if (0 != m_callback) { // FIXME: this isn't right, but simplewallet just logs that we got a block. @@ -2352,7 +2347,6 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // Update lw heights m_light_wallet_scanned_block_height = res.scanned_block_height; m_light_wallet_blockchain_height = res.blockchain_height; - m_local_bc_height = res.blockchain_height; // If new height - call new_block callback if(m_light_wallet_blockchain_height != prev_height) { @@ -2607,7 +2601,6 @@ void wallet2::detach_blockchain(uint64_t height) size_t blocks_detached = m_blockchain.size() - height; m_blockchain.crop(height); - m_local_bc_height -= blocks_detached; for (auto it = m_payments.begin(); it != m_payments.end(); ) { @@ -2649,7 +2642,6 @@ bool wallet2::clear() m_scanned_pool_txs[0].clear(); m_scanned_pool_txs[1].clear(); m_address_book.clear(); - m_local_bc_height = 1; m_subaddresses.clear(); m_subaddress_labels.clear(); return true; @@ -4025,8 +4017,6 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass if (get_num_subaddress_accounts() == 0) add_subaddress_account(tr("Primary account")); - m_local_bc_height = m_blockchain.size(); - try { find_and_save_rings(false); @@ -4406,7 +4396,6 @@ void wallet2::rescan_blockchain(bool refresh) crypto::hash genesis_hash = get_block_hash(genesis); m_blockchain.push_back(genesis_hash); add_subaddress_account(tr("Primary account")); - m_local_bc_height = 1; if (refresh) this->refresh(); @@ -4422,7 +4411,7 @@ bool wallet2::is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) if(!is_tx_spendtime_unlocked(unlock_time, block_height)) return false; - if(block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > m_local_bc_height) + if(block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > get_blockchain_current_height()) return false; return true; @@ -4433,7 +4422,7 @@ bool wallet2::is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_heig if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { //interpret as block index - if(m_local_bc_height-1 + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) + if(get_blockchain_current_height()-1 + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) return true; else return false; @@ -10001,7 +9990,6 @@ void wallet2::import_blockchain(const std::tuple wallet2::export_outputs() const diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 522a2404a..b3c842288 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -745,7 +745,7 @@ namespace tools void get_unconfirmed_payments_out(std::list>& unconfirmed_payments, const boost::optional& subaddr_account = boost::none, const std::set& subaddr_indices = {}) const; void get_unconfirmed_payments(std::list>& unconfirmed_payments, const boost::optional& subaddr_account = boost::none, const std::set& subaddr_indices = {}) const; - uint64_t get_blockchain_current_height() const { return m_local_bc_height; } + uint64_t get_blockchain_current_height() const { return m_light_wallet_blockchain_height ? m_light_wallet_blockchain_height : m_blockchain.size(); } void rescan_spent(); void rescan_blockchain(bool refresh = true); bool is_transfer_unlocked(const transfer_details& td) const; @@ -1200,7 +1200,6 @@ namespace tools std::string m_keys_file; epee::net_utils::http::http_simple_client m_http_client; hashchain m_blockchain; - std::atomic m_local_bc_height; //temporary workaround std::unordered_map m_unconfirmed_txs; std::unordered_map m_confirmed_txs; std::unordered_multimap m_unconfirmed_payments; From 2704624eae34016642c940811e9bcfc18aa58a82 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 24 Apr 2018 01:04:35 +0100 Subject: [PATCH 15/17] wallet2: remove unneeded divisions --- src/wallet/wallet2.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index f911b3a97..4c0223eab 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1674,7 +1674,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " << miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms"); }else { - if (!(height % 100)) + if (!(height % 128)) LOG_PRINT_L2( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime()); } m_blockchain.push_back(bl_id); @@ -2284,7 +2284,7 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, { if(current_index >= m_blockchain.size()) { - if (!(current_index % 1000)) + if (!(current_index % 1024)) LOG_PRINT_L2( "Skipped block by height: " << current_index); m_blockchain.push_back(bl_id); From 2771a18e85bd91dd881f89f6c14d26cba35a1844 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 26 Apr 2018 11:44:47 +0100 Subject: [PATCH 16/17] threadpool: allow leaf functions to run concurrently Decrease the number of worker threads by one to account for the fact the calling thread acts as a worker thread now --- src/common/threadpool.cpp | 37 +++++++++++----- src/common/threadpool.h | 7 +-- src/cryptonote_core/blockchain.cpp | 12 +++--- src/cryptonote_core/cryptonote_core.cpp | 4 +- src/ringct/rctSigs.cpp | 12 +++--- src/wallet/wallet2.cpp | 34 +++++++-------- tests/unit_tests/threadpool.cpp | 57 ++++++++++++++++++++++--- 7 files changed, 112 insertions(+), 51 deletions(-) diff --git a/src/common/threadpool.cpp b/src/common/threadpool.cpp index fb238dca7..6b69e2a12 100644 --- a/src/common/threadpool.cpp +++ b/src/common/threadpool.cpp @@ -36,6 +36,7 @@ #include "common/util.h" static __thread int depth = 0; +static __thread bool is_leaf = false; namespace tools { @@ -43,9 +44,9 @@ threadpool::threadpool(unsigned int max_threads) : running(true), active(0) { boost::thread::attributes attrs; attrs.set_stack_size(THREAD_STACK_SIZE); max = max_threads ? max_threads : tools::get_max_concurrency(); - unsigned int i = max; + size_t i = max ? max - 1 : 0; while(i--) { - threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this))); + threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this, false))); } } @@ -60,20 +61,25 @@ threadpool::~threadpool() { } } -void threadpool::submit(waiter *obj, std::function f) { - entry e = {obj, f}; +void threadpool::submit(waiter *obj, std::function f, bool leaf) { + CHECK_AND_ASSERT_THROW_MES(!is_leaf, "A leaf routine is using a thread pool"); boost::unique_lock lock(mutex); - if ((active == max && !queue.empty()) || depth > 0) { + if (!leaf && ((active == max && !queue.empty()) || depth > 0)) { // if all available threads are already running // and there's work waiting, just run in current thread lock.unlock(); ++depth; + is_leaf = leaf; f(); --depth; + is_leaf = false; } else { if (obj) obj->inc(); - queue.push_back(e); + if (leaf) + queue.push_front({obj, f, leaf}); + else + queue.push_back({obj, f, leaf}); has_work.notify_one(); } } @@ -91,7 +97,7 @@ threadpool::waiter::~waiter() } try { - wait(); + wait(NULL); } catch (const std::exception &e) { @@ -99,9 +105,12 @@ threadpool::waiter::~waiter() } } -void threadpool::waiter::wait() { +void threadpool::waiter::wait(threadpool *tpool) { + if (tpool) + tpool->run(true); boost::unique_lock lock(mt); - while(num) cv.wait(lock); + while(num) + cv.wait(lock); } void threadpool::waiter::inc() { @@ -113,15 +122,19 @@ void threadpool::waiter::dec() { const boost::unique_lock lock(mt); num--; if (!num) - cv.notify_one(); + cv.notify_all(); } -void threadpool::run() { +void threadpool::run(bool flush) { boost::unique_lock lock(mutex); while (running) { entry e; while(queue.empty() && running) + { + if (flush) + return; has_work.wait(lock); + } if (!running) break; active++; @@ -129,8 +142,10 @@ void threadpool::run() { queue.pop_front(); lock.unlock(); ++depth; + is_leaf = e.leaf; e.f(); --depth; + is_leaf = false; if (e.wo) e.wo->dec(); diff --git a/src/common/threadpool.h b/src/common/threadpool.h index bf80a87f6..a43e38a76 100644 --- a/src/common/threadpool.h +++ b/src/common/threadpool.h @@ -59,7 +59,7 @@ public: public: void inc(); void dec(); - void wait(); //! Wait for a set of tasks to finish. + void wait(threadpool *tpool); //! Wait for a set of tasks to finish. waiter() : num(0){} ~waiter(); }; @@ -67,7 +67,7 @@ public: // Submit a task to the pool. The waiter pointer may be // NULL if the caller doesn't care to wait for the // task to finish. - void submit(waiter *waiter, std::function f); + void submit(waiter *waiter, std::function f, bool leaf = false); unsigned int get_max_concurrency() const; @@ -78,6 +78,7 @@ public: typedef struct entry { waiter *wo; std::function f; + bool leaf; } entry; std::deque queue; boost::condition_variable has_work; @@ -86,7 +87,7 @@ public: unsigned int active; unsigned int max; bool running; - void run(); + void run(bool flush = false); }; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 208ea1efa..854570eb5 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2818,7 +2818,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { // ND: Speedup // 1. Thread ring signature verification if possible. - tpool.submit(&waiter, boost::bind(&Blockchain::check_ring_signature, this, std::cref(tx_prefix_hash), std::cref(in_to_key.k_image), std::cref(pubkeys[sig_index]), std::cref(tx.signatures[sig_index]), std::ref(results[sig_index]))); + tpool.submit(&waiter, boost::bind(&Blockchain::check_ring_signature, this, std::cref(tx_prefix_hash), std::cref(in_to_key.k_image), std::cref(pubkeys[sig_index]), std::cref(tx.signatures[sig_index]), std::ref(results[sig_index])), true); } else { @@ -2842,7 +2842,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, sig_index++; } if (tx.version == 1 && threads > 1) - waiter.wait(); + waiter.wait(&tpool); if (tx.version == 1) { @@ -4128,11 +4128,11 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector hwdev_lock (hwdev); @@ -1277,9 +1277,9 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote for (size_t i = 0; i < tx.vout.size(); ++i) { tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, - std::cref(is_out_data_ptr), std::ref(tx_scan_info[i]))); + std::cref(is_out_data_ptr), std::ref(tx_scan_info[i])), true); } - waiter.wait(); + waiter.wait(&tpool); hw::device &hwdev = m_account.get_device(); boost::unique_lock hwdev_lock (hwdev); @@ -1822,7 +1822,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); const size_t n_vouts = m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : parsed_blocks[i].block.miner_tx.vout.size(); - tpool.submit(&waiter, [&, i, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }); + tpool.submit(&waiter, [&, i, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }, true); } ++txidx; for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) { THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); - tpool.submit(&waiter, [&, i, j, txidx](){ geniod(parsed_blocks[i].txes[j], parsed_blocks[i].txes[j].vout.size(), txidx); }); + tpool.submit(&waiter, [&, i, j, txidx](){ geniod(parsed_blocks[i].txes[j], parsed_blocks[i].txes[j].vout.size(), txidx); }, true); ++txidx; } } THROW_WALLET_EXCEPTION_IF(txidx != tx_cache_data.size(), error::wallet_internal_error, "txidx did not reach expected value"); - waiter.wait(); + waiter.wait(&tpool); hwdev.set_mode(hw::device::NONE); size_t tx_cache_data_offset = 0; @@ -1960,9 +1960,9 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks for (size_t i = 0; i < blocks.size(); ++i) { tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(blocks[i].block), - std::ref(parsed_blocks[i].block), std::ref(parsed_blocks[i].hash), std::ref(parsed_blocks[i].error))); + std::ref(parsed_blocks[i].block), std::ref(parsed_blocks[i].hash), std::ref(parsed_blocks[i].error)), true); } - waiter.wait(); + waiter.wait(&tpool); for (size_t i = 0; i < blocks.size(); ++i) { if (parsed_blocks[i].error) @@ -1985,10 +1985,10 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks boost::unique_lock lock(error_lock); error = true; } - }); + }, true); } } - waiter.wait(); + waiter.wait(&tpool); } catch(...) { @@ -2423,7 +2423,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re process_parsed_blocks(blocks_start_height, blocks, parsed_blocks, added_blocks); blocks_fetched += added_blocks; } - waiter.wait(); + waiter.wait(&tpool); if(!first && blocks_start_height == next_blocks_start_height) { m_node_rpc_proxy.set_height(m_blockchain.size()); @@ -2446,7 +2446,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re catch (const std::exception&) { blocks_fetched += added_blocks; - waiter.wait(); + waiter.wait(&tpool); if(try_count < 3) { LOG_PRINT_L1("Another try pull_blocks (try_count=" << try_count << ")..."); diff --git a/tests/unit_tests/threadpool.cpp b/tests/unit_tests/threadpool.cpp index 34be1417a..1307cd738 100644 --- a/tests/unit_tests/threadpool.cpp +++ b/tests/unit_tests/threadpool.cpp @@ -35,7 +35,7 @@ TEST(threadpool, wait_nothing) { std::shared_ptr tpool(tools::threadpool::getNewForUnitTests()); tools::threadpool::waiter waiter; - waiter.wait(); + waiter.wait(tpool.get()); } TEST(threadpool, wait_waits) @@ -45,7 +45,7 @@ TEST(threadpool, wait_waits) std::atomic b(false); tpool->submit(&waiter, [&b](){ epee::misc_utils::sleep_no_w(1000); b = true; }); ASSERT_FALSE(b); - waiter.wait(); + waiter.wait(tpool.get()); ASSERT_TRUE(b); } @@ -59,7 +59,7 @@ TEST(threadpool, one_thread) { tpool->submit(&waiter, [&counter](){++counter;}); } - waiter.wait(); + waiter.wait(tpool.get()); ASSERT_EQ(counter, 4096); } @@ -73,7 +73,7 @@ TEST(threadpool, many_threads) { tpool->submit(&waiter, [&counter](){++counter;}); } - waiter.wait(); + waiter.wait(tpool.get()); ASSERT_EQ(counter, 4096); } @@ -85,7 +85,7 @@ static uint64_t fibonacci(std::shared_ptr tpool, uint64_t n) tools::threadpool::waiter waiter; tpool->submit(&waiter, [&tpool, &f1, n](){ f1 = fibonacci(tpool, n-1); }); tpool->submit(&waiter, [&tpool, &f2, n](){ f2 = fibonacci(tpool, n-2); }); - waiter.wait(); + waiter.wait(tpool.get()); return f1 + f2; } @@ -95,7 +95,52 @@ TEST(threadpool, reentrency) tools::threadpool::waiter waiter; uint64_t f = fibonacci(tpool, 13); - waiter.wait(); + waiter.wait(tpool.get()); ASSERT_EQ(f, 233); } +TEST(threadpool, reentrancy) +{ + std::shared_ptr tpool(tools::threadpool::getNewForUnitTests(4)); + tools::threadpool::waiter waiter; + + uint64_t f = fibonacci(tpool, 13); + waiter.wait(tpool.get()); + ASSERT_EQ(f, 233); +} + +TEST(threadpool, leaf_throws) +{ + std::shared_ptr tpool(tools::threadpool::getNewForUnitTests()); + tools::threadpool::waiter waiter; + + bool thrown = false, executed = false; + tpool->submit(&waiter, [&](){ + try { tpool->submit(&waiter, [&](){ executed = true; }); } + catch(const std::exception &e) { thrown = true; } + }, true); + waiter.wait(tpool.get()); + ASSERT_TRUE(thrown); + ASSERT_FALSE(executed); +} + +TEST(threadpool, leaf_reentrancy) +{ + std::shared_ptr tpool(tools::threadpool::getNewForUnitTests(4)); + tools::threadpool::waiter waiter; + + std::atomic counter(0); + for (int i = 0; i < 1000; ++i) + { + tpool->submit(&waiter, [&](){ + tools::threadpool::waiter waiter; + for (int j = 0; j < 500; ++j) + { + tpool->submit(&waiter, [&](){ ++counter; }, true); + } + waiter.wait(tpool.get()); + }); + } + waiter.wait(tpool.get()); + ASSERT_EQ(counter, 500000); +} From 0e4c7d0fae398a46cdc223fc6186821b56e5a8e9 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 27 Apr 2018 11:08:24 +0100 Subject: [PATCH 17/17] wallet2: cache ringdb key while refreshing Speeds up syncing with a lot of outgoing outputs as key generation runs Cryptonight. --- src/wallet/wallet2.cpp | 56 +++++++++++++++++++++++++----------------- src/wallet/wallet2.h | 14 +++++++++++ 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 476bd4a13..a2798c203 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -119,6 +119,7 @@ using namespace cryptonote; static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; +std::atomic tools::wallet2::key_ref::refs(0); namespace { @@ -2334,6 +2335,8 @@ bool wallet2::delete_address_book_row(std::size_t row_id) { //---------------------------------------------------------------------------------------------------- void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) { + key_ref kref(*this); + if(m_light_wallet) { // MyMonero get_address_info needs to be called occasionally to trigger wallet sync. @@ -5757,6 +5760,24 @@ bool wallet2::set_ring_database(const std::string &filename) return true; } +crypto::chacha_key wallet2::get_ringdb_key() +{ + if (!m_ringdb_key) + { + MINFO("caching ringdb key"); + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + m_ringdb_key = key; + } + return *m_ringdb_key; +} + +void wallet2::clear_ringdb_key() +{ + MINFO("clearing ringdb key"); + m_ringdb_key = boost::none; +} + bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) { if (!m_ringdb) @@ -5767,9 +5788,8 @@ bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transac bool wallet2::add_rings(const cryptonote::transaction_prefix &tx) { - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - try { return add_rings(key, tx); } + key_ref kref(*this); + try { return add_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5777,9 +5797,8 @@ bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx) { if (!m_ringdb) return false; - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - try { return m_ringdb->remove_rings(key, tx); } + key_ref kref(*this); + try { return m_ringdb->remove_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5816,10 +5835,8 @@ bool wallet2::get_rings(const crypto::hash &txid, std::vector &outs) { - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - - try { return get_ring(key, key_image, outs); } + key_ref kref(*this); + try { return get_ring(get_ringdb_key(), key_image, outs); } catch (const std::exception &e) { return false; } } @@ -5828,10 +5845,8 @@ bool wallet2::set_ring(const crypto::key_image &key_image, const std::vectorset_ring(key, key_image, outs, relative); } + key_ref kref(*this); + try { return m_ringdb->set_ring(get_ringdb_key(), key_image, outs, relative); } catch (const std::exception &e) { return false; } } @@ -5842,6 +5857,7 @@ bool wallet2::find_and_save_rings(bool force) if (!m_ringdb) return false; + key_ref kref(*this); COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); @@ -5859,9 +5875,6 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Found " << std::to_string(txs_hashes.size()) << " transactions"); - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - // get those transactions from the daemon static const size_t SLICE_SIZE = 200; for (size_t slice = 0; slice < txs_hashes.size(); slice += SLICE_SIZE) @@ -5898,7 +5911,7 @@ bool wallet2::find_and_save_rings(bool force) crypto::hash tx_hash, tx_prefix_hash; THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch"); - THROW_WALLET_EXCEPTION_IF(!add_rings(key, tx), error::wallet_internal_error, "Failed to save ring"); + THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring"); } } @@ -6077,9 +6090,6 @@ void wallet2::get_outs(std::vector> return; } - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - if (fake_outputs_count > 0) { uint64_t segregation_fork_height = get_segregation_fork_height(); @@ -6257,7 +6267,7 @@ void wallet2::get_outs(std::vector> if (td.m_key_image_known && !td.m_key_image_partial) { std::vector ring; - if (get_ring(key, td.m_key_image, ring)) + if (get_ring(get_ringdb_key(), td.m_key_image, ring)) { MINFO("This output has a known ring, reusing (size " << ring.size() << ")"); THROW_WALLET_EXCEPTION_IF(ring.size() > fake_outputs_count + 1, error::wallet_internal_error, @@ -6449,7 +6459,7 @@ void wallet2::get_outs(std::vector> if (td.m_key_image_known && !td.m_key_image_partial) { std::vector ring; - if (get_ring(key, td.m_key_image, ring)) + if (get_ring(get_ringdb_key(), td.m_key_image, ring)) { for (uint64_t out: ring) { diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index b3c842288..09d99efd8 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -475,6 +475,16 @@ namespace tools std::vector additional; }; + struct key_ref + { + key_ref(tools::wallet2 &w): wallet(w) { ++refs; } + ~key_ref() { if (!--refs) wallet.clear_ringdb_key(); } + + private: + tools::wallet2 &wallet; + static std::atomic refs; + }; + /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -1186,6 +1196,9 @@ namespace tools bool add_rings(const cryptonote::transaction_prefix &tx); bool remove_rings(const cryptonote::transaction_prefix &tx); bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector &outs); + crypto::chacha_key get_ringdb_key(); + void cache_ringdb_key(); + void clear_ringdb_key(); bool get_output_distribution(uint64_t &start_height, std::vector &distribution); @@ -1282,6 +1295,7 @@ namespace tools std::string m_ring_database; bool m_ring_history_saved; std::unique_ptr m_ringdb; + boost::optional m_ringdb_key; }; } BOOST_CLASS_VERSION(tools::wallet2, 24)