From 3bbc36614717a85c02763cf3deb761f039e9fa10 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 11 Nov 2018 14:51:03 +0000 Subject: [PATCH 1/6] Avoid repeated (de)serialization when syncing --- src/cryptonote_core/blockchain.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index f5bd9bbb5..d110cc139 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -3706,8 +3706,7 @@ leave: try { uint64_t long_term_block_weight = get_next_long_term_block_weight(block_weight); - cryptonote::blobdata bd = cryptonote::block_to_blob(bl); - new_height = m_db->add_block(std::make_pair(std::move(bl), std::move(bd)), block_weight, long_term_block_weight, cumulative_difficulty, already_generated_coins, txs); + new_height = m_db->add_block(std::make_pair(std::move(bl), cryptonote::block_to_blob(bl)), block_weight, long_term_block_weight, cumulative_difficulty, already_generated_coins, txs); } catch (const KEY_IMAGE_EXISTS& e) { From 8461df0405474b4ae54cfa13f4dedb046f9185f6 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 20 Nov 2018 20:19:39 +0000 Subject: [PATCH 2/6] save some database calls when getting top block hash and height --- src/cryptonote_core/blockchain.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index d110cc139..eb3c8d75b 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -3126,6 +3126,7 @@ bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const if (version >= HF_VERSION_DYNAMIC_FEE) { median = m_current_block_cumul_weight_limit / 2; + const uint64_t blockchain_height = m_db->height(); already_generated_coins = blockchain_height ? m_db->get_block_already_generated_coins(blockchain_height - 1) : 0; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) return false; From 11604b6da63d630ee3ba495d20f4ddb8e8807c83 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 21 Nov 2018 12:57:51 +0000 Subject: [PATCH 3/6] blockchain: avoid unneeded block copy --- src/cryptonote_core/blockchain.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index eb3c8d75b..c663d7cef 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -3707,7 +3707,8 @@ leave: try { uint64_t long_term_block_weight = get_next_long_term_block_weight(block_weight); - new_height = m_db->add_block(std::make_pair(std::move(bl), cryptonote::block_to_blob(bl)), block_weight, long_term_block_weight, cumulative_difficulty, already_generated_coins, txs); + cryptonote::blobdata bd = cryptonote::block_to_blob(bl); + new_height = m_db->add_block(std::make_pair(std::move(bl), std::move(bd)), block_weight, long_term_block_weight, cumulative_difficulty, already_generated_coins, txs); } catch (const KEY_IMAGE_EXISTS& e) { From 547a9708de91c175ceab0ddde93785ba93c9efca Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 13 Dec 2018 19:47:47 +0000 Subject: [PATCH 4/6] cryptonote: block parsing + hash calculation speedup This saves a duplicate serialization step --- .../cryptonote_format_utils.cpp | 30 +++++++++++++++++-- .../cryptonote_format_utils.h | 4 ++- src/cryptonote_core/blockchain.cpp | 10 ++++--- src/cryptonote_core/cryptonote_core.cpp | 3 +- .../cryptonote_protocol_handler.inl | 8 ++--- src/wallet/wallet2.cpp | 4 +-- 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 094057b1f..14500361c 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -1143,12 +1143,19 @@ namespace cryptonote return blob; } //--------------------------------------------------------------- - bool calculate_block_hash(const block& b, crypto::hash& res) + bool calculate_block_hash(const block& b, crypto::hash& res, const blobdata *blob) { + blobdata bd; + if (!blob) + { + bd = block_to_blob(b); + blob = &bd; + } + // EXCEPTION FOR BLOCK 202612 const std::string correct_blob_hash_202612 = "3a8a2b3a29b50fc86ff73dd087ea43c6f0d6b8f936c849194d5c84c737903966"; const std::string existing_block_id_202612 = "bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698"; - crypto::hash block_blob_hash = get_blob_hash(block_to_blob(b)); + crypto::hash block_blob_hash = get_blob_hash(*blob); if (string_tools::pod_to_hex(block_blob_hash) == correct_blob_hash_202612) { @@ -1239,7 +1246,7 @@ namespace cryptonote return p; } //--------------------------------------------------------------- - bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b) + bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash *block_hash) { std::stringstream ss; ss << b_blob; @@ -1248,9 +1255,26 @@ namespace cryptonote CHECK_AND_ASSERT_MES(r, false, "Failed to parse block from blob"); b.invalidate_hashes(); b.miner_tx.invalidate_hashes(); + if (block_hash) + { + calculate_block_hash(b, *block_hash, &b_blob); + ++block_hashes_calculated_count; + b.hash = *block_hash; + b.set_hash_valid(true); + } return true; } //--------------------------------------------------------------- + bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b) + { + return parse_and_validate_block_from_blob(b_blob, b, NULL); + } + //--------------------------------------------------------------- + bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash &block_hash) + { + return parse_and_validate_block_from_blob(b_blob, b, &block_hash); + } + //--------------------------------------------------------------- blobdata block_to_blob(const block& b) { return t_serializable_object_to_blob(b); diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 40a9907be..3f8eef076 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -114,12 +114,14 @@ namespace cryptonote crypto::hash get_pruned_transaction_hash(const transaction& t, const crypto::hash &pruned_data_hash); blobdata get_block_hashing_blob(const block& b); - bool calculate_block_hash(const block& b, crypto::hash& res); + bool calculate_block_hash(const block& b, crypto::hash& res, const blobdata *blob = NULL); bool get_block_hash(const block& b, crypto::hash& res); crypto::hash get_block_hash(const block& b); bool get_block_longhash(const block& b, crypto::hash& res, uint64_t height); crypto::hash get_block_longhash(const block& b, uint64_t height); + bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash *block_hash); bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b); + bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash &block_hash); bool get_inputs_money_amount(const transaction& tx, uint64_t& money); uint64_t get_outs_money_amount(const transaction& tx); bool check_inputs_types_supported(const transaction& tx); diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index c663d7cef..6862d3209 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -4283,8 +4283,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vectorblock, block)) + if (!parse_and_validate_block_from_blob(it->block, block, block_hash)) return false; // check first block and skip all blocks if its not chained properly @@ -4297,7 +4298,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vectorblock, block)) + if (!parse_and_validate_block_from_blob(it->block, block, block_hash)) return false; - if (have_block(get_block_hash(block))) + if (have_block(block_hash)) blocks_exist = true; std::advance(it, 1); diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 387203cc0..18b2368b1 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1432,7 +1432,8 @@ namespace cryptonote block lb; if (!b) { - if(!parse_and_validate_block_from_blob(block_blob, lb)) + crypto::hash block_hash; + if(!parse_and_validate_block_from_blob(block_blob, lb, block_hash)) { LOG_PRINT_L1("Failed to parse and validate new block"); bvc.m_verifivation_failed = true; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index b33867e8b..e1675e686 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -993,7 +993,8 @@ namespace cryptonote return 1; } - if(!parse_and_validate_block_from_blob(block_entry.block, b)) + crypto::hash block_hash; + if(!parse_and_validate_block_from_blob(block_entry.block, b, block_hash)) { LOG_ERROR_CCONTEXT("sent wrong block: failed to parse and validate block: " << epee::string_tools::buff_to_hex_nodelimer(block_entry.block) << ", dropping connection"); @@ -1012,7 +1013,6 @@ namespace cryptonote if (start_height == std::numeric_limits::max()) start_height = boost::get(b.miner_tx.vin[0]).height; - const crypto::hash block_hash = get_block_hash(b); auto req_it = context.m_requested_objects.find(block_hash); if(req_it == context.m_requested_objects.end()) { @@ -1119,13 +1119,13 @@ namespace cryptonote << ", we need " << previous_height); block new_block; - if (!parse_and_validate_block_from_blob(blocks.back().block, new_block)) + crypto::hash last_block_hash; + if (!parse_and_validate_block_from_blob(blocks.back().block, new_block, last_block_hash)) { MERROR(context << "Failed to parse block, but it should already have been parsed"); m_block_queue.remove_spans(span_connection_id, start_height); continue; } - const crypto::hash last_block_hash = cryptonote::get_block_hash(new_block); if (m_core.have_block(last_block_hash)) { const uint64_t subchain_height = start_height + blocks.size(); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 53388d659..1e7d8bf42 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2238,9 +2238,7 @@ void wallet2::get_short_chain_history(std::list& ids, uint64_t gra //---------------------------------------------------------------------------------------------------- void wallet2::parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const { - error = !cryptonote::parse_and_validate_block_from_blob(blob, bl); - if (!error) - bl_id = get_block_hash(bl); + error = !cryptonote::parse_and_validate_block_from_blob(blob, bl, bl_id); } //---------------------------------------------------------------------------------------------------- void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const std::list &short_chain_history, std::vector &blocks, std::vector &o_indices) From e40eb2ad9d18dd57fb369b2e161a9b0abf6340ca Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 13 Dec 2018 19:58:20 +0000 Subject: [PATCH 5/6] cryptonote_basic: speedup calculate_block_hash The block 202612 fix can be left tested at the end, if we already know we're not in the general case --- src/cryptonote_basic/cryptonote_format_utils.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 14500361c..410696cb4 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -1152,6 +1152,13 @@ namespace cryptonote blob = &bd; } + bool hash_result = get_object_hash(get_block_hashing_blob(b), res); + if (!hash_result) + return false; + + if (get_block_height(b) != 202612) + return true; + // EXCEPTION FOR BLOCK 202612 const std::string correct_blob_hash_202612 = "3a8a2b3a29b50fc86ff73dd087ea43c6f0d6b8f936c849194d5c84c737903966"; const std::string existing_block_id_202612 = "bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698"; @@ -1162,9 +1169,7 @@ namespace cryptonote string_tools::hex_to_pod(existing_block_id_202612, res); return true; } - bool hash_result = get_object_hash(get_block_hashing_blob(b), res); - if (hash_result) { // make sure that we aren't looking at a block with the 202612 block id but not the correct blobdata if (string_tools::pod_to_hex(res) == existing_block_id_202612) From c84ea2993faf94537adbe2cb187a9d4de8d98ef1 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 14 Dec 2018 00:22:34 +0000 Subject: [PATCH 6/6] cryptonote_basic: some more minor speedups --- .../cryptonote_format_utils.cpp | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 410696cb4..fecd67729 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -1090,7 +1090,14 @@ namespace cryptonote // we still need the size if (blob_size) - *blob_size = get_object_blobsize(t); + { + if (!t.is_blob_size_valid()) + { + t.blob_size = blob.size(); + t.set_blob_size_valid(true); + } + *blob_size = t.blob_size; + } return true; } @@ -1156,8 +1163,12 @@ namespace cryptonote if (!hash_result) return false; - if (get_block_height(b) != 202612) - return true; + if (b.miner_tx.vin.size() == 1 && b.miner_tx.vin[0].type() == typeid(cryptonote::txin_gen)) + { + const cryptonote::txin_gen &txin_gen = boost::get(b.miner_tx.vin[0]); + if (txin_gen.height != 202612) + return true; + } // EXCEPTION FOR BLOCK 202612 const std::string correct_blob_hash_202612 = "3a8a2b3a29b50fc86ff73dd087ea43c6f0d6b8f936c849194d5c84c737903966"; @@ -1212,9 +1223,9 @@ namespace cryptonote bool get_block_longhash(const block& b, crypto::hash& res, uint64_t height) { // block 202612 bug workaround - const std::string longhash_202612 = "84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000"; if (height == 202612) { + static const std::string longhash_202612 = "84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000"; string_tools::hex_to_pod(longhash_202612, res); return true; } @@ -1315,6 +1326,7 @@ namespace cryptonote crypto::hash get_tx_tree_hash(const block& b) { std::vector txs_ids; + txs_ids.reserve(1 + b.tx_hashes.size()); crypto::hash h = null_hash; size_t bl_sz = 0; get_transaction_hash(b.miner_tx, h, bl_sz);