From e69477bf25630c94a5aff666433f3da9ab29298c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 11 Nov 2018 13:39:00 +0000 Subject: [PATCH] db: speedup block addition by avoiding repeated (de)serialization --- src/blockchain_db/lmdb/db_lmdb.cpp | 27 ++++---- src/cryptonote_basic/cryptonote_basic.h | 25 +++++++- .../cryptonote_format_utils.cpp | 62 ++++++++++++------- .../cryptonote_format_utils.h | 6 +- 4 files changed, 83 insertions(+), 37 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 9d2f7821e..376170a6b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -865,21 +865,26 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons cryptonote::blobdata blob = tx_to_blob(tx); MDB_val_sized(blobval, blob); - std::stringstream ss; - binary_archive ba(ss); - bool r = const_cast(tx).serialize_base(ba); - if (!r) - throw0(DB_ERROR("Failed to serialize pruned tx")); - std::string pruned = ss.str(); - MDB_val_sized(pruned_blob, pruned); + unsigned int unprunable_size = tx.unprunable_size; + if (unprunable_size == 0) + { + std::stringstream ss; + binary_archive ba(ss); + bool r = const_cast(tx).serialize_base(ba); + if (!r) + throw0(DB_ERROR("Failed to serialize pruned tx")); + unprunable_size = ss.str().size(); + } + + if (unprunable_size > blob.size()) + throw0(DB_ERROR("pruned tx size is larger than tx size")); + + MDB_val pruned_blob = {unprunable_size, (void*)blob.data()}; result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str())); - if (pruned.size() > blob.size()) - throw0(DB_ERROR("pruned tx size is larger than tx size")); - cryptonote::blobdata prunable(blob.data() + pruned.size(), blob.size() - pruned.size()); - MDB_val_sized(prunable_blob, prunable); + MDB_val prunable_blob = {blob.size() - unprunable_size, (void*)(blob.data() + unprunable_size)}; result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str())); diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index c9c783a56..172a99e84 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -152,6 +152,10 @@ namespace cryptonote }; + template static inline unsigned int getpos(T &ar) { return 0; } + template<> inline unsigned int getpos(binary_archive &ar) { return ar.stream().tellp(); } + template<> inline unsigned int getpos(binary_archive &ar) { return ar.stream().tellg(); } + class transaction_prefix { @@ -203,9 +207,12 @@ namespace cryptonote bool pruned; + std::atomic unprunable_size; + std::atomic prefix_size; + transaction(); - transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } - transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; return *this; } + transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned), unprunable_size(t.unprunable_size.load()), prefix_size(t.prefix_size.load()) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } + transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; unprunable_size = t.unprunable_size.load(); prefix_size = t.prefix_size.load(); return *this; } virtual ~transaction(); void set_null(); void invalidate_hashes(); @@ -223,10 +230,18 @@ namespace cryptonote set_blob_size_valid(false); } + const unsigned int start_pos = getpos(ar); + FIELDS(*static_cast(this)) + if (std::is_same, binary_archive>()) + prefix_size = getpos(ar) - start_pos; + if (version == 1) { + if (std::is_same, binary_archive>()) + unprunable_size = getpos(ar) - start_pos; + ar.tag("signatures"); ar.begin_array(); PREPARE_CUSTOM_VECTOR_SERIALIZATION(vin.size(), signatures); @@ -265,6 +280,10 @@ namespace cryptonote bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); if (!r || !ar.stream().good()) return false; ar.end_object(); + + if (std::is_same, binary_archive>()) + unprunable_size = getpos(ar) - start_pos; + if (!pruned && rct_signatures.type != rct::RCTTypeNull) { ar.tag("rctsig_prunable"); @@ -329,6 +348,8 @@ namespace cryptonote set_hash_valid(false); set_blob_size_valid(false); pruned = false; + unprunable_size = 0; + prefix_size = 0; } inline diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 10fb5444c..85c5131f6 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -878,6 +878,11 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + void get_blob_hash(const epee::span& blob, crypto::hash& res) + { + cn_fast_hash(blob.data(), blob.size(), res); + } + //--------------------------------------------------------------- void get_blob_hash(const blobdata& blob, crypto::hash& res) { cn_fast_hash(blob.data(), blob.size(), res); @@ -946,6 +951,13 @@ namespace cryptonote return h; } //--------------------------------------------------------------- + crypto::hash get_blob_hash(const epee::span& blob) + { + crypto::hash h = null_hash; + get_blob_hash(blob, h); + return h; + } + //--------------------------------------------------------------- crypto::hash get_transaction_hash(const transaction& t) { crypto::hash h = null_hash; @@ -958,26 +970,35 @@ namespace cryptonote return get_transaction_hash(t, res, NULL); } //--------------------------------------------------------------- - bool calculate_transaction_prunable_hash(const transaction& t, crypto::hash& res) + bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob, crypto::hash& res) { if (t.version == 1) return false; - transaction &tt = const_cast(t); - std::stringstream ss; - binary_archive ba(ss); - const size_t inputs = t.vin.size(); - const size_t outputs = t.vout.size(); - const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get(t.vin[0]).key_offsets.size() - 1 : 0; - bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); - CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); - cryptonote::get_blob_hash(ss.str(), res); + const unsigned int unprunable_size = t.unprunable_size; + if (blob && unprunable_size) + { + CHECK_AND_ASSERT_MES(unprunable_size <= blob->size(), false, "Inconsistent transaction unprunable and blob sizes"); + cryptonote::get_blob_hash(epee::span(blob->data() + unprunable_size, blob->size() - unprunable_size), res); + } + else + { + transaction &tt = const_cast(t); + std::stringstream ss; + binary_archive ba(ss); + const size_t inputs = t.vin.size(); + const size_t outputs = t.vout.size(); + const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get(t.vin[0]).key_offsets.size() - 1 : 0; + bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); + CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); + cryptonote::get_blob_hash(ss.str(), res); + } return true; } //--------------------------------------------------------------- - crypto::hash get_transaction_prunable_hash(const transaction& t) + crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blobdata) { crypto::hash res; - CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, res), "Failed to calculate tx prunable hash"); + CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, blobdata, res), "Failed to calculate tx prunable hash"); return res; } //--------------------------------------------------------------- @@ -1030,16 +1051,13 @@ namespace cryptonote transaction &tt = const_cast(t); + const blobdata blob = tx_to_blob(t); + const unsigned int unprunable_size = t.unprunable_size; + const unsigned int prefix_size = t.prefix_size; + // base rct - { - std::stringstream ss; - binary_archive ba(ss); - const size_t inputs = t.vin.size(); - const size_t outputs = t.vout.size(); - bool r = tt.rct_signatures.serialize_rctsig_base(ba, inputs, outputs); - CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures base"); - cryptonote::get_blob_hash(ss.str(), hashes[1]); - } + CHECK_AND_ASSERT_MES(prefix_size <= unprunable_size && unprunable_size <= blob.size(), false, "Inconsistent transaction prefix, unprunable and blob sizes"); + cryptonote::get_blob_hash(epee::span(blob.data() + prefix_size, unprunable_size - prefix_size), hashes[1]); // prunable rct if (t.rct_signatures.type == rct::RCTTypeNull) @@ -1048,7 +1066,7 @@ namespace cryptonote } else { - CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, hashes[2]), false, "Failed to get tx prunable hash"); + CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, &blob, hashes[2]), false, "Failed to get tx prunable hash"); } // the tx hash is the hash of the 3 hashes diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 994978c10..4247e13da 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -98,15 +98,17 @@ namespace cryptonote bool generate_key_image_helper(const account_keys& ack, const std::unordered_map& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev); bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev); void get_blob_hash(const blobdata& blob, crypto::hash& res); + void get_blob_hash(const epee::span& blob, crypto::hash& res); crypto::hash get_blob_hash(const blobdata& blob); + crypto::hash get_blob_hash(const epee::span& blob); std::string short_hash_str(const crypto::hash& h); crypto::hash get_transaction_hash(const transaction& t); bool get_transaction_hash(const transaction& t, crypto::hash& res); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); - bool calculate_transaction_prunable_hash(const transaction& t, crypto::hash& res); - crypto::hash get_transaction_prunable_hash(const transaction& t); + bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob, crypto::hash& res); + crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob = NULL); bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); crypto::hash get_pruned_transaction_hash(const transaction& t, const crypto::hash &pruned_data_hash);