From 7d413f635fb42f8cfe7567b07e981f3a312f8900 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 14 Sep 2016 20:23:06 +0100 Subject: [PATCH] rct: rework serialization to avoid storing vector sizes --- src/cryptonote_core/cryptonote_basic.h | 28 ++-- .../cryptonote_boost_serialization.h | 16 +- .../cryptonote_format_utils.cpp | 34 +++- src/ringct/rctSigs.cpp | 11 +- src/ringct/rctTypes.h | 155 ++++++++++++++---- tests/unit_tests/serialization.cpp | 7 +- 6 files changed, 191 insertions(+), 60 deletions(-) diff --git a/src/cryptonote_core/cryptonote_basic.h b/src/cryptonote_core/cryptonote_basic.h index f54b8c2b3..da069a21a 100644 --- a/src/cryptonote_core/cryptonote_basic.h +++ b/src/cryptonote_core/cryptonote_basic.h @@ -230,24 +230,22 @@ namespace cryptonote } else { - FIELD(rct_signatures) - switch (rct_signatures.type) + ar.tag("rct_signatures"); + if (!vin.empty()) { - case rct::RCTTypeNull: - break; - case rct::RCTTypeSimple: - if (rct_signatures.mixRing.size() && rct_signatures.mixRing.size() != vin.size()) - return false; - break; - case rct::RCTTypeFull: - for (size_t i = 0; i < rct_signatures.mixRing.size(); ++i) + ar.begin_object(); + bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); + if (!r || !ar.stream().good()) return false; + ar.end_object(); + if (rct_signatures.type != rct::RCTTypeNull) { - if (rct_signatures.mixRing[i].size() != vin.size()) - return false; + ar.tag("rctsig_prunable"); + ar.begin_object(); + r = rct_signatures.p.serialize_rctsig_prunable(ar, rct_signatures.type, vin.size(), vout.size(), + vin[0].type() == typeid(txin_to_key) ? boost::get(vin[0]).key_offsets.size() - 1 : 0); + if (!r || !ar.stream().good()) return false; + ar.end_object(); } - break; - default: - return false; } } END_SERIALIZE() diff --git a/src/cryptonote_core/cryptonote_boost_serialization.h b/src/cryptonote_core/cryptonote_boost_serialization.h index c91f78c58..19b1a687e 100644 --- a/src/cryptonote_core/cryptonote_boost_serialization.h +++ b/src/cryptonote_core/cryptonote_boost_serialization.h @@ -162,12 +162,17 @@ namespace boost a & x.vout; a & x.extra; if (x.version == 1) + { a & x.signatures; + } else - a & x.rct_signatures; + { + a & (rct::rctSigBase&)x.rct_signatures; + if (x.rct_signatures.type != rct::RCTTypeNull) + a & x.rct_signatures.p; + } } - template inline void serialize(Archive &a, cryptonote::block &b, const boost::serialization::version_type ver) { @@ -262,6 +267,13 @@ namespace boost a & x.txnFee; } + template + inline void serialize(Archive &a, rct::rctSigPrunable &x, const boost::serialization::version_type ver) + { + a & x.rangeSigs; + a & x.MGs; + } + template inline void serialize(Archive &a, rct::rctSig &x, const boost::serialization::version_type ver) { diff --git a/src/cryptonote_core/cryptonote_format_utils.cpp b/src/cryptonote_core/cryptonote_format_utils.cpp index 44f9380b0..9667006a3 100644 --- a/src/cryptonote_core/cryptonote_format_utils.cpp +++ b/src/cryptonote_core/cryptonote_format_utils.cpp @@ -475,7 +475,7 @@ namespace cryptonote tx.vin.clear(); tx.vout.clear(); tx.signatures.clear(); - tx.rct_signatures = rct::rctSig(); + tx.rct_signatures.type = rct::RCTTypeNull; amount_keys.clear(); tx.version = rct ? 2 : 1; @@ -948,11 +948,35 @@ namespace cryptonote // prefix get_transaction_prefix_hash(t, hashes[0]); - // base rct data - get_blob_hash(t_serializable_object_to_blob((const rct::rctSigBase&)t.rct_signatures), hashes[1]); + transaction &tt = const_cast(t); - // prunable rct data - get_blob_hash(t_serializable_object_to_blob(t.rct_signatures.p), hashes[2]); + // base rct + { + std::stringstream ss; + binary_archive ba(ss); + const size_t inputs = t.vin.size(); + const size_t outputs = t.vout.size(); + bool r = tt.rct_signatures.serialize_rctsig_base(ba, inputs, outputs); + CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures base"); + cryptonote::get_blob_hash(ss.str(), hashes[1]); + } + + // prunable rct + if (t.rct_signatures.type == rct::RCTTypeNull) + { + hashes[2] = cryptonote::null_hash; + } + else + { + std::stringstream ss; + binary_archive ba(ss); + const size_t inputs = t.vin.size(); + const size_t outputs = t.vout.size(); + const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get(t.vin[0]).key_offsets.size() - 1 : 0; + bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); + CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); + cryptonote::get_blob_hash(ss.str(), hashes[2]); + } // the tx hash is the hash of the 3 hashes res = cn_fast_hash(hashes, sizeof(hashes)); diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index f4dbd65c5..ed1f8cc0e 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -350,8 +350,16 @@ namespace rct { keyV hashes; hashes.push_back(rv.message); crypto::hash h; - cryptonote::get_blob_hash(cryptonote::t_serializable_object_to_blob((const rctSigBase&)rv), h); + + std::stringstream ss; + binary_archive ba(ss); + const size_t inputs = rv.pseudoOuts.size(); + const size_t outputs = rv.ecdhInfo.size(); + CHECK_AND_ASSERT_THROW_MES(const_cast(rv).serialize_rctsig_base(ba, inputs, outputs), + "Failed to serialize rctSigBase"); + cryptonote::get_blob_hash(ss.str(), h); hashes.push_back(hash2rct(h)); + keyV kv; for (auto r: rv.p.rangeSigs) { @@ -364,7 +372,6 @@ namespace rct { kv.push_back(r.Ci[n]); } hashes.push_back(cn_fast_hash(kv)); - return cn_fast_hash(hashes); } diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index f231d30fb..bfafebb83 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -188,49 +188,136 @@ namespace rct { ctkeyV outPk; xmr_amount txnFee; // contains b - BEGIN_SERIALIZE() - FIELD(type) - if (type == RCTTypeNull) - return true; - // FIELD(message) - not serialized, it can be reconstructed - // FIELD(mixRing) - not serialized, it can be reconstructed - if (type == RCTTypeSimple) - FIELD(pseudoOuts) - FIELD(ecdhInfo) - if (typename Archive::is_saving()) { - keyV outPk(this->outPk.size()); - for (size_t n = 0; n < outPk.size(); ++n) - outPk[n] = this->outPk[n].mask; - FIELD(outPk) + template class Archive> + bool serialize_rctsig_base(Archive &ar, size_t inputs, size_t outputs) + { + FIELD(type) + if (type == RCTTypeNull) + return true; + if (type != RCTTypeFull && type != RCTTypeSimple) + return false; + VARINT_FIELD(txnFee) + // inputs/outputs not saved, only here for serialization help + // FIELD(message) - not serialized, it can be reconstructed + // FIELD(mixRing) - not serialized, it can be reconstructed + if (type == RCTTypeSimple) + { + ar.tag("pseudoOuts"); + ar.begin_array(); + PREPARE_CUSTOM_VECTOR_SERIALIZATION(inputs, pseudoOuts); + if (pseudoOuts.size() != inputs) + return false; + for (size_t i = 0; i < inputs; ++i) + { + FIELDS(pseudoOuts[i]) + if (inputs - i > 1) + ar.delimit_array(); } - else { - keyV outPk; - FIELD(outPk) - this->outPk.resize(outPk.size()); - for (size_t n = 0; n < outPk.size(); ++n) - this->outPk[n].mask = outPk[n]; - } - VARINT_FIELD(txnFee) - END_SERIALIZE() + ar.end_array(); + } + + ar.tag("ecdhInfo"); + ar.begin_array(); + PREPARE_CUSTOM_VECTOR_SERIALIZATION(outputs, ecdhInfo); + if (ecdhInfo.size() != outputs) + return false; + for (size_t i = 0; i < outputs; ++i) + { + FIELDS(ecdhInfo[i]) + if (outputs - i > 1) + ar.delimit_array(); + } + ar.end_array(); + + ar.tag("outPk"); + ar.begin_array(); + PREPARE_CUSTOM_VECTOR_SERIALIZATION(outputs, outPk); + if (outPk.size() != outputs) + return false; + for (size_t i = 0; i < outputs; ++i) + { + FIELDS(outPk[i].mask) + if (outputs - i > 1) + ar.delimit_array(); + } + ar.end_array(); + return true; + } }; struct rctSigPrunable { vector rangeSigs; vector MGs; // simple rct has N, full has 1 - BEGIN_SERIALIZE() - FIELD(rangeSigs) - FIELD(MGs) - END_SERIALIZE() + template class Archive> + bool serialize_rctsig_prunable(Archive &ar, uint8_t type, size_t inputs, size_t outputs, size_t mixin) + { + if (type == RCTTypeNull) + return true; + if (type != RCTTypeFull && type != RCTTypeSimple) + return false; + ar.tag("rangeSigs"); + ar.begin_array(); + PREPARE_CUSTOM_VECTOR_SERIALIZATION(outputs, rangeSigs); + if (rangeSigs.size() != outputs) + return false; + for (size_t i = 0; i < outputs; ++i) + { + FIELDS(rangeSigs[i]) + if (outputs - i > 1) + ar.delimit_array(); + } + ar.end_array(); + + ar.tag("MGs"); + ar.begin_array(); + // we keep a byte for size of MGs, because we don't know whether this is + // a simple or full rct signature, and it's starting to annoy the hell out of me + size_t mg_elements = type == RCTTypeSimple ? inputs : 1; + PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_elements, MGs); + if (MGs.size() != mg_elements) + return false; + for (size_t i = 0; i < mg_elements; ++i) + { + // we save the MGs contents directly, because we want it to save its + // arrays and matrices without the size prefixes, and the load can't + // know what size to expect if it's not in the data + ar.tag("ss"); + ar.begin_array(); + PREPARE_CUSTOM_VECTOR_SERIALIZATION(mixin + 1, MGs[i].ss); + if (MGs[i].ss.size() != mixin + 1) + return false; + for (size_t j = 0; j < mixin + 1; ++j) + { + ar.begin_array(); + size_t mg_ss2_elements = (type == RCTTypeSimple ? 1 : inputs) + 1; + PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_ss2_elements, MGs[i].ss[j]); + if (MGs[i].ss[j].size() != mg_ss2_elements) + return false; + for (size_t k = 0; k < mg_ss2_elements; ++k) + { + FIELDS(MGs[i].ss[j][k]) + if (mg_ss2_elements - j > 1) + ar.delimit_array(); + } + ar.end_array(); + + if (mixin + 1 - j > 1) + ar.delimit_array(); + } + ar.end_array(); + + FIELDS(MGs[i].cc) + // MGs[i].II not saved, it can be reconstructed + if (mg_elements - i > 1) + ar.delimit_array(); + } + ar.end_array(); + return true; + } + }; struct rctSig: public rctSigBase { rctSigPrunable p; - - BEGIN_SERIALIZE_OBJECT() - FIELDS(*static_cast(this)) - if (type == RCTTypeNull) - return true; - FIELDS(p); - END_SERIALIZE() }; //other basepoint H = toPoint(cn_fast_hash(G)), G the basepoint diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 40209f6ed..f51f9ec67 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -589,6 +589,7 @@ TEST(Serialization, serializes_ringct_types) ASSERT_TRUE(serialization::parse_binary(blob, rg1)); ASSERT_TRUE(!memcmp(&rg0, &rg1, sizeof(rg0))); +#if 0 ASSERT_TRUE(serialization::dump_binary(s0, blob)); ASSERT_TRUE(serialization::parse_binary(blob, s1)); ASSERT_TRUE(s0.type == s1.type); @@ -621,16 +622,18 @@ TEST(Serialization, serializes_ringct_types) // serialization only does the mask ASSERT_TRUE(!memcmp(&s0.outPk[n].mask, &s1.outPk[n].mask, sizeof(s0.outPk[n].mask))); } +#endif tx0.set_null(); tx0.version = 2; cryptonote::txin_to_key txin_to_key1; - txin_to_key1.key_offsets.resize(2); + txin_to_key1.key_offsets.resize(4); cryptonote::txin_to_key txin_to_key2; - txin_to_key2.key_offsets.resize(2); + txin_to_key2.key_offsets.resize(4); tx0.vin.push_back(txin_to_key1); tx0.vin.push_back(txin_to_key2); tx0.vout.push_back(cryptonote::tx_out()); + tx0.vout.push_back(cryptonote::tx_out()); tx0.rct_signatures = s0; ASSERT_EQ(tx0.rct_signatures.p.rangeSigs.size(), 2); ASSERT_TRUE(serialization::dump_binary(tx0, blob));