From 769ae42a7b0e7e73a13ef578ec9a0ecb06df22c1 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 10 Sep 2018 13:08:34 +0000 Subject: [PATCH 1/3] wallet2: faster output and key image import/export --- src/wallet/wallet2.cpp | 94 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 18 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index c27e4e820..31246d0b1 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -69,6 +69,7 @@ using namespace epee; #include "common/base58.h" #include "common/dns_utils.h" #include "common/notify.h" +#include "common/perf_timer.h" #include "ringct/rctSigs.h" #include "ringdb.h" #include "device/device_cold.hpp" @@ -10517,11 +10518,13 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename) const { + PERF_TIMER(export_key_images); std::vector> ski = export_key_images(); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; std::string data; + data.reserve(ski.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); for (const auto &i: ski) @@ -10531,6 +10534,7 @@ bool wallet2::export_key_images(const std::string &filename) const } // encrypt data, keep magic plaintext + PERF_TIMER(export_key_images_encrypt); std::string ciphertext = encrypt_with_view_secret_key(data); return epee::file_io_utils::save_string_to_file(filename, magic + ciphertext); } @@ -10538,6 +10542,7 @@ bool wallet2::export_key_images(const std::string &filename) const //---------------------------------------------------------------------------------------------------- std::vector> wallet2::export_key_images() const { + PERF_TIMER(export_key_images_raw); std::vector> ski; ski.reserve(m_transfers.size()); @@ -10590,6 +10595,7 @@ std::vector> wallet2::export_key uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent) { + PERF_TIMER(import_key_images_fsu); std::string data; bool r = epee::file_io_utils::load_file_to_string(filename, data); @@ -10603,6 +10609,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent try { + PERF_TIMER(import_key_images_decrypt); data = decrypt_with_view_secret_key(std::string(data, magiclen)); } catch (const std::exception &e) @@ -10641,6 +10648,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent //---------------------------------------------------------------------------------------------------- uint64_t wallet2::import_key_images(const std::vector> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent) { + PERF_TIMER(import_key_images_lots); COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req); COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp); @@ -10654,6 +10662,9 @@ uint64_t wallet2::import_key_images(const std::vector(out.target); const crypto::public_key pkey = o.key; - std::vector pkeys; - pkeys.push_back(&pkey); - THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), - error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast(n) + "/" - + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); - - THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), - error::signature_check_failed, boost::lexical_cast(n) + "/" - + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) - + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + if (!td.m_key_image_known || !(key_image == td.m_key_image)) + { + std::vector pkeys; + pkeys.push_back(&pkey); + THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), + error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast(n) + "/" + + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); + THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), + error::signature_check_failed, boost::lexical_cast(n) + "/" + + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) + + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + } req.key_images.push_back(epee::string_tools::pod_to_hex(key_image)); } + PERF_TIMER_STOP(import_key_images_A); + PERF_TIMER_START(import_key_images_B); for (size_t n = 0; n < signed_key_images.size(); ++n) { m_transfers[n].m_key_image = signed_key_images[n].first; @@ -10688,9 +10703,11 @@ uint64_t wallet2::import_key_images(const std::vector spent_key_images; + PERF_TIMER_START(import_key_images_C); for (const transfer_details &td: m_transfers) { for (const cryptonote::txin_v& in : td.m_tx.vin) @@ -10721,10 +10739,12 @@ uint64_t wallet2::import_key_images(const std::vector(in).k_image, td.m_txid)); } } + PERF_TIMER_STOP(import_key_images_C); + PERF_TIMER_START(import_key_images_D); for(size_t i = 0; i < signed_key_images.size(); ++i) { - transfer_details &td = m_transfers[i]; + const transfer_details &td = m_transfers[i]; uint64_t amount = td.amount(); if (td.m_spent) spent += amount; @@ -10742,6 +10762,8 @@ uint64_t wallet2::import_key_images(const std::vectorsecond); } } + PERF_TIMER_STOP(import_key_images_D); + MDEBUG("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent"); if (check_spent) @@ -10751,8 +10773,12 @@ uint64_t wallet2::import_key_images(const std::vector wallet2::export_outputs() const { + PERF_TIMER(export_outputs); std::vector outs; outs.reserve(m_transfers.size()); @@ -10970,29 +11002,53 @@ std::vector wallet2::export_outputs() const //---------------------------------------------------------------------------------------------------- std::string wallet2::export_outputs_to_str() const { - std::vector outs = export_outputs(); + PERF_TIMER(export_outputs_to_str); std::stringstream oss; boost::archive::portable_binary_oarchive ar(oss); - ar << outs; + ar << m_transfers; std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; std::string header; header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); + PERF_TIMER(export_outputs_encryption); std::string ciphertext = encrypt_with_view_secret_key(header + oss.str()); return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- size_t wallet2::import_outputs(const std::vector &outputs) { - m_transfers.clear(); - m_transfers.reserve(outputs.size()); + PERF_TIMER(import_outputs); + const size_t original_size = m_transfers.size(); + m_transfers.resize(outputs.size()); for (size_t i = 0; i < outputs.size(); ++i) { transfer_details td = outputs[i]; + // skip those we've already imported, or which have different data + if (i < original_size) + { + // compare the data used to create the key image below + const transfer_details &org_td = m_transfers[i]; + if (!org_td.m_key_image_known) + goto process; +#define CMPF(f) if (!(td.f == org_td.f)) goto process + CMPF(m_txid); + CMPF(m_key_image); + CMPF(m_internal_output_index); +#undef CMPF + if (!(get_transaction_prefix_hash(td.m_tx) == get_transaction_prefix_hash(org_td.m_tx))) + goto process; + + // copy anyway, since the comparison does not include ancillary fields which may have changed + m_transfers[i] = std::move(td); + continue; + } + +process: + // the hot wallet wouldn't have known about key images (except if we already exported them) cryptonote::keypair in_ephemeral; @@ -11011,9 +11067,9 @@ size_t wallet2::import_outputs(const std::vector(i)); - m_key_images[td.m_key_image] = m_transfers.size(); - m_pub_keys[td.get_public_key()] = m_transfers.size(); - m_transfers.push_back(std::move(td)); + m_key_images[td.m_key_image] = i; + m_pub_keys[td.get_public_key()] = i; + m_transfers[i] = std::move(td); } return m_transfers.size(); @@ -11021,6 +11077,7 @@ size_t wallet2::import_outputs(const std::vector Date: Wed, 24 Oct 2018 18:24:11 +0000 Subject: [PATCH 2/3] wallet2: only export necessary outputs and key images and disable annoying test that requires ridiculous amounts of skullduggery every time some format changes --- src/device_trezor/device_trezor.cpp | 5 +- src/simplewallet/simplewallet.cpp | 4 +- src/wallet/wallet2.cpp | 122 ++++++++++++------- src/wallet/wallet2.h | 24 +++- src/wallet/wallet_rpc_server.cpp | 13 +- src/wallet/wallet_rpc_server_commands_defs.h | 6 +- tests/data/fuzz/cold-outputs/OUTPUTS2 | Bin 256 -> 581 bytes tests/fuzz/cold-outputs.cpp | 2 +- tests/unit_tests/serialization.cpp | 14 ++- 9 files changed, 125 insertions(+), 65 deletions(-) diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index 07c03fc66..f55cbb15d 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -212,9 +212,10 @@ namespace trezor { tools::wallet2::signed_tx_set & signed_tx, hw::tx_aux_data & aux_data) { + CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset"); size_t num_tx = unsigned_tx.txes.size(); signed_tx.key_images.clear(); - signed_tx.key_images.resize(unsigned_tx.transfers.size()); + signed_tx.key_images.resize(unsigned_tx.transfers.second.size()); for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) { std::shared_ptr signer; @@ -360,4 +361,4 @@ namespace trezor { } #endif //WITH_DEVICE_TREZOR -}} \ No newline at end of file +}} diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 58d4cdced..babc59ea3 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -6105,8 +6105,8 @@ bool simple_wallet::accept_loaded_tx(const std::function get_num_txes, bool simple_wallet::accept_loaded_tx(const tools::wallet2::unsigned_tx_set &txs) { std::string extra_message; - if (!txs.transfers.empty()) - extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.size()).str(); + if (!txs.transfers.second.empty()) + extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.second.size()).str(); return accept_loaded_tx([&txs](){return txs.txes.size();}, [&txs](size_t n)->const tools::wallet2::tx_construction_data&{return txs.txes[n];}, extra_message); } //---------------------------------------------------------------------------------------------------- diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 31246d0b1..a90a93321 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -113,11 +113,11 @@ using namespace cryptonote; #define SUBADDRESS_LOOKAHEAD_MAJOR 50 #define SUBADDRESS_LOOKAHEAD_MINOR 200 -#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\002" +#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\003" #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001" -#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\003" +#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\004" #define SEGREGATION_FORK_HEIGHT 99999999 #define TESTNET_SEGREGATION_FORK_HEIGHT 99999999 @@ -1606,6 +1606,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_txid = txid; td.m_key_image = tx_scan_info[o].ki; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_requested = false; td.m_key_image_partial = m_multisig; td.m_amount = amount; td.m_pk_index = pk_index - 1; @@ -5466,7 +5467,7 @@ std::string wallet2::dump_tx_to_str(const std::vector &ptx_vector) c txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); } - txs.transfers = m_transfers; + txs.transfers = export_outputs(); // save as binary std::ostringstream oss; boost::archive::portable_binary_oarchive ar(oss); @@ -7952,6 +7953,7 @@ void wallet2::light_wallet_get_unspent_outs() td.m_key_image = unspent_key_image; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_requested = false; td.m_key_image_partial = m_multisig; td.m_amount = o.amount; td.m_pk_index = 0; @@ -9125,7 +9127,7 @@ void wallet2::cold_sign_tx(const std::vector& ptx_vector, signed_tx_ { txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); } - txs.transfers = m_transfers; + txs.transfers = std::make_pair(0, m_transfers); auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); @@ -9156,7 +9158,7 @@ uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) { dev_cold->ki_sync(&wallet_shim, m_transfers, ski); - return import_key_images(ski, spent, unspent); + return import_key_images(ski, 0, spent, unspent); } //---------------------------------------------------------------------------------------------------- void wallet2::get_hard_fork_info(uint8_t version, uint64_t &earliest_height) const @@ -10519,15 +10521,21 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename) const { PERF_TIMER(export_key_images); - std::vector> ski = export_key_images(); + std::pair>> ski = export_key_images(); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; + const uint32_t offset = ski.first; std::string data; - data.reserve(ski.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); + data.reserve(4 + ski.second.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); + data.resize(4); + data[0] = offset & 0xff; + data[1] = (offset >> 8) & 0xff; + data[2] = (offset >> 16) & 0xff; + data[3] = (offset >> 24) & 0xff; data += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); - for (const auto &i: ski) + for (const auto &i: ski.second) { data += std::string((const char *)&i.first, sizeof(crypto::key_image)); data += std::string((const char *)&i.second, sizeof(crypto::signature)); @@ -10540,13 +10548,17 @@ bool wallet2::export_key_images(const std::string &filename) const } //---------------------------------------------------------------------------------------------------- -std::vector> wallet2::export_key_images() const +std::pair>> wallet2::export_key_images() const { PERF_TIMER(export_key_images_raw); std::vector> ski; - ski.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && !m_transfers[offset].m_key_image_requested) + ++offset; + + ski.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; @@ -10590,7 +10602,7 @@ std::vector> wallet2::export_key ski.push_back(std::make_pair(td.m_key_image, signature)); } - return ski; + return std::make_pair(offset, ski); } uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent) @@ -10617,15 +10629,17 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Failed to decrypt ") + filename + ": " + e.what()); } - const size_t headerlen = 2 * sizeof(crypto::public_key); + const size_t headerlen = 4 + 2 * sizeof(crypto::public_key); THROW_WALLET_EXCEPTION_IF(data.size() < headerlen, error::wallet_internal_error, std::string("Bad data size from file ") + filename); - const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[0]; - const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[sizeof(crypto::public_key)]; + const uint32_t offset = (uint8_t)data[0] | (((uint8_t)data[1]) << 8) | (((uint8_t)data[2]) << 16) | (((uint8_t)data[3]) << 24); + const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[4]; + const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[4 + sizeof(crypto::public_key)]; const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; if (public_spend_key != keys.m_spend_public_key || public_view_key != keys.m_view_public_key) { THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string( "Key images from ") + filename + " are for a different account"); } + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); const size_t record_size = sizeof(crypto::key_image) + sizeof(crypto::signature); THROW_WALLET_EXCEPTION_IF((data.size() - headerlen) % record_size, @@ -10642,20 +10656,21 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent ski.push_back(std::make_pair(key_image, signature)); } - return import_key_images(ski, spent, unspent); + return import_key_images(ski, offset, spent, unspent); } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::import_key_images(const std::vector> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent) +uint64_t wallet2::import_key_images(const std::vector> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent) { PERF_TIMER(import_key_images_lots); COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req); COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp); - THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size(), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); + THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size() - offset, error::wallet_internal_error, "The blockchain is out of date compared to the signed key images"); - if (signed_key_images.empty()) + if (signed_key_images.empty() && offset == 0) { spent = 0; unspent = 0; @@ -10667,7 +10682,7 @@ uint64_t wallet2::import_key_images(const std::vector pkeys; pkeys.push_back(&pkey); THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), - error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast(n) + "/" + error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast(n + offset) + "/" + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), - error::signature_check_failed, boost::lexical_cast(n) + "/" + error::signature_check_failed, boost::lexical_cast(n + offset) + "/" + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); } @@ -10698,10 +10713,11 @@ uint64_t wallet2::import_key_images(const std::vector key_images) td.m_key_image = key_images[i]; m_key_images[m_transfers[i].m_key_image] = i; td.m_key_image_known = true; + td.m_key_image_requested = false; td.m_key_image_partial = false; m_pub_keys[m_transfers[i].get_public_key()] = i; } @@ -10984,20 +11001,24 @@ void wallet2::import_blockchain(const std::tuple wallet2::export_outputs() const +std::pair> wallet2::export_outputs() const { PERF_TIMER(export_outputs); std::vector outs; - outs.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && m_transfers[offset].m_key_image_known) + ++offset; + + outs.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; outs.push_back(td); } - return outs; + return std::make_pair(offset, outs); } //---------------------------------------------------------------------------------------------------- std::string wallet2::export_outputs_to_str() const @@ -11006,7 +11027,7 @@ std::string wallet2::export_outputs_to_str() const std::stringstream oss; boost::archive::portable_binary_oarchive ar(oss); - ar << m_transfers; + ar << export_outputs(); std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; @@ -11018,20 +11039,27 @@ std::string wallet2::export_outputs_to_str() const return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- -size_t wallet2::import_outputs(const std::vector &outputs) +size_t wallet2::import_outputs(const std::pair> &outputs) { PERF_TIMER(import_outputs); + + THROW_WALLET_EXCEPTION_IF(outputs.first > m_transfers.size(), error::wallet_internal_error, + "Imported outputs omit more outputs that we know of"); + + const size_t offset = outputs.first; const size_t original_size = m_transfers.size(); - m_transfers.resize(outputs.size()); - for (size_t i = 0; i < outputs.size(); ++i) + m_transfers.resize(offset + outputs.second.size()); + for (size_t i = 0; i < offset; ++i) + m_transfers[i].m_key_image_requested = false; + for (size_t i = 0; i < outputs.second.size(); ++i) { - transfer_details td = outputs[i]; + transfer_details td = outputs.second[i]; // skip those we've already imported, or which have different data - if (i < original_size) + if (i + offset < original_size) { // compare the data used to create the key image below - const transfer_details &org_td = m_transfers[i]; + const transfer_details &org_td = m_transfers[i + offset]; if (!org_td.m_key_image_known) goto process; #define CMPF(f) if (!(td.f == org_td.f)) goto process @@ -11043,7 +11071,7 @@ size_t wallet2::import_outputs(const std::vector(i)); + THROW_WALLET_EXCEPTION_IF(td.m_tx.vout.empty(), error::wallet_internal_error, "tx with no outputs at index " + boost::lexical_cast(i + offset)); crypto::public_key tx_pub_key = get_tx_pub_key_from_received_outs(td); const std::vector additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); @@ -11063,13 +11091,14 @@ process: THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image"); expand_subaddresses(td.m_subaddr_index); td.m_key_image_known = true; + td.m_key_image_requested = true; td.m_key_image_partial = false; THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, - error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast(i)); + error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast(i + offset)); - m_key_images[td.m_key_image] = i; - m_pub_keys[td.get_public_key()] = i; - m_transfers[i] = std::move(td); + m_key_images[td.m_key_image] = i + offset; + m_pub_keys[td.get_public_key()] = i + offset; + m_transfers[i + offset] = std::move(td); } return m_transfers.size(); @@ -11114,7 +11143,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) std::string body(data, headerlen); std::stringstream iss; iss << body; - std::vector outputs; + std::pair> outputs; try { boost::archive::portable_binary_iarchive ar(iss); @@ -11306,6 +11335,7 @@ void wallet2::update_multisig_rescan_info(const std::vector txes; - wallet2::transfer_container transfers; + std::pair transfers; }; struct signed_tx_set @@ -1071,9 +1073,9 @@ namespace tools bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const; // Import/Export wallet data - std::vector export_outputs() const; + std::pair> export_outputs() const; std::string export_outputs_to_str() const; - size_t import_outputs(const std::vector &outputs); + size_t import_outputs(const std::pair> &outputs); size_t import_outputs_from_str(const std::string &outputs_st); payment_container export_payments() const; void import_payments(const payment_container &payments); @@ -1081,8 +1083,8 @@ namespace tools std::tuple> export_blockchain() const; void import_blockchain(const std::tuple> &bc); bool export_key_images(const std::string &filename) const; - std::vector> export_key_images() const; - uint64_t import_key_images(const std::vector> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent = true); + std::pair>> export_key_images() const; + uint64_t import_key_images(const std::vector> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); bool import_key_images(std::vector key_images); crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; @@ -1396,7 +1398,7 @@ namespace tools }; } BOOST_CLASS_VERSION(tools::wallet2, 26) -BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9) +BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 10) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) BOOST_CLASS_VERSION(tools::wallet2::multisig_tx_set, 1) @@ -1454,6 +1456,10 @@ namespace boost x.m_multisig_k.clear(); x.m_multisig_info.clear(); } + if (ver < 10) + { + x.m_key_image_requested = false; + } } template @@ -1535,6 +1541,12 @@ namespace boost a & x.m_multisig_info; a & x.m_multisig_k; a & x.m_key_image_partial; + if (ver < 10) + { + initialize_transfer_details(a, x, ver); + return; + } + a & x.m_key_image_requested; } template diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 5e6100dfd..eabdd9a6a 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -2460,12 +2460,13 @@ namespace tools if (!m_wallet) return not_open(er); try { - std::vector> ski = m_wallet->export_key_images(); - res.signed_key_images.resize(ski.size()); - for (size_t n = 0; n < ski.size(); ++n) + std::pair>> ski = m_wallet->export_key_images(); + res.offset = ski.first; + res.signed_key_images.resize(ski.second.size()); + for (size_t n = 0; n < ski.second.size(); ++n) { - res.signed_key_images[n].key_image = epee::string_tools::pod_to_hex(ski[n].first); - res.signed_key_images[n].signature = epee::string_tools::pod_to_hex(ski[n].second); + res.signed_key_images[n].key_image = epee::string_tools::pod_to_hex(ski.second[n].first); + res.signed_key_images[n].signature = epee::string_tools::pod_to_hex(ski.second[n].second); } } @@ -2518,7 +2519,7 @@ namespace tools ski[n].second = *reinterpret_cast(bd.data()); } uint64_t spent = 0, unspent = 0; - uint64_t height = m_wallet->import_key_images(ski, spent, unspent); + uint64_t height = m_wallet->import_key_images(ski, req.offset, spent, unspent); res.spent = spent; res.unspent = unspent; res.height = height; diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 924f3a0f1..026b75a9e 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 5 +#define WALLET_RPC_VERSION_MINOR 6 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -1579,9 +1579,11 @@ namespace wallet_rpc struct response { + uint32_t offset; std::vector signed_key_images; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(offset); KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; @@ -1602,9 +1604,11 @@ namespace wallet_rpc struct request { + uint32_t offset; std::vector signed_key_images; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(offset, (uint32_t)0); KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; diff --git a/tests/data/fuzz/cold-outputs/OUTPUTS2 b/tests/data/fuzz/cold-outputs/OUTPUTS2 index 907bcdb913d7c4cd244f1ac8a18fd812db5869d8..33cf39024f6bd7c087fd01a35a9568d8304b724a 100644 GIT binary patch literal 581 zcmV-L0=oT8Z*FCBZy;}VbZ~WaAZ2)PZ*p`5G4tp3cqZ(Q5}Lcq2LyX_vmER9<1fN# z;*B`&%2cAssqBhs7ekHk$|^si5j`U``G{LIVivfC8iZw}9~KF$kF4DmoU&PRJHU!& zS;G|gMb_d#19^;aZ+Y037ZNTANPHS+DoA>h23FQyh;r!auZUcZxi(X7heD@2@iW+w z0SK=Pl3EVpvuA=kZpg`Rk~l*!oHqxWwjuL{VWmn=m>JUwv!8>@pDE6H*@*#|ZM$R$-vX9~+^G5~+%jjzW;{_#A-5$@=t#1a* z!+531rB9Y8w7fu48`;L|1XuYsFVDEGhu}ZXn{+s{_{(T5YkX0RX|q2;LY0iQ*ohQX zGA3j4jzP8NWF*EDA*$Di2 literal 256 zcmZQzU|?WmWMpSxWM{4q;bUI*mI)-n#JoF%46;)#V-V~ikQu5!sS8sy! z>^zWVQ1<}UV+U~8F)%8WoR}1N$H(Wnqxq3V8G?7I^sZsxql`hnhN GWB>qDdP|}J diff --git a/tests/fuzz/cold-outputs.cpp b/tests/fuzz/cold-outputs.cpp index 488a3b931..29b3ed267 100644 --- a/tests/fuzz/cold-outputs.cpp +++ b/tests/fuzz/cold-outputs.cpp @@ -77,7 +77,7 @@ int ColdOutputsFuzzer::run(const std::string &filename) s = std::string("\x01\x16serialization::archive") + s; try { - std::vector outputs; + std::pair> outputs; std::stringstream iss; iss << s; boost::archive::portable_binary_iarchive ar(iss); diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 2f7b5aac7..e1404d637 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -908,9 +908,21 @@ TEST(Serialization, portability_outputs) ASSERT_TRUE(td2.m_pk_index == 0); } +struct unsigned_tx_set +{ + std::vector txes; + tools::wallet2::transfer_container transfers; +}; +template +inline void serialize(Archive &a, unsigned_tx_set &x, const boost::serialization::version_type ver) +{ + a & x.txes; + a & x.transfers; +} #define UNSIGNED_TX_PREFIX "Monero unsigned tx set\003" TEST(Serialization, portability_unsigned_tx) { + const boost::filesystem::path filename = unit_test::data_dir / "unsigned_monero_tx"; std::string s; const cryptonote::network_type nettype = cryptonote::TESTNET; @@ -918,7 +930,7 @@ TEST(Serialization, portability_unsigned_tx) ASSERT_TRUE(r); const size_t magiclen = strlen(UNSIGNED_TX_PREFIX); ASSERT_FALSE(strncmp(s.c_str(), UNSIGNED_TX_PREFIX, magiclen)); - tools::wallet2::unsigned_tx_set exported_txs; + unsigned_tx_set exported_txs; s = s.substr(magiclen); r = false; try From 5f614ba968388ae231579c392f8e793d569b9b3d Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 24 Oct 2018 13:23:44 +0000 Subject: [PATCH 3/3] simplewallet: print the number of show/all transfers --- src/simplewallet/simplewallet.cpp | 8 ++++++-- tests/unit_tests/serialization.cpp | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index babc59ea3..f54f3f129 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -4483,7 +4483,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector& args tools::wallet2::transfer_container transfers; m_wallet->get_transfers(transfers); - bool transfers_found = false; + size_t transfers_found = 0; for (const auto& td : transfers) { if (!filter || available != td.m_spent) @@ -4496,7 +4496,6 @@ bool simple_wallet::show_incoming_transfers(const std::vector& args if (verbose) verbose_string = (boost::format("%68s%68s") % tr("pubkey") % tr("key image")).str(); message_writer() << boost::format("%21s%8s%12s%8s%16s%68s%16s%s") % tr("amount") % tr("spent") % tr("unlocked") % tr("ringct") % tr("global index") % tr("tx id") % tr("addr index") % verbose_string; - transfers_found = true; } std::string verbose_string; if (verbose) @@ -4511,6 +4510,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector& args td.m_txid % td.m_subaddr_index.minor % verbose_string; + ++transfers_found; } } @@ -4529,6 +4529,10 @@ bool simple_wallet::show_incoming_transfers(const std::vector& args success_msg_writer() << tr("No incoming unavailable transfers"); } } + else + { + success_msg_writer() << boost::format("Found %u/%u transfers") % transfers_found % transfers.size(); + } return true; } diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index e1404d637..b4517af2f 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -922,7 +922,6 @@ inline void serialize(Archive &a, unsigned_tx_set &x, const boost::serialization #define UNSIGNED_TX_PREFIX "Monero unsigned tx set\003" TEST(Serialization, portability_unsigned_tx) { - const boost::filesystem::path filename = unit_test::data_dir / "unsigned_monero_tx"; std::string s; const cryptonote::network_type nettype = cryptonote::TESTNET;