From e44e8b164030cfbddb9ae8b39251c1ccf396e2b6 Mon Sep 17 00:00:00 2001 From: j-berman Date: Thu, 13 Oct 2022 18:33:33 -0700 Subject: [PATCH] wallet: background sync with just the view key - When background syncing, the wallet wipes the spend key from memory and processes all new transactions. The wallet saves all receives, spends, and "plausible" spends of receives the wallet does not know key images for. - When background sync disabled, the wallet processes all background synced txs and then clears the background sync cache. - Adding "plausible" spends to the background sync cache ensures that the wallet does not need to query the daemon to see if any received outputs were spent while background sync was enabled. This would harm privacy especially for users of 3rd party daemons. - To enable the feature in the CLI wallet, the user can set background-sync to reuse-wallet-password or custom-background-password and the wallet automatically syncs in the background when the wallet locks, then processes all background synced txs when the wallet is unlocked. - The custom-background-password option enables the user to open a distinct background wallet that only has a view key saved and can be opened/closed/synced separately from the main wallet. When the main wallet opens, it processes the background wallet's cache. - To enable the feature in the RPC wallet, there is a new `/setup_background_sync` endpoint. - HW, multsig and view-only wallets cannot background sync. --- src/cryptonote_basic/account.cpp | 11 + src/cryptonote_basic/account.h | 1 + src/cryptonote_config.h | 2 + src/simplewallet/simplewallet.cpp | 204 +++- src/simplewallet/simplewallet.h | 1 + src/wallet/api/wallet.cpp | 213 +++- src/wallet/api/wallet.h | 12 + src/wallet/api/wallet2_api.h | 42 + src/wallet/wallet2.cpp | 1040 ++++++++++++++++-- src/wallet/wallet2.h | 156 ++- src/wallet/wallet_errors.h | 39 + src/wallet/wallet_rpc_server.cpp | 162 +++ src/wallet/wallet_rpc_server.h | 6 + src/wallet/wallet_rpc_server_commands_defs.h | 64 ++ src/wallet/wallet_rpc_server_error_codes.h | 2 + tests/functional_tests/transfer.py | 400 ++++++- tests/functional_tests/util_resources.py | 25 + tests/functional_tests/wallet.py | 43 +- tests/unit_tests/wipeable_string.cpp | 12 + utils/python-rpc/framework/wallet.py | 42 + 20 files changed, 2342 insertions(+), 135 deletions(-) diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index 2ac455fda..4e87d4477 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -152,6 +152,17 @@ DISABLE_VS_WARNINGS(4244 4345) m_keys.m_multisig_keys.clear(); } //----------------------------------------------------------------- + void account_base::set_spend_key(const crypto::secret_key& spend_secret_key) + { + // make sure derived spend public key matches saved public spend key + crypto::public_key spend_public_key; + crypto::secret_key_to_public_key(spend_secret_key, spend_public_key); + CHECK_AND_ASSERT_THROW_MES(m_keys.m_account_address.m_spend_public_key == spend_public_key, + "Unexpected derived public spend key"); + + m_keys.m_spend_secret_key = spend_secret_key; + } + //----------------------------------------------------------------- crypto::secret_key account_base::generate(const crypto::secret_key& recovery_key, bool recover, bool two_random) { crypto::secret_key first = generate_keys(m_keys.m_account_address.m_spend_public_key, m_keys.m_spend_secret_key, recovery_key, recover); diff --git a/src/cryptonote_basic/account.h b/src/cryptonote_basic/account.h index 2ee9545d4..93d1d28f0 100644 --- a/src/cryptonote_basic/account.h +++ b/src/cryptonote_basic/account.h @@ -95,6 +95,7 @@ namespace cryptonote bool store(const std::string& file_path); void forget_spend_key(); + void set_spend_key(const crypto::secret_key& spend_secret_key); const std::vector &get_multisig_keys() const { return m_keys.m_multisig_keys; } void encrypt_keys(const crypto::chacha_key &key) { m_keys.encrypt(key); } diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 61146a114..f9e6a6cb9 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -241,6 +241,8 @@ namespace config const unsigned char HASH_KEY_ENCRYPTED_PAYMENT_ID = 0x8d; const unsigned char HASH_KEY_WALLET = 0x8c; const unsigned char HASH_KEY_WALLET_CACHE = 0x8d; + const unsigned char HASH_KEY_BACKGROUND_CACHE = 0x8e; + const unsigned char HASH_KEY_BACKGROUND_KEYS_FILE = 0x8f; const unsigned char HASH_KEY_RPC_PAYMENT_NONCE = 0x58; const unsigned char HASH_KEY_MEMORY = 'k'; const unsigned char HASH_KEY_MULTISIG[] = {'M', 'u', 'l', 't' , 'i', 's', 'i', 'g', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index b9e30f9d9..2c51337ef 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -155,6 +155,17 @@ typedef cryptonote::simple_wallet sw; } \ } while(0) +#define CHECK_IF_BACKGROUND_SYNCING(msg) \ + do \ + { \ + if (m_wallet->is_background_wallet() || m_wallet->is_background_syncing()) \ + { \ + std::string type = m_wallet->is_background_wallet() ? "background wallet" : "background syncing wallet"; \ + fail_msg_writer() << boost::format(tr("%s %s")) % type % msg; \ + return false; \ + } \ + } while (0) + static std::string get_human_readable_timespan(std::chrono::seconds seconds); static std::string get_human_readable_timespan(uint64_t seconds); @@ -325,7 +336,7 @@ namespace auto pwd_container = tools::password_container::prompt(verify, prompt); if (!pwd_container) { - tools::fail_msg_writer() << sw::tr("failed to read wallet password"); + tools::fail_msg_writer() << sw::tr("failed to read password"); } return pwd_container; } @@ -335,6 +346,11 @@ namespace return password_prompter(verify ? sw::tr("Enter a new password for the wallet") : sw::tr("Wallet password"), verify); } + boost::optional background_sync_cache_password_prompter(bool verify) + { + return password_prompter(verify ? sw::tr("Enter a custom password for the background sync cache") : sw::tr("Background sync cache password"), verify); + } + inline std::string interpret_rpc_response(bool ok, const std::string& status) { std::string err; @@ -452,6 +468,41 @@ namespace return "invalid"; } + const struct + { + const char *name; + tools::wallet2::BackgroundSyncType background_sync_type; + } background_sync_type_names[] = + { + { "off", tools::wallet2::BackgroundSyncOff }, + { "reuse-wallet-password", tools::wallet2::BackgroundSyncReusePassword }, + { "custom-background-password", tools::wallet2::BackgroundSyncCustomPassword }, + }; + + bool parse_background_sync_type(const std::string &s, tools::wallet2::BackgroundSyncType &background_sync_type) + { + for (size_t n = 0; n < sizeof(background_sync_type_names) / sizeof(background_sync_type_names[0]); ++n) + { + if (s == background_sync_type_names[n].name) + { + background_sync_type = background_sync_type_names[n].background_sync_type; + return true; + } + } + fail_msg_writer() << cryptonote::simple_wallet::tr("failed to parse background sync type"); + return false; + } + + std::string get_background_sync_type_name(tools::wallet2::BackgroundSyncType type) + { + for (size_t n = 0; n < sizeof(background_sync_type_names) / sizeof(background_sync_type_names[0]); ++n) + { + if (type == background_sync_type_names[n].background_sync_type) + return background_sync_type_names[n].name; + } + return "invalid"; + } + std::string get_version_string(uint32_t version) { return boost::lexical_cast(version >> 16) + "." + boost::lexical_cast(version & 0xffff); @@ -805,6 +856,7 @@ bool simple_wallet::spendkey(const std::vector &args/* = std::vecto fail_msg_writer() << tr("wallet is watch-only and has no spend key"); return true; } + CHECK_IF_BACKGROUND_SYNCING("has no spend key"); // don't log PAUSE_READLINE(); if (m_wallet->key_on_device()) { @@ -836,6 +888,7 @@ bool simple_wallet::print_seed(bool encrypted) fail_msg_writer() << tr("wallet is watch-only and has no seed"); return true; } + CHECK_IF_BACKGROUND_SYNCING("has no seed"); multisig = m_wallet->multisig(&ready); if (multisig) @@ -913,6 +966,7 @@ bool simple_wallet::seed_set_language(const std::vector &args/* = s fail_msg_writer() << tr("wallet is watch-only and has no seed"); return true; } + CHECK_IF_BACKGROUND_SYNCING("has no seed"); epee::wipeable_string password; { @@ -1059,6 +1113,7 @@ bool simple_wallet::prepare_multisig_main(const std::vector &args, fail_msg_writer() << tr("wallet is watch-only and cannot be made multisig"); return false; } + CHECK_IF_BACKGROUND_SYNCING("cannot be made multisig"); if(m_wallet->get_num_transfer_details()) { @@ -2195,6 +2250,7 @@ bool simple_wallet::save_known_rings(const std::vector &args) bool simple_wallet::freeze_thaw(const std::vector &args, bool freeze) { + CHECK_IF_BACKGROUND_SYNCING("cannot freeze/thaw"); if (args.empty()) { fail_msg_writer() << boost::format(tr("usage: %s |")) % (freeze ? "freeze" : "thaw"); @@ -2234,6 +2290,7 @@ bool simple_wallet::thaw(const std::vector &args) bool simple_wallet::frozen(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot see frozen key images"); if (args.empty()) { size_t ntd = m_wallet->get_num_transfer_details(); @@ -3005,6 +3062,57 @@ bool simple_wallet::set_track_uses(const std::vector &args/* = std: return true; } +bool simple_wallet::setup_background_sync(const std::vector &args/* = std::vector()*/) +{ + if (m_wallet->multisig()) + { + fail_msg_writer() << tr("background sync not implemented for multisig wallet"); + return true; + } + if (m_wallet->watch_only()) + { + fail_msg_writer() << tr("background sync not implemented for watch only wallet"); + return true; + } + if (m_wallet->key_on_device()) + { + fail_msg_writer() << tr("command not supported by HW wallet"); + return true; + } + + tools::wallet2::BackgroundSyncType background_sync_type; + if (!parse_background_sync_type(args[1], background_sync_type)) + { + fail_msg_writer() << tr("invalid option"); + return true; + } + + const auto pwd_container = get_and_verify_password(); + if (!pwd_container) + return true; + + try + { + boost::optional background_cache_password = boost::none; + if (background_sync_type == tools::wallet2::BackgroundSyncCustomPassword) + { + const auto background_pwd_container = background_sync_cache_password_prompter(true); + if (!background_pwd_container) + return true; + background_cache_password = background_pwd_container->password(); + } + + LOCK_IDLE_SCOPE(); + m_wallet->setup_background_sync(background_sync_type, pwd_container->password(), background_cache_password); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Error setting background sync type: ") << e.what(); + } + + return true; +} + bool simple_wallet::set_show_wallet_name_when_locked(const std::vector &args/* = std::vector()*/) { const auto pwd_container = get_and_verify_password(); @@ -3237,6 +3345,7 @@ bool simple_wallet::apropos(const std::vector &args) bool simple_wallet::scan_tx(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot scan tx"); if (args.empty()) { PRINT_USAGE(USAGE_SCAN_TX); @@ -3458,6 +3567,8 @@ simple_wallet::simple_wallet() " Ignore outputs of amount below this threshold when spending.\n " "track-uses <1|0>\n " " Whether to keep track of owned outputs uses.\n " + "background-sync \n " + " Set this to enable scanning in the background with just the view key while the wallet is locked.\n " "setup-background-mining <1|0>\n " " Whether to enable background mining. Set this to support the network and to get a chance to receive new monero.\n " "device-name \n " @@ -3876,6 +3987,7 @@ bool simple_wallet::set_variable(const std::vector &args) success_msg_writer() << "ignore-outputs-above = " << cryptonote::print_money(m_wallet->ignore_outputs_above()); success_msg_writer() << "ignore-outputs-below = " << cryptonote::print_money(m_wallet->ignore_outputs_below()); success_msg_writer() << "track-uses = " << m_wallet->track_uses(); + success_msg_writer() << "background-sync = " << get_background_sync_type_name(m_wallet->background_sync_type()); success_msg_writer() << "setup-background-mining = " << setup_background_mining_string; success_msg_writer() << "device-name = " << m_wallet->device_name(); success_msg_writer() << "export-format = " << (m_wallet->export_format() == tools::wallet2::ExportFormat::Ascii ? "ascii" : "binary"); @@ -3894,6 +4006,7 @@ bool simple_wallet::set_variable(const std::vector &args) } else { + CHECK_IF_BACKGROUND_SYNCING("cannot change wallet settings"); #define CHECK_SIMPLE_VARIABLE(name, f, help) do \ if (args[0] == name) { \ @@ -3947,6 +4060,7 @@ bool simple_wallet::set_variable(const std::vector &args) CHECK_SIMPLE_VARIABLE("ignore-outputs-above", set_ignore_outputs_above, tr("amount")); CHECK_SIMPLE_VARIABLE("ignore-outputs-below", set_ignore_outputs_below, tr("amount")); CHECK_SIMPLE_VARIABLE("track-uses", set_track_uses, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("background-sync", setup_background_sync, tr("off (default); reuse-wallet-password (reuse the wallet password to encrypt the background cache); custom-background-password (use a custom background password to encrypt the background cache)")); CHECK_SIMPLE_VARIABLE("show-wallet-name-when-locked", set_show_wallet_name_when_locked, tr("1 or 0")); CHECK_SIMPLE_VARIABLE("inactivity-lock-timeout", set_inactivity_lock_timeout, tr("unsigned integer (seconds, 0 to disable)")); CHECK_SIMPLE_VARIABLE("setup-background-mining", set_setup_background_mining, tr("1/yes or 0/no")); @@ -4900,7 +5014,10 @@ std::string simple_wallet::get_mnemonic_language() //---------------------------------------------------------------------------------------------------- boost::optional simple_wallet::get_and_verify_password() const { - auto pwd_container = default_password_prompter(m_wallet_file.empty()); + const bool verify = m_wallet_file.empty(); + auto pwd_container = (m_wallet->is_background_wallet() && m_wallet->background_sync_type() == tools::wallet2::BackgroundSyncCustomPassword) + ? background_sync_cache_password_prompter(verify) + : default_password_prompter(verify); if (!pwd_container) return boost::none; @@ -5203,6 +5320,8 @@ boost::optional simple_wallet::open_wallet(const boost::p prefix = tr("Opened watch-only wallet"); else if (m_wallet->multisig(&ready, &threshold, &total)) prefix = (boost::format(tr("Opened %u/%u multisig wallet%s")) % threshold % total % (ready ? "" : " (not yet finalized)")).str(); + else if (m_wallet->is_background_wallet()) + prefix = tr("Opened background wallet"); else prefix = tr("Opened wallet"); message_writer(console_color_white, true) << @@ -5411,6 +5530,10 @@ void simple_wallet::stop_background_mining() //---------------------------------------------------------------------------------------------------- void simple_wallet::check_background_mining(const epee::wipeable_string &password) { + // Background mining can be toggled from the main wallet + if (m_wallet->is_background_wallet() || m_wallet->is_background_syncing()) + return; + tools::wallet2::BackgroundMiningSetupType setup = m_wallet->setup_background_mining(); if (setup == tools::wallet2::BackgroundMiningNo) { @@ -6275,6 +6398,7 @@ bool simple_wallet::show_blockchain_height(const std::vector& args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_spent(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot rescan spent"); if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); @@ -6532,10 +6656,27 @@ void simple_wallet::check_for_inactivity_lock(bool user) " || ||" << std::endl << "" << std::endl; } + + bool started_background_sync = false; + if (!m_wallet->is_background_wallet() && + m_wallet->background_sync_type() != tools::wallet2::BackgroundSyncOff) + { + LOCK_IDLE_SCOPE(); + m_wallet->start_background_sync(); + started_background_sync = true; + } + while (1) { const char *inactivity_msg = user ? "" : tr("Locked due to inactivity."); - tools::msg_writer() << inactivity_msg << (inactivity_msg[0] ? " " : "") << tr("The wallet password is required to unlock the console."); + tools::msg_writer() << inactivity_msg << (inactivity_msg[0] ? " " : "") << ( + (m_wallet->is_background_wallet() && m_wallet->background_sync_type() == tools::wallet2::BackgroundSyncCustomPassword) + ? tr("The background password is required to unlock the console.") + : tr("The wallet password is required to unlock the console.") + ); + + if (m_wallet->is_background_syncing()) + tools::msg_writer() << tr("\nSyncing in the background while locked...") << std::endl; const bool show_wallet_name = m_wallet->show_wallet_name_when_locked(); if (show_wallet_name) @@ -6548,8 +6689,16 @@ void simple_wallet::check_for_inactivity_lock(bool user) } try { - if (get_and_verify_password()) + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + if (started_background_sync) + { + LOCK_IDLE_SCOPE(); + m_wallet->stop_background_sync(pwd_container->password()); + } break; + } } catch (...) { /* do nothing, just let the loop loop */ } } @@ -6576,6 +6725,7 @@ bool simple_wallet::on_command(bool (simple_wallet::*cmd)(const std::vector &args_, bool called_by_mms) { // "transfer [index=[,,...]] [] []
[]" + CHECK_IF_BACKGROUND_SYNCING("cannot transfer"); if (!try_connect_to_daemon()) return false; @@ -7004,6 +7154,7 @@ bool simple_wallet::transfer_main(const std::vector &args_, bool ca //---------------------------------------------------------------------------------------------------- bool simple_wallet::transfer(const std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot transfer"); if (args_.size() < 1) { PRINT_USAGE(USAGE_TRANSFER); @@ -7016,6 +7167,7 @@ bool simple_wallet::transfer(const std::vector &args_) bool simple_wallet::sweep_unmixable(const std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot sweep"); if (!try_connect_to_daemon()) return true; @@ -7123,6 +7275,7 @@ bool simple_wallet::sweep_unmixable(const std::vector &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_main(uint32_t account, uint64_t below, const std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot sweep"); auto print_usage = [this, account, below]() { if (below) @@ -7404,6 +7557,7 @@ bool simple_wallet::sweep_main(uint32_t account, uint64_t below, const std::vect //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_single(const std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot sweep"); if (!try_connect_to_daemon()) return true; @@ -7642,12 +7796,14 @@ bool simple_wallet::sweep_single(const std::vector &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_all(const std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot sweep"); sweep_main(m_current_subaddress_account, 0, args_); return true; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_account(const std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot sweep"); auto local_args = args_; if (local_args.empty()) { @@ -7668,6 +7824,7 @@ bool simple_wallet::sweep_account(const std::vector &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_below(const std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot sweep"); uint64_t below = 0; if (args_.size() < 1) { @@ -7686,6 +7843,7 @@ bool simple_wallet::sweep_below(const std::vector &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::donate(const std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot donate"); std::vector local_args = args_; if(local_args.empty() || local_args.size() > 5) { @@ -7747,6 +7905,7 @@ bool simple_wallet::donate(const std::vector &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::accept_loaded_tx(const std::function get_num_txes, const std::function &get_tx, const std::string &extra_message) { + CHECK_IF_BACKGROUND_SYNCING("cannot load tx"); // gather info to ask the user uint64_t amount = 0, amount_to_dests = 0, change = 0; size_t min_ring_size = ~0; @@ -7927,6 +8086,7 @@ bool simple_wallet::sign_transfer(const std::vector &args_) fail_msg_writer() << tr("This is a watch only wallet"); return true; } + CHECK_IF_BACKGROUND_SYNCING("cannot sign transfer"); bool export_raw = false; std::string unsigned_filename = "unsigned_monero_tx"; @@ -8034,6 +8194,8 @@ std::string get_tx_key_stream(crypto::secret_key tx_key, std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot get tx key"); + std::vector local_args = args_; if (m_wallet->key_on_device() && m_wallet->get_account().get_device().get_type() != hw::device::TREZOR) @@ -8074,6 +8236,8 @@ bool simple_wallet::get_tx_key(const std::vector &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::set_tx_key(const std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot set tx key"); + std::vector local_args = args_; if(local_args.size() != 2 && local_args.size() != 3) { @@ -8150,6 +8314,8 @@ bool simple_wallet::set_tx_key(const std::vector &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::get_tx_proof(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot get tx proof"); + if (args.size() != 2 && args.size() != 3) { PRINT_USAGE(USAGE_GET_TX_PROOF); @@ -8356,6 +8522,7 @@ bool simple_wallet::check_tx_proof(const std::vector &args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::get_spend_proof(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot get spend proof"); if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); @@ -8440,6 +8607,7 @@ bool simple_wallet::check_spend_proof(const std::vector &args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::get_reserve_proof(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot get reserve proof"); if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); @@ -9126,6 +9294,8 @@ bool simple_wallet::unspent_outputs(const std::vector &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_blockchain(const std::vector &args_) { + CHECK_IF_BACKGROUND_SYNCING("cannot rescan"); + uint64_t start_height = 0; ResetType reset_type = ResetSoft; @@ -9423,6 +9593,7 @@ bool simple_wallet::account(const std::vector &args/* = std::vector if (command == "new") { // create a new account and switch to it + CHECK_IF_BACKGROUND_SYNCING("cannot create new account"); std::string label = boost::join(local_args, " "); if (label.empty()) label = tr("(Untitled account)"); @@ -9453,6 +9624,7 @@ bool simple_wallet::account(const std::vector &args/* = std::vector else if (command == "label" && local_args.size() >= 1) { // set label of the specified account + CHECK_IF_BACKGROUND_SYNCING("cannot modify account"); uint32_t index_major; if (!epee::string_tools::get_xtype_from_string(index_major, local_args[0])) { @@ -9474,6 +9646,7 @@ bool simple_wallet::account(const std::vector &args/* = std::vector } else if (command == "tag" && local_args.size() >= 2) { + CHECK_IF_BACKGROUND_SYNCING("cannot modify account"); const std::string tag = local_args[0]; std::set account_indices; for (size_t i = 1; i < local_args.size(); ++i) @@ -9498,6 +9671,7 @@ bool simple_wallet::account(const std::vector &args/* = std::vector } else if (command == "untag" && local_args.size() >= 1) { + CHECK_IF_BACKGROUND_SYNCING("cannot modify account"); std::set account_indices; for (size_t i = 0; i < local_args.size(); ++i) { @@ -9521,6 +9695,7 @@ bool simple_wallet::account(const std::vector &args/* = std::vector } else if (command == "tag_description" && local_args.size() >= 1) { + CHECK_IF_BACKGROUND_SYNCING("cannot modify account"); const std::string tag = local_args[0]; std::string description; if (local_args.size() > 1) @@ -9638,6 +9813,7 @@ bool simple_wallet::print_address(const std::vector &args/* = std:: } else if (local_args[0] == "new") { + CHECK_IF_BACKGROUND_SYNCING("cannot add address"); local_args.erase(local_args.begin()); std::string label; if (local_args.size() > 0) @@ -9650,6 +9826,7 @@ bool simple_wallet::print_address(const std::vector &args/* = std:: } else if (local_args[0] == "mnew") { + CHECK_IF_BACKGROUND_SYNCING("cannot add addresses"); local_args.erase(local_args.begin()); if (local_args.size() != 1) { @@ -9675,6 +9852,7 @@ bool simple_wallet::print_address(const std::vector &args/* = std:: } else if (local_args[0] == "one-off") { + CHECK_IF_BACKGROUND_SYNCING("cannot add address"); local_args.erase(local_args.begin()); std::string label; if (local_args.size() != 2) @@ -9693,6 +9871,7 @@ bool simple_wallet::print_address(const std::vector &args/* = std:: } else if (local_args.size() >= 2 && local_args[0] == "label") { + CHECK_IF_BACKGROUND_SYNCING("cannot modify address"); if (!epee::string_tools::get_xtype_from_string(index, local_args[1])) { fail_msg_writer() << tr("failed to parse index: ") << local_args[1]; @@ -9839,6 +10018,8 @@ bool simple_wallet::print_integrated_address(const std::vector &arg //---------------------------------------------------------------------------------------------------- bool simple_wallet::address_book(const std::vector &args/* = std::vector()*/) { + CHECK_IF_BACKGROUND_SYNCING("cannot get address book"); + if (args.size() == 0) { } @@ -9899,6 +10080,8 @@ bool simple_wallet::address_book(const std::vector &args/* = std::v //---------------------------------------------------------------------------------------------------- bool simple_wallet::set_tx_note(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot set tx note"); + if (args.size() == 0) { PRINT_USAGE(USAGE_SET_TX_NOTE); @@ -9927,6 +10110,8 @@ bool simple_wallet::set_tx_note(const std::vector &args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::get_tx_note(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot get tx note"); + if (args.size() != 1) { PRINT_USAGE(USAGE_GET_TX_NOTE); @@ -9952,6 +10137,8 @@ bool simple_wallet::get_tx_note(const std::vector &args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::set_description(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot set description"); + // 0 arguments allowed, for setting the description to empty string std::string description = ""; @@ -9968,6 +10155,8 @@ bool simple_wallet::set_description(const std::vector &args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::get_description(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot get description"); + if (args.size() != 0) { PRINT_USAGE(USAGE_GET_DESCRIPTION); @@ -10026,6 +10215,8 @@ bool simple_wallet::wallet_info(const std::vector &args) type = tr("Watch only"); else if (m_wallet->multisig(&ready, &threshold, &total)) type = (boost::format(tr("%u/%u multisig%s")) % threshold % total % (ready ? "" : " (not yet finalized)")).str(); + else if (m_wallet->is_background_wallet()) + type = tr("Background wallet"); else type = tr("Normal"); message_writer() << tr("Type: ") << type; @@ -10037,6 +10228,7 @@ bool simple_wallet::wallet_info(const std::vector &args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::sign(const std::vector &args) { + CHECK_IF_BACKGROUND_SYNCING("cannot sign"); if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); @@ -10144,6 +10336,7 @@ bool simple_wallet::export_key_images(const std::vector &args_) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } + CHECK_IF_BACKGROUND_SYNCING("cannot export key images"); auto args = args_; if (m_wallet->watch_only()) @@ -10197,6 +10390,7 @@ bool simple_wallet::import_key_images(const std::vector &args) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } + CHECK_IF_BACKGROUND_SYNCING("cannot import key images"); if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); @@ -10305,6 +10499,7 @@ bool simple_wallet::export_outputs(const std::vector &args_) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } + CHECK_IF_BACKGROUND_SYNCING("cannot export outputs"); auto args = args_; bool all = false; @@ -10354,6 +10549,7 @@ bool simple_wallet::import_outputs(const std::vector &args) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } + CHECK_IF_BACKGROUND_SYNCING("cannot import outputs"); if (args.size() != 1) { PRINT_USAGE(USAGE_IMPORT_OUTPUTS); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 652708f5a..159da2c45 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -147,6 +147,7 @@ namespace cryptonote bool set_ignore_outputs_above(const std::vector &args = std::vector()); bool set_ignore_outputs_below(const std::vector &args = std::vector()); bool set_track_uses(const std::vector &args = std::vector()); + bool setup_background_sync(const std::vector &args = std::vector()); bool set_show_wallet_name_when_locked(const std::vector &args = std::vector()); bool set_inactivity_lock_timeout(const std::vector &args = std::vector()); bool set_setup_background_mining(const std::vector &args = std::vector()); diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index fc4f89128..e9f76f4cf 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -54,6 +54,40 @@ using namespace cryptonote; #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "WalletAPI" +#define LOCK_REFRESH() \ + bool refresh_enabled = m_refreshEnabled; \ + m_refreshEnabled = false; \ + m_wallet->stop(); \ + m_refreshCV.notify_one(); \ + boost::mutex::scoped_lock lock(m_refreshMutex); \ + boost::mutex::scoped_lock lock2(m_refreshMutex2); \ + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ \ + /* m_refreshMutex's still locked here */ \ + if (refresh_enabled) \ + startRefresh(); \ + }) + +#define PRE_VALIDATE_BACKGROUND_SYNC() \ + do \ + { \ + clearStatus(); \ + if (m_wallet->key_on_device()) \ + { \ + setStatusError(tr("HW wallet cannot use background sync")); \ + return false; \ + } \ + if (m_wallet->watch_only()) \ + { \ + setStatusError(tr("View only wallet cannot use background sync")); \ + return false; \ + } \ + if (m_wallet->multisig()) \ + { \ + setStatusError(tr("Multisig wallet cannot use background sync")); \ + return false; \ + } \ + } while (0) + namespace Monero { namespace { @@ -792,6 +826,8 @@ bool WalletImpl::close(bool store) std::string WalletImpl::seed(const std::string& seed_offset) const { + if (checkBackgroundSync("cannot get seed")) + return std::string(); epee::wipeable_string seed; if (m_wallet) m_wallet->get_seed(seed, seed_offset); @@ -805,6 +841,8 @@ std::string WalletImpl::getSeedLanguage() const void WalletImpl::setSeedLanguage(const std::string &arg) { + if (checkBackgroundSync("cannot set seed language")) + return; m_wallet->set_seed_language(arg); } @@ -828,6 +866,8 @@ void WalletImpl::statusWithErrorString(int& status, std::string& errorString) co bool WalletImpl::setPassword(const std::string &password) { + if (checkBackgroundSync("cannot change password")) + return false; clearStatus(); try { m_wallet->change_password(m_wallet->get_wallet_file(), m_password, password); @@ -988,6 +1028,8 @@ bool WalletImpl::lightWalletImportWalletRequest(std::string &payment_id, uint64_ void WalletImpl::setRefreshFromBlockHeight(uint64_t refresh_from_block_height) { + if (checkBackgroundSync("cannot change refresh height")) + return; m_wallet->set_refresh_from_block_height(refresh_from_block_height); } @@ -1105,6 +1147,8 @@ void WalletImpl::refreshAsync() bool WalletImpl::rescanBlockchain() { + if (checkBackgroundSync("cannot rescan blockchain")) + return false; clearStatus(); m_refreshShouldRescan = true; doRefresh(); @@ -1113,6 +1157,8 @@ bool WalletImpl::rescanBlockchain() void WalletImpl::rescanBlockchainAsync() { + if (checkBackgroundSync("cannot rescan blockchain")) + return; m_refreshShouldRescan = true; refreshAsync(); } @@ -1136,7 +1182,7 @@ int WalletImpl::autoRefreshInterval() const UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_filename) { clearStatus(); UnsignedTransactionImpl * transaction = new UnsignedTransactionImpl(*this); - if (!m_wallet->load_unsigned_tx(unsigned_filename, transaction->m_unsigned_tx_set)){ + if (checkBackgroundSync("cannot load tx") || !m_wallet->load_unsigned_tx(unsigned_filename, transaction->m_unsigned_tx_set)){ setStatusError(tr("Failed to load unsigned transactions")); transaction->m_status = UnsignedTransaction::Status::Status_Error; transaction->m_errorString = errorString(); @@ -1156,6 +1202,8 @@ UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_file bool WalletImpl::submitTransaction(const string &fileName) { clearStatus(); + if (checkBackgroundSync("cannot submit tx")) + return false; std::unique_ptr transaction(new PendingTransactionImpl(*this)); bool r = m_wallet->load_tx(fileName, transaction->m_pending_tx); @@ -1179,6 +1227,8 @@ bool WalletImpl::exportKeyImages(const string &filename, bool all) setStatusError(tr("Wallet is view only")); return false; } + if (checkBackgroundSync("cannot export key images")) + return false; try { @@ -1199,6 +1249,8 @@ bool WalletImpl::exportKeyImages(const string &filename, bool all) bool WalletImpl::importKeyImages(const string &filename) { + if (checkBackgroundSync("cannot import key images")) + return false; if (!trustedDaemon()) { setStatusError(tr("Key images can only be imported with a trusted daemon")); return false; @@ -1222,6 +1274,8 @@ bool WalletImpl::importKeyImages(const string &filename) bool WalletImpl::exportOutputs(const string &filename, bool all) { + if (checkBackgroundSync("cannot export outputs")) + return false; if (m_wallet->key_on_device()) { setStatusError(string(tr("Not supported on HW wallets.")) + filename); @@ -1252,6 +1306,8 @@ bool WalletImpl::exportOutputs(const string &filename, bool all) bool WalletImpl::importOutputs(const string &filename) { + if (checkBackgroundSync("cannot import outputs")) + return false; if (m_wallet->key_on_device()) { setStatusError(string(tr("Not supported on HW wallets.")) + filename); @@ -1284,6 +1340,8 @@ bool WalletImpl::importOutputs(const string &filename) bool WalletImpl::scanTransactions(const std::vector &txids) { + if (checkBackgroundSync("cannot scan transactions")) + return false; if (txids.empty()) { setStatusError(string(tr("Failed to scan transactions: no transaction ids provided."))); @@ -1322,8 +1380,86 @@ bool WalletImpl::scanTransactions(const std::vector &txids) return true; } +bool WalletImpl::setupBackgroundSync(const Wallet::BackgroundSyncType background_sync_type, const std::string &wallet_password, const optional &background_cache_password) +{ + try + { + PRE_VALIDATE_BACKGROUND_SYNC(); + + tools::wallet2::BackgroundSyncType bgs_type; + switch (background_sync_type) + { + case Wallet::BackgroundSync_Off: bgs_type = tools::wallet2::BackgroundSyncOff; break; + case Wallet::BackgroundSync_ReusePassword: bgs_type = tools::wallet2::BackgroundSyncReusePassword; break; + case Wallet::BackgroundSync_CustomPassword: bgs_type = tools::wallet2::BackgroundSyncCustomPassword; break; + default: setStatusError(tr("Unknown background sync type")); return false; + } + + boost::optional bgc_password = background_cache_password + ? boost::optional(*background_cache_password) + : boost::none; + + LOCK_REFRESH(); + m_wallet->setup_background_sync(bgs_type, wallet_password, bgc_password); + } + catch (const std::exception &e) + { + LOG_ERROR("Failed to setup background sync: " << e.what()); + setStatusError(string(tr("Failed to setup background sync: ")) + e.what()); + return false; + } + return true; +} + +Wallet::BackgroundSyncType WalletImpl::getBackgroundSyncType() const +{ + switch (m_wallet->background_sync_type()) + { + case tools::wallet2::BackgroundSyncOff: return Wallet::BackgroundSync_Off; + case tools::wallet2::BackgroundSyncReusePassword: return Wallet::BackgroundSync_ReusePassword; + case tools::wallet2::BackgroundSyncCustomPassword: return Wallet::BackgroundSync_CustomPassword; + default: setStatusError(tr("Unknown background sync type")); return Wallet::BackgroundSync_Off; + } +} + +bool WalletImpl::startBackgroundSync() +{ + try + { + PRE_VALIDATE_BACKGROUND_SYNC(); + LOCK_REFRESH(); + m_wallet->start_background_sync(); + } + catch (const std::exception &e) + { + LOG_ERROR("Failed to start background sync: " << e.what()); + setStatusError(string(tr("Failed to start background sync: ")) + e.what()); + return false; + } + return true; +} + +bool WalletImpl::stopBackgroundSync(const std::string &wallet_password) +{ + try + { + PRE_VALIDATE_BACKGROUND_SYNC(); + LOCK_REFRESH(); + m_wallet->stop_background_sync(epee::wipeable_string(wallet_password)); + } + catch (const std::exception &e) + { + LOG_ERROR("Failed to stop background sync: " << e.what()); + setStatusError(string(tr("Failed to stop background sync: ")) + e.what()); + return false; + } + return true; +} + void WalletImpl::addSubaddressAccount(const std::string& label) { + if (checkBackgroundSync("cannot add account")) + return; m_wallet->add_subaddress_account(label); } size_t WalletImpl::numSubaddressAccounts() const @@ -1336,10 +1472,14 @@ size_t WalletImpl::numSubaddresses(uint32_t accountIndex) const } void WalletImpl::addSubaddress(uint32_t accountIndex, const std::string& label) { + if (checkBackgroundSync("cannot add subbaddress")) + return; m_wallet->add_subaddress(accountIndex, label); } std::string WalletImpl::getSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex) const { + if (checkBackgroundSync("cannot get subbaddress label")) + return ""; try { return m_wallet->get_subaddress_label({accountIndex, addressIndex}); @@ -1353,6 +1493,8 @@ std::string WalletImpl::getSubaddressLabel(uint32_t accountIndex, uint32_t addre } void WalletImpl::setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label) { + if (checkBackgroundSync("cannot set subbaddress label")) + return; try { return m_wallet->set_subaddress_label({accountIndex, addressIndex}, label); @@ -1366,12 +1508,16 @@ void WalletImpl::setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex MultisigState WalletImpl::multisig() const { MultisigState state; + if (checkBackgroundSync("cannot use multisig")) + return state; state.isMultisig = m_wallet->multisig(&state.isReady, &state.threshold, &state.total); return state; } string WalletImpl::getMultisigInfo() const { + if (checkBackgroundSync("cannot use multisig")) + return string(); try { clearStatus(); return m_wallet->get_multisig_first_kex_msg(); @@ -1384,6 +1530,8 @@ string WalletImpl::getMultisigInfo() const { } string WalletImpl::makeMultisig(const vector& info, const uint32_t threshold) { + if (checkBackgroundSync("cannot make multisig")) + return string(); try { clearStatus(); @@ -1524,6 +1672,9 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector extra; std::string extra_nonce; vector dsts; @@ -1690,6 +1841,9 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() PendingTransactionImpl * transaction = new PendingTransactionImpl(*this); do { + if (checkBackgroundSync("cannot sweep")) + break; + try { transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(); pendingTxPostProcess(transaction); @@ -1823,11 +1977,15 @@ uint32_t WalletImpl::defaultMixin() const void WalletImpl::setDefaultMixin(uint32_t arg) { + if (checkBackgroundSync("cannot set default mixin")) + return; m_wallet->default_mixin(arg); } bool WalletImpl::setCacheAttribute(const std::string &key, const std::string &val) { + if (checkBackgroundSync("cannot set cache attribute")) + return false; m_wallet->set_attribute(key, val); return true; } @@ -1841,6 +1999,8 @@ std::string WalletImpl::getCacheAttribute(const std::string &key) const bool WalletImpl::setUserNote(const std::string &txid, const std::string ¬e) { + if (checkBackgroundSync("cannot set user note")) + return false; cryptonote::blobdata txid_data; if(!epee::string_tools::parse_hexstr_to_binbuff(txid, txid_data) || txid_data.size() != sizeof(crypto::hash)) return false; @@ -1852,6 +2012,8 @@ bool WalletImpl::setUserNote(const std::string &txid, const std::string ¬e) std::string WalletImpl::getUserNote(const std::string &txid) const { + if (checkBackgroundSync("cannot get user note")) + return ""; cryptonote::blobdata txid_data; if(!epee::string_tools::parse_hexstr_to_binbuff(txid, txid_data) || txid_data.size() != sizeof(crypto::hash)) return ""; @@ -1862,6 +2024,9 @@ std::string WalletImpl::getUserNote(const std::string &txid) const std::string WalletImpl::getTxKey(const std::string &txid_str) const { + if (checkBackgroundSync("cannot get tx key")) + return ""; + crypto::hash txid; if(!epee::string_tools::hex_to_pod(txid_str, txid)) { @@ -1946,6 +2111,9 @@ bool WalletImpl::checkTxKey(const std::string &txid_str, std::string tx_key_str, std::string WalletImpl::getTxProof(const std::string &txid_str, const std::string &address_str, const std::string &message) const { + if (checkBackgroundSync("cannot get tx proof")) + return ""; + crypto::hash txid; if (!epee::string_tools::hex_to_pod(txid_str, txid)) { @@ -2002,6 +2170,9 @@ bool WalletImpl::checkTxProof(const std::string &txid_str, const std::string &ad } std::string WalletImpl::getSpendProof(const std::string &txid_str, const std::string &message) const { + if (checkBackgroundSync("cannot get spend proof")) + return ""; + crypto::hash txid; if(!epee::string_tools::hex_to_pod(txid_str, txid)) { @@ -2044,6 +2215,9 @@ bool WalletImpl::checkSpendProof(const std::string &txid_str, const std::string } std::string WalletImpl::getReserveProof(bool all, uint32_t account_index, uint64_t amount, const std::string &message) const { + if (checkBackgroundSync("cannot get reserve proof")) + return ""; + try { clearStatus(); @@ -2090,6 +2264,9 @@ bool WalletImpl::checkReserveProof(const std::string &address, const std::string std::string WalletImpl::signMessage(const std::string &message, const std::string &address) { + if (checkBackgroundSync("cannot sign message")) + return ""; + if (address.empty()) { return m_wallet->sign(message, tools::wallet2::sign_with_spend_key); } @@ -2217,6 +2394,16 @@ bool WalletImpl::isDeterministic() const return m_wallet->is_deterministic(); } +bool WalletImpl::isBackgroundSyncing() const +{ + return m_wallet->is_background_syncing(); +} + +bool WalletImpl::isBackgroundWallet() const +{ + return m_wallet->is_background_wallet(); +} + void WalletImpl::clearStatus() const { boost::lock_guard l(m_statusMutex); @@ -2285,9 +2472,7 @@ void WalletImpl::doRefresh() if(rescan) m_wallet->rescan_blockchain(false); m_wallet->refresh(trustedDaemon()); - if (!m_synchronized) { - m_synchronized = true; - } + m_synchronized = m_wallet->is_synced(); // assuming if we have empty history, it wasn't initialized yet // for further history changes client need to update history in // "on_money_received" and "on_money_sent" callbacks @@ -2391,6 +2576,24 @@ bool WalletImpl::doInit(const string &daemon_address, const std::string &proxy_a return true; } +bool WalletImpl::checkBackgroundSync(const std::string &message) const +{ + clearStatus(); + if (m_wallet->is_background_wallet()) + { + LOG_ERROR("Background wallets " + message); + setStatusError(tr("Background wallets ") + message); + return true; + } + if (m_wallet->is_background_syncing()) + { + LOG_ERROR(message + " while background syncing"); + setStatusError(message + tr(" while background syncing. Stop background syncing first.")); + return true; + } + return false; +} + bool WalletImpl::parse_uri(const std::string &uri, std::string &address, std::string &payment_id, uint64_t &amount, std::string &tx_description, std::string &recipient_name, std::vector &unknown_parameters, std::string &error) { return m_wallet->parse_uri(uri, address, payment_id, amount, tx_description, recipient_name, unknown_parameters, error); @@ -2409,6 +2612,8 @@ std::string WalletImpl::getDefaultDataDir() const bool WalletImpl::rescanSpent() { clearStatus(); + if (checkBackgroundSync("cannot rescan spent")) + return false; if (!trustedDaemon()) { setStatusError(tr("Rescan spent can only be used with a trusted daemon")); return false; diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index ec2d7e9b3..1f199a72c 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -171,6 +171,13 @@ public: bool importOutputs(const std::string &filename) override; bool scanTransactions(const std::vector &txids) override; + bool setupBackgroundSync(const BackgroundSyncType background_sync_type, const std::string &wallet_password, const optional &background_cache_password = optional()) override; + BackgroundSyncType getBackgroundSyncType() const override; + bool startBackgroundSync() override; + bool stopBackgroundSync(const std::string &wallet_password) override; + bool isBackgroundSyncing() const override; + bool isBackgroundWallet() const override; + virtual void disposeTransaction(PendingTransaction * t) override; virtual uint64_t estimateTransactionFee(const std::vector> &destinations, PendingTransaction::Priority priority) const override; @@ -239,6 +246,7 @@ private: bool isNewWallet() const; void pendingTxPostProcess(PendingTransactionImpl * pending); bool doInit(const std::string &daemon_address, const std::string &proxy_address, uint64_t upper_transaction_size_limit = 0, bool ssl = false); + bool checkBackgroundSync(const std::string &message) const; private: friend class PendingTransactionImpl; @@ -253,6 +261,10 @@ private: mutable boost::mutex m_statusMutex; mutable int m_status; mutable std::string m_errorString; + // TODO: harden password handling in the wallet API, see relevant discussion + // https://github.com/monero-project/monero-gui/issues/1537 + // https://github.com/feather-wallet/feather/issues/72#issuecomment-1405602142 + // https://github.com/monero-project/monero/pull/8619#issuecomment-1632951461 std::string m_password; std::unique_ptr m_history; std::unique_ptr m_wallet2Callback; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 71991df0d..e349df176 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -445,6 +445,12 @@ struct Wallet ConnectionStatus_WrongVersion }; + enum BackgroundSyncType { + BackgroundSync_Off = 0, + BackgroundSync_ReusePassword = 1, + BackgroundSync_CustomPassword = 2 + }; + virtual ~Wallet() = 0; virtual std::string seed(const std::string& seed_offset = "") const = 0; virtual std::string getSeedLanguage() const = 0; @@ -936,6 +942,42 @@ struct Wallet */ virtual bool scanTransactions(const std::vector &txids) = 0; + /*! + * \brief setupBackgroundSync - setup background sync mode with just a view key + * \param background_sync_type - the mode the wallet background syncs in + * \param wallet_password + * \param background_cache_password - custom password to encrypt background cache, only needed for custom password background sync type + * \return - true on success + */ + virtual bool setupBackgroundSync(const BackgroundSyncType background_sync_type, const std::string &wallet_password, const optional &background_cache_password) = 0; + + /*! + * \brief getBackgroundSyncType - get mode the wallet background syncs in + * \return - the type, or off if type is unknown + */ + virtual BackgroundSyncType getBackgroundSyncType() const = 0; + + /** + * @brief startBackgroundSync - sync the chain in the background with just view key + */ + virtual bool startBackgroundSync() = 0; + + /** + * @brief stopBackgroundSync - bring back spend key and process background synced txs + * \param wallet_password + */ + virtual bool stopBackgroundSync(const std::string &wallet_password) = 0; + + /** + * @brief isBackgroundSyncing - returns true if the wallet is background syncing + */ + virtual bool isBackgroundSyncing() const = 0; + + /** + * @brief isBackgroundWallet - returns true if the wallet is a background wallet + */ + virtual bool isBackgroundWallet() const = 0; + virtual TransactionHistory * history() = 0; virtual AddressBook * addressBook() = 0; virtual Subaddress * subaddress() = 0; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index b746dee44..e8991a326 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -157,6 +157,8 @@ static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; static const std::string ASCII_OUTPUT_MAGIC = "MoneroAsciiDataV1"; +static const std::string BACKGROUND_WALLET_SUFFIX = ".background"; + boost::mutex tools::wallet2::default_daemon_address_lock; std::string tools::wallet2::default_daemon_address = ""; @@ -1009,14 +1011,14 @@ uint64_t num_priv_multisig_keys_post_setup(uint64_t threshold, uint64_t total) * @param keys_data_key the chacha key that encrypts wallet keys files * @return crypto::chacha_key the chacha key that encrypts the wallet cache files */ -crypto::chacha_key derive_cache_key(const crypto::chacha_key& keys_data_key) +crypto::chacha_key derive_cache_key(const crypto::chacha_key& keys_data_key, const unsigned char domain_separator) { static_assert(HASH_SIZE == sizeof(crypto::chacha_key), "Mismatched sizes of hash and chacha key"); crypto::chacha_key cache_key; epee::mlocked> cache_key_data; memcpy(cache_key_data.data(), &keys_data_key, HASH_SIZE); - cache_key_data[HASH_SIZE] = config::HASH_KEY_WALLET_CACHE; + cache_key_data[HASH_SIZE] = domain_separator; cn_fast_hash(cache_key_data.data(), HASH_SIZE+1, (crypto::hash&) cache_key); return cache_key; @@ -1104,7 +1106,7 @@ wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, const boost::optional lock(lockers_lock); if (lockers++ > 0) locked = false; - if (!locked || w.is_unattended() || w.ask_password() != tools::wallet2::AskPasswordToDecrypt || w.watch_only()) + if (!locked || w.is_unattended() || w.ask_password() != tools::wallet2::AskPasswordToDecrypt || w.watch_only() || w.is_background_syncing()) { locked = false; return; @@ -1221,6 +1223,11 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std m_ignore_outputs_above(MONEY_SUPPLY), m_ignore_outputs_below(0), m_track_uses(false), + m_is_background_wallet(false), + m_background_sync_type(BackgroundSyncOff), + m_background_syncing(false), + m_processing_background_cache(false), + m_custom_background_key(boost::none), m_show_wallet_name_when_locked(false), m_inactivity_lock_timeout(DEFAULT_INACTIVITY_LOCK_TIMEOUT), m_setup_background_mining(BackgroundMiningMaybe), @@ -1860,6 +1867,9 @@ bool has_nonrequested_tx_at_height_or_above_requested(uint64_t height, const std //---------------------------------------------------------------------------------------------------- void wallet2::scan_tx(const std::unordered_set &txids) { + THROW_WALLET_EXCEPTION_IF(m_background_syncing || m_is_background_wallet, error::wallet_internal_error, + "cannot scan tx from background wallet"); + // Get the transactions from daemon in batches sorted lowest height to highest tx_entry_data txs_to_scan = get_tx_entries(txids); if (txs_to_scan.tx_entries.empty()) @@ -2167,11 +2177,11 @@ void wallet2::scan_output(const cryptonote::transaction &tx, bool miner_tx, cons THROW_WALLET_EXCEPTION_IF(i >= tx.vout.size(), error::wallet_internal_error, "Invalid vout index"); // if keys are encrypted, ask for password - if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only && !m_multisig_rescan_k) + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only && !m_multisig_rescan_k && !m_background_syncing) { static critical_section password_lock; CRITICAL_REGION_LOCAL(password_lock); - if (!m_encrypt_keys_after_refresh) + if (!m_encrypt_keys_after_refresh && !m_processing_background_cache) { boost::optional pwd = m_callback->on_get_password(pool ? "output found in pool" : "output received"); THROW_WALLET_EXCEPTION_IF(!pwd, error::password_needed, tr("Password is needed to compute key image for incoming monero")); @@ -2183,7 +2193,7 @@ void wallet2::scan_output(const cryptonote::transaction &tx, bool miner_tx, cons crypto::public_key output_public_key; THROW_WALLET_EXCEPTION_IF(!get_output_public_key(tx.vout[i], output_public_key), error::wallet_internal_error, "Failed to get output public key"); - if (m_multisig) + if (m_multisig || m_background_syncing/*no spend key*/) { tx_scan_info.in_ephemeral.pub = output_public_key; tx_scan_info.in_ephemeral.sec = crypto::null_skey; @@ -2440,6 +2450,22 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote THROW_WALLET_EXCEPTION_IF(tx.vout.size() != o_indices.size(), error::wallet_internal_error, "transactions outputs size=" + std::to_string(tx.vout.size()) + " not match with daemon response size=" + std::to_string(o_indices.size())); + + // we're going to re-process this receive when background sync is disabled + if (m_background_syncing && m_background_sync_data.txs.find(txid) == m_background_sync_data.txs.end()) + { + size_t bgs_idx = m_background_sync_data.txs.size(); + background_synced_tx_t bgs_tx = { + .index_in_background_sync_data = bgs_idx, + .tx = tx, + .output_indices = o_indices, + .height = height, + .block_timestamp = ts, + .double_spend_seen = double_spend_seen + }; + LOG_PRINT_L2("Adding received tx " << txid << " to background sync data (idx=" << bgs_idx << ")"); + m_background_sync_data.txs.insert({txid, std::move(bgs_tx)}); + } } for(size_t o: outs) @@ -2465,7 +2491,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_tx = (const cryptonote::transaction_prefix&)tx; td.m_txid = txid; td.m_key_image = tx_scan_info[o].ki; - td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_known = !m_watch_only && !m_multisig && !m_background_syncing; if (!td.m_key_image_known) { // we might have cold signed, and have a mapping to key images @@ -2655,10 +2681,25 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote set_spent(it->second, height); if (!ignore_callbacks && 0 != m_callback) m_callback->on_money_spent(height, txid, tx, amount, tx, td.m_subaddr_index); + + if (m_background_syncing && m_background_sync_data.txs.find(txid) == m_background_sync_data.txs.end()) + { + size_t bgs_idx = m_background_sync_data.txs.size(); + background_synced_tx_t bgs_tx = { + .index_in_background_sync_data = bgs_idx, + .tx = tx, + .output_indices = o_indices, + .height = height, + .block_timestamp = ts, + .double_spend_seen = double_spend_seen + }; + LOG_PRINT_L2("Adding spent tx " << txid << " to background sync data (idx=" << bgs_idx << ")"); + m_background_sync_data.txs.insert({txid, std::move(bgs_tx)}); + } } } - if (!pool && m_track_uses) + if (!pool && (m_track_uses || (m_background_syncing && it == m_key_images.end()))) { PERF_TIMER(track_uses); const uint64_t amount = in_to_key.amount; @@ -2672,7 +2713,27 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote { size_t idx = i->second; THROW_WALLET_EXCEPTION_IF(idx >= m_transfers.size(), error::wallet_internal_error, "Output tracker cache index out of range"); - m_transfers[idx].m_uses.push_back(std::make_pair(height, txid)); + + if (m_track_uses) + m_transfers[idx].m_uses.push_back(std::make_pair(height, txid)); + + // We'll re-process all txs which *might* be spends when we disable + // background sync and retrieve the spend key. We don't know if an + // output is a spend in this tx if we don't know its key image. + if (m_background_syncing && !m_transfers[idx].m_key_image_known && m_background_sync_data.txs.find(txid) == m_background_sync_data.txs.end()) + { + size_t bgs_idx = m_background_sync_data.txs.size(); + background_synced_tx_t bgs_tx = { + .index_in_background_sync_data = bgs_idx, + .tx = tx, + .output_indices = o_indices, + .height = height, + .block_timestamp = ts, + .double_spend_seen = double_spend_seen + }; + LOG_PRINT_L2("Adding plausible spent tx " << txid << " to background sync data (idx=" << bgs_idx << ")"); + m_background_sync_data.txs.insert({txid, std::move(bgs_tx)}); + } } } } @@ -2682,7 +2743,24 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote continue; for (uint64_t offset: offsets) if (offset == td.m_global_output_index) - td.m_uses.push_back(std::make_pair(height, txid)); + { + if (m_track_uses) + td.m_uses.push_back(std::make_pair(height, txid)); + if (m_background_syncing && !td.m_key_image_known && m_background_sync_data.txs.find(txid) == m_background_sync_data.txs.end()) + { + size_t bgs_idx = m_background_sync_data.txs.size(); + background_synced_tx_t bgs_tx = { + .index_in_background_sync_data = bgs_idx, + .tx = tx, + .output_indices = o_indices, + .height = height, + .block_timestamp = ts, + .double_spend_seen = double_spend_seen + }; + LOG_PRINT_L2("Adding plausible spent tx " << txid << " to background sync data (idx=" << bgs_idx << ")"); + m_background_sync_data.txs.insert({txid, std::move(bgs_tx)}); + } + } } } } @@ -3055,8 +3133,8 @@ void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_heigh req.start_height = start_height; req.no_miner_tx = m_refresh_type == RefreshNoCoinbase; - req.requested_info = first ? COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL : COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY; - if (try_incremental) + req.requested_info = (first && !m_background_syncing) ? COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL : COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY; + if (try_incremental && !m_background_syncing) req.pool_info_since = m_pool_info_query_time; { @@ -3083,7 +3161,7 @@ void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_heigh << ", height " << blocks_start_height + blocks.size() << ", node height " << res.current_height << ", pool info " << static_cast(res.pool_info_extent)); - if (first) + if (first && !m_background_syncing) { if (res.pool_info_extent != COMMAND_RPC_GET_BLOCKS_FAST::NONE) { @@ -3595,6 +3673,9 @@ void wallet2::process_unconfirmed_transfer(bool incremental, const crypto::hash // incremental update anymore, because with that we might miss some txs altogether. void wallet2::update_pool_state(std::vector> &process_txs, bool refreshed, bool try_incremental) { + process_txs.clear(); + if (m_background_syncing) + return; bool updated = false; if (m_pool_info_query_time != 0 && try_incremental) { @@ -4165,6 +4246,8 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo } m_first_refresh_done = true; + if (m_background_syncing || m_is_background_wallet) + m_background_sync_data.first_refresh_done = true; LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all(false)) << ", unlocked: " << print_money(unlocked_balance_all(false))); } @@ -4250,6 +4333,14 @@ wallet2::detached_blockchain_data wallet2::detach_blockchain(uint64_t height, st td.m_uses.pop_back(); } + for (auto it = m_background_sync_data.txs.begin(); it != m_background_sync_data.txs.end(); ) + { + if(height <= it->second.height) + it = m_background_sync_data.txs.erase(it); + else + ++it; + } + if (output_tracker_cache) output_tracker_cache->clear(); @@ -4324,8 +4415,12 @@ void wallet2::handle_reorg(uint64_t height, std::map m_blockchain.offset(), error::wallet_internal_error, "Daemon claims reorg below last checkpoint"); + detached_blockchain_data dbd = detach_blockchain(height, output_tracker_cache); + if (m_background_syncing && height < m_background_sync_data.start_height) + m_background_sync_data.start_height = height; + if (m_callback) m_callback->on_reorg(height, dbd.detached_blockchain.size(), dbd.detached_tx_hashes.size()); } @@ -4335,6 +4430,7 @@ bool wallet2::deinit() if(m_is_initialized) { m_is_initialized = false; unlock_keys_file(); + unlock_background_keys_file(); m_account.deinit(); } return true; @@ -4361,6 +4457,7 @@ bool wallet2::clear() m_device_last_key_image_sync = 0; m_pool_info_query_time = 0; m_skip_to_height = 0; + m_background_sync_data = background_sync_data_t{}; return true; } //---------------------------------------------------------------------------------------------------- @@ -4379,13 +4476,30 @@ void wallet2::clear_soft(bool keep_key_images) m_scanned_pool_txs[1].clear(); m_pool_info_query_time = 0; m_skip_to_height = 0; + m_background_sync_data = background_sync_data_t{}; cryptonote::block b; generate_genesis(b); m_blockchain.push_back(get_block_hash(b)); m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); } - +//---------------------------------------------------------------------------------------------------- +void wallet2::clear_user_data() +{ + for (auto i = m_confirmed_txs.begin(); i != m_confirmed_txs.end(); ++i) + i->second.m_dests.clear(); + for (auto i = m_unconfirmed_txs.begin(); i != m_unconfirmed_txs.end(); ++i) + i->second.m_dests.clear(); + for (auto i = m_transfers.begin(); i != m_transfers.end(); ++i) + i->m_frozen = false; + m_tx_keys.clear(); + m_tx_notes.clear(); + m_address_book.clear(); + m_subaddress_labels.clear(); + m_attributes.clear(); + m_account_tags = std::pair, std::vector>(); +} +//---------------------------------------------------------------------------------------------------- /*! * \brief Stores wallet information to wallet file. * \param keys_file_name Name of wallet file @@ -4397,16 +4511,35 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable { boost::optional keys_file_data = get_keys_file_data(password, watch_only); CHECK_AND_ASSERT_MES(keys_file_data != boost::none, false, "failed to generate wallet keys data"); - + return store_keys_file_data(keys_file_name, keys_file_data.get()); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::store_keys(const std::string& keys_file_name, const crypto::chacha_key& key, bool watch_only, bool background_keys_file) +{ + boost::optional keys_file_data = get_keys_file_data(key, watch_only, background_keys_file); + CHECK_AND_ASSERT_MES(keys_file_data != boost::none, false, "failed to generate wallet keys data"); + return store_keys_file_data(keys_file_name, keys_file_data.get(), background_keys_file); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::store_keys_file_data(const std::string& keys_file_name, wallet2::keys_file_data &keys_file_data, bool background_keys_file) +{ std::string tmp_file_name = keys_file_name + ".new"; std::string buf; - bool r = ::serialization::dump_binary(keys_file_data.get(), buf); + bool r = ::serialization::dump_binary(keys_file_data, buf); r = r && save_to_file(tmp_file_name, buf); CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name); - unlock_keys_file(); + if (!background_keys_file) + unlock_keys_file(); + else + unlock_background_keys_file(); + std::error_code e = tools::replace_file(tmp_file_name, keys_file_name); - lock_keys_file(); + + if (!background_keys_file) + lock_keys_file(); + else + lock_background_keys_file(keys_file_name); if (e) { boost::filesystem::remove(tmp_file_name); @@ -4418,26 +4551,27 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable } //---------------------------------------------------------------------------------------------------- boost::optional wallet2::get_keys_file_data(const epee::wipeable_string& password, bool watch_only) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + verify_password_with_cached_key(key); + return get_keys_file_data(key, watch_only); +} +//---------------------------------------------------------------------------------------------------- +boost::optional wallet2::get_keys_file_data(const crypto::chacha_key& key, bool watch_only, bool background_keys_file) { epee::byte_slice account_data; std::string multisig_signers; std::string multisig_derivations; cryptonote::account_base account = m_account; - crypto::chacha_key key; - crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); - - // We use m_cache_key as a deterministic test to see if given key corresponds to original password - const crypto::chacha_key cache_key = derive_cache_key(key); - THROW_WALLET_EXCEPTION_IF(cache_key != m_cache_key, error::invalid_password); - if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) { account.encrypt_viewkey(key); account.decrypt_keys(key); } - if (watch_only) + if (watch_only || background_keys_file) account.forget_spend_key(); account.encrypt_keys(key); @@ -4572,6 +4706,9 @@ boost::optional wallet2::get_keys_file_data(const epee: value2.SetInt(m_track_uses ? 1 : 0); json.AddMember("track_uses", value2, json.GetAllocator()); + value2.SetInt(m_background_sync_type); + json.AddMember("background_sync_type", value2, json.GetAllocator()); + value2.SetInt(m_show_wallet_name_when_locked ? 1 : 0); json.AddMember("show_wallet_name_when_locked", value2, json.GetAllocator()); @@ -4629,6 +4766,12 @@ boost::optional wallet2::get_keys_file_data(const epee: value2.SetInt(m_enable_multisig ? 1 : 0); json.AddMember("enable_multisig", value2, json.GetAllocator()); + if (m_background_sync_type == BackgroundSyncCustomPassword && !background_keys_file && m_custom_background_key) + { + value.SetString(reinterpret_cast(m_custom_background_key.get().data()), m_custom_background_key.get().size()); + json.AddMember("custom_background_key", value, json.GetAllocator()); + } + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); @@ -4655,13 +4798,81 @@ void wallet2::setup_keys(const epee::wipeable_string &password) m_account.decrypt_viewkey(key); } - m_cache_key = derive_cache_key(key); + m_cache_key = derive_cache_key(key, config::HASH_KEY_WALLET_CACHE); get_ringdb_key(); } //---------------------------------------------------------------------------------------------------- +void validate_background_cache_password_usage(const tools::wallet2::BackgroundSyncType background_sync_type, const boost::optional &background_cache_password, const bool multisig, const bool watch_only, const bool key_on_device) +{ + THROW_WALLET_EXCEPTION_IF(multisig || watch_only || key_on_device, error::wallet_internal_error, multisig + ? "Background sync not implemented for multisig wallets" : watch_only + ? "Background sync not implemented for view only wallets" + : "Background sync not implemented for HW wallets"); + + switch (background_sync_type) + { + case tools::wallet2::BackgroundSyncOff: + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, "background sync is not enabled"); + break; + } + case tools::wallet2::BackgroundSyncReusePassword: + { + THROW_WALLET_EXCEPTION_IF(background_cache_password, error::wallet_internal_error, + "unexpected custom background cache password"); + break; + } + case tools::wallet2::BackgroundSyncCustomPassword: + { + THROW_WALLET_EXCEPTION_IF(!background_cache_password, error::wallet_internal_error, + "expected custom background cache password"); + break; + } + default: THROW_WALLET_EXCEPTION(error::wallet_internal_error, "unknown background sync type"); + } +} +//---------------------------------------------------------------------------------------------------- +void get_custom_background_key(const epee::wipeable_string &password, crypto::chacha_key &custom_background_key, const uint64_t kdf_rounds) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, kdf_rounds); + custom_background_key = derive_cache_key(key, config::HASH_KEY_BACKGROUND_KEYS_FILE); +} +//---------------------------------------------------------------------------------------------------- +const crypto::chacha_key wallet2::get_cache_key() +{ + if (m_background_sync_type == BackgroundSyncCustomPassword && m_background_syncing) + { + THROW_WALLET_EXCEPTION_IF(!m_custom_background_key, error::wallet_internal_error, "Custom background key not set"); + // Domain separate keys used to encrypt background keys file and cache + return derive_cache_key(m_custom_background_key.get(), config::HASH_KEY_BACKGROUND_CACHE); + } + else + { + return m_cache_key; + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::verify_password_with_cached_key(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + verify_password_with_cached_key(key); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::verify_password_with_cached_key(const crypto::chacha_key &key) +{ + // We use m_cache_key as a deterministic test to see if given key corresponds to original password + const crypto::chacha_key cache_key = derive_cache_key(key, config::HASH_KEY_WALLET_CACHE); + THROW_WALLET_EXCEPTION_IF(cache_key != m_cache_key, error::invalid_password); +} +//---------------------------------------------------------------------------------------------------- void wallet2::change_password(const std::string &filename, const epee::wipeable_string &original_password, const epee::wipeable_string &new_password) { + THROW_WALLET_EXCEPTION_IF(m_background_syncing || m_is_background_wallet, error::wallet_internal_error, + "cannot change password from background wallet"); + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) decrypt_keys(original_password); setup_keys(new_password); @@ -4720,8 +4931,24 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st std::string account_data; account_data.resize(keys_file_data.account_data.size()); crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); - if (json.Parse(account_data.c_str()).HasParseError() || !json.IsObject()) + const bool try_v0_format = json.Parse(account_data.c_str()).HasParseError() || !json.IsObject(); + if (try_v0_format) crypto::chacha8(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + + // Check if it's a background keys file if both of the above formats fail + { + m_is_background_wallet = false; + m_background_syncing = false; + cryptonote::account_base account_data_check; + if (try_v0_format && !epee::serialization::load_t_from_binary(account_data_check, account_data)) + { + get_custom_background_key(password, key, m_kdf_rounds); + crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + m_is_background_wallet = !json.Parse(account_data.c_str()).HasParseError() && json.IsObject(); + m_background_syncing = m_is_background_wallet; // start a background wallet background syncing + } + } + // The contents should be JSON if the wallet follows the new format. if (json.Parse(account_data.c_str()).HasParseError()) { @@ -4759,6 +4986,7 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st m_ignore_outputs_above = MONEY_SUPPLY; m_ignore_outputs_below = 0; m_track_uses = false; + m_background_sync_type = BackgroundSyncOff; m_show_wallet_name_when_locked = false; m_inactivity_lock_timeout = DEFAULT_INACTIVITY_LOCK_TIMEOUT; m_setup_background_mining = BackgroundMiningMaybe; @@ -4776,6 +5004,7 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st m_credits_target = 0; m_enable_multisig = false; m_allow_mismatched_daemon_version = false; + m_custom_background_key = boost::none; } else if(json.IsObject()) { @@ -5012,6 +5241,39 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st m_credits_target = field_credits_target; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, enable_multisig, int, Int, false, false); m_enable_multisig = field_enable_multisig; + + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, background_sync_type, BackgroundSyncType, Int, false, BackgroundSyncOff); + m_background_sync_type = field_background_sync_type; + + // Load encryption key used to encrypt background cache + crypto::chacha_key custom_background_key; + m_custom_background_key = boost::none; + if (m_background_sync_type == BackgroundSyncCustomPassword && !m_is_background_wallet) + { + if (!json.HasMember("custom_background_key")) + { + LOG_ERROR("Field custom_background_key not found in JSON"); + return false; + } + else if (!json["custom_background_key"].IsString()) + { + LOG_ERROR("Field custom_background_key found in JSON, but not String"); + return false; + } + else if (json["custom_background_key"].GetStringLength() != sizeof(crypto::chacha_key)) + { + LOG_ERROR("Field custom_background_key found in JSON, but not correct length"); + return false; + } + const char *field_custom_background_key = json["custom_background_key"].GetString(); + memcpy(custom_background_key.data(), field_custom_background_key, sizeof(crypto::chacha_key)); + m_custom_background_key = boost::optional(custom_background_key); + LOG_PRINT_L1("Loaded custom background key derived from custom password"); + } + else if (json.HasMember("custom_background_key")) + { + LOG_ERROR("Unexpected field custom_background_key found in JSON"); + } } else { @@ -5075,12 +5337,17 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st const cryptonote::account_keys& keys = m_account.get_keys(); hw::device &hwdev = m_account.get_device(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); - if (!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD) + if (!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD && !m_is_background_wallet) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); if (r) - setup_keys(password); + { + if (!m_is_background_wallet) + setup_keys(password); + else + m_custom_background_key = boost::optional(key); + } return true; } @@ -5095,11 +5362,12 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password * */ -bool wallet2::verify_password(const epee::wipeable_string& password) +bool wallet2::verify_password(const epee::wipeable_string& password, crypto::secret_key &spend_key_out) { // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). unlock_keys_file(); - bool r = verify_password(m_keys_file, password, m_account.get_device().device_protocol() == hw::device::PROTOCOL_COLD || m_watch_only || m_multisig, m_account.get_device(), m_kdf_rounds); + const bool no_spend_key = m_account.get_device().device_protocol() == hw::device::PROTOCOL_COLD || m_watch_only || m_multisig || m_is_background_wallet; + bool r = verify_password(m_keys_file, password, no_spend_key, m_account.get_device(), m_kdf_rounds, spend_key_out); lock_keys_file(); return r; } @@ -5117,7 +5385,7 @@ bool wallet2::verify_password(const epee::wipeable_string& password) * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password * */ -bool wallet2::verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds) +bool wallet2::verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds, crypto::secret_key &spend_key_out) { rapidjson::Document json; wallet2::keys_file_data keys_file_data; @@ -5134,9 +5402,22 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip std::string account_data; account_data.resize(keys_file_data.account_data.size()); crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); - if (json.Parse(account_data.c_str()).HasParseError() || !json.IsObject()) + const bool try_v0_format = json.Parse(account_data.c_str()).HasParseError() || !json.IsObject(); + if (try_v0_format) crypto::chacha8(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + // Check if it's a background keys file if both of the above formats fail + { + cryptonote::account_base account_data_check; + if (try_v0_format && !epee::serialization::load_t_from_binary(account_data_check, account_data)) + { + get_custom_background_key(password, key, kdf_rounds); + crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + const bool is_background_wallet = json.Parse(account_data.c_str()).HasParseError() && json.IsObject(); + no_spend_key = no_spend_key || is_background_wallet; + } + } + // The contents should be JSON if the wallet follows the new format. if (json.Parse(account_data.c_str()).HasParseError()) { @@ -5161,6 +5442,7 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); if(!no_spend_key) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); + spend_key_out = (!no_spend_key && r) ? keys.m_spend_secret_key : crypto::null_skey; return r; } @@ -5172,9 +5454,7 @@ void wallet2::encrypt_keys(const crypto::chacha_key &key) void wallet2::decrypt_keys(const crypto::chacha_key &key) { - // We use m_cache_key as a deterministic test to see if given key corresponds to original password - const crypto::chacha_key cache_key = derive_cache_key(key); - THROW_WALLET_EXCEPTION_IF(cache_key != m_cache_key, error::invalid_password); + verify_password_with_cached_key(key); m_account.encrypt_viewkey(key); m_account.decrypt_keys(key); @@ -5850,11 +6130,30 @@ void wallet2::rewrite(const std::string& wallet_name, const epee::wipeable_strin { if (wallet_name.empty()) return; + THROW_WALLET_EXCEPTION_IF(m_background_syncing || m_is_background_wallet, error::wallet_internal_error, + "cannot change wallet settings from background wallet"); prepare_file_names(wallet_name); boost::system::error_code ignored_ec; THROW_WALLET_EXCEPTION_IF(!boost::filesystem::exists(m_keys_file, ignored_ec), error::file_not_found, m_keys_file); bool r = store_keys(m_keys_file, password, m_watch_only); THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + + // Update the background keys file when we rewrite the main wallet keys file + if (m_background_sync_type == BackgroundSyncCustomPassword && m_custom_background_key) + { + const std::string background_keys_filename = make_background_keys_file_name(wallet_name); + if (!lock_background_keys_file(background_keys_filename)) + { + LOG_ERROR("Background keys file " << background_keys_filename << " is opened by another wallet program and cannot be rewritten"); + return; // not fatal, background keys file will just have different wallet settings + } + store_background_keys(m_custom_background_key.get()); + store_background_cache(m_custom_background_key.get(), true/*do_reset_background_sync_data*/); + } + else if (m_background_sync_type == BackgroundSyncReusePassword) + { + reset_background_sync_data(m_background_sync_data); + } } /*! * \brief Writes to a file named based on the normal wallet (doesn't generate key, assumes it's already there) @@ -5888,6 +6187,16 @@ bool wallet2::wallet_valid_path_format(const std::string& file_path) return !file_path.empty(); } //---------------------------------------------------------------------------------------------------- +std::string wallet2::make_background_wallet_file_name(const std::string &wallet_file) +{ + return wallet_file + BACKGROUND_WALLET_SUFFIX; +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::make_background_keys_file_name(const std::string &wallet_file) +{ + return make_background_wallet_file_name(wallet_file) + ".keys"; +} +//---------------------------------------------------------------------------------------------------- bool wallet2::parse_long_payment_id(const std::string& payment_id_str, crypto::hash& payment_id) { cryptonote::blobdata payment_id_data; @@ -6123,10 +6432,81 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, "failed to load keys from buffer"); } - wallet_keys_unlocker unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only, password); + wallet_keys_unlocker unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only && !m_is_background_wallet, password); //keys loaded ok! //try to load wallet cache. but even if we failed, it is not big problem + load_wallet_cache(use_fs, cache_buf); + + if (!m_persistent_rpc_client_id) + set_rpc_client_secret_key(rct::rct2sk(rct::skGen())); + + // Wallets used to wipe, but not erase, old unused multisig key info, which lead to huge memory leaks. + // Here we erase these multisig keys if they're zero'd out to free up space. + for (auto &td : m_transfers) + { + auto mk_it = td.m_multisig_k.begin(); + while (mk_it != td.m_multisig_k.end()) + { + if (*mk_it == rct::zero()) + mk_it = td.m_multisig_k.erase(mk_it); + else + ++mk_it; + } + } + + cryptonote::block genesis; + generate_genesis(genesis); + crypto::hash genesis_hash = get_block_hash(genesis); + + if (m_blockchain.empty()) + { + m_blockchain.push_back(genesis_hash); + m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); + } + else + { + check_genesis(genesis_hash); + } + + trim_hashchain(); + + if (get_num_subaddress_accounts() == 0) + add_subaddress_account(tr("Primary account")); + + try + { + find_and_save_rings(false); + } + catch (const std::exception &e) + { + MERROR("Failed to save rings, will try again next time"); + } + + try + { + if (use_fs) + m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file, m_load_deprecated_formats); + } + catch (const std::exception &e) + { + MERROR("Failed to initialize MMS, it will be unusable"); + } + + try + { + if (use_fs) + process_background_cache_on_open(); + } + catch (const std::exception &e) + { + MERROR("Failed to process background cache on open: " << e.what()); + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::load_wallet_cache(const bool use_fs, const std::string& cache_buf) +{ + boost::system::error_code e; bool cache_missing = use_fs ? (!boost::filesystem::exists(m_wallet_file, e) || e) : cache_buf.empty(); if (cache_missing) { @@ -6140,7 +6520,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass bool r = true; if (use_fs) { - load_from_file(m_wallet_file, cache_file_buf, std::numeric_limits::max()); + r = load_from_file(m_wallet_file, cache_file_buf, std::numeric_limits::max()); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); } @@ -6153,7 +6533,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + m_wallet_file + '\"'); std::string cache_data; cache_data.resize(cache_file_data.cache_data.size()); - crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cache_data[0]); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), get_cache_key(), cache_file_data.iv, &cache_data[0]); try { bool loaded = false; @@ -6243,60 +6623,76 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass m_account_public_address.m_view_public_key != m_account.get_keys().m_account_address.m_view_public_key, error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::process_background_cache_on_open() +{ + if (m_wallet_file.empty()) + return; + if (m_background_syncing || m_is_background_wallet) + return; + if (m_background_sync_type == BackgroundSyncOff) + return; - if (!m_persistent_rpc_client_id) - set_rpc_client_secret_key(rct::rct2sk(rct::skGen())); - - // Wallets used to wipe, but not erase, old unused multisig key info, which lead to huge memory leaks. - // Here we erase these multisig keys if they're zero'd out to free up space. - for (auto &td : m_transfers) + if (m_background_sync_type == BackgroundSyncReusePassword) { - auto mk_it = td.m_multisig_k.begin(); - while (mk_it != td.m_multisig_k.end()) - { - if (*mk_it == rct::zero()) - mk_it = td.m_multisig_k.erase(mk_it); - else - ++mk_it; - } + const background_sync_data_t background_sync_data = m_background_sync_data; + const hashchain blockchain = m_blockchain; + process_background_cache(background_sync_data, blockchain, m_last_block_reward); + + // Reset the background cache after processing + reset_background_sync_data(m_background_sync_data); } - - cryptonote::block genesis; - generate_genesis(genesis); - crypto::hash genesis_hash = get_block_hash(genesis); - - if (m_blockchain.empty()) + else if (m_background_sync_type == BackgroundSyncCustomPassword) { - m_blockchain.push_back(genesis_hash); - m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); + // If the background wallet files don't exist, recreate them + const std::string background_keys_file = make_background_keys_file_name(m_wallet_file); + const std::string background_wallet_file = make_background_wallet_file_name(m_wallet_file); + const bool background_keys_file_exists = boost::filesystem::exists(background_keys_file); + const bool background_wallet_exists = boost::filesystem::exists(background_wallet_file); + + THROW_WALLET_EXCEPTION_IF(!lock_background_keys_file(background_keys_file), error::background_wallet_already_open, background_wallet_file); + THROW_WALLET_EXCEPTION_IF(!m_custom_background_key, error::wallet_internal_error, "Custom background key not set"); + + if (!background_keys_file_exists) + { + MDEBUG("Background keys file not found, restoring"); + store_background_keys(m_custom_background_key.get()); + } + + if (!background_wallet_exists) + { + MDEBUG("Background cache not found, restoring"); + store_background_cache(m_custom_background_key.get(), true/*do_reset_background_sync_data*/); + return; + } + + MDEBUG("Loading background cache"); + + // Set up a minimal background wallet2 instance + std::unique_ptr background_w2(new wallet2(m_nettype)); + background_w2->m_is_background_wallet = true; + background_w2->m_background_syncing = true; + background_w2->m_background_sync_type = m_background_sync_type; + background_w2->m_custom_background_key = m_custom_background_key; + + cryptonote::account_base account = m_account; + account.forget_spend_key(); + background_w2->m_account = account; + + // Load background cache from file + background_w2->clear(); + background_w2->prepare_file_names(background_wallet_file); + background_w2->load_wallet_cache(true/*use_fs*/); + + process_background_cache(background_w2->m_background_sync_data, background_w2->m_blockchain, background_w2->m_last_block_reward); + + // Reset the background cache after processing + store_background_cache(m_custom_background_key.get(), true/*do_reset_background_sync_data*/); } else { - check_genesis(genesis_hash); - } - - trim_hashchain(); - - if (get_num_subaddress_accounts() == 0) - add_subaddress_account(tr("Primary account")); - - try - { - find_and_save_rings(false); - } - catch (const std::exception &e) - { - MERROR("Failed to save rings, will try again next time"); - } - - try - { - if (use_fs) - m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file, m_load_deprecated_formats); - } - catch (const std::exception &e) - { - MERROR("Failed to initialize MMS, it will be unusable"); + THROW_WALLET_EXCEPTION(error::wallet_internal_error, "unknown background sync type"); } } //---------------------------------------------------------------------------------------------------- @@ -6378,6 +6774,8 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas same_file = canonical_old_path == canonical_new_path; } + THROW_WALLET_EXCEPTION_IF(m_is_background_wallet && !same_file, error::wallet_internal_error, + "Cannot save background wallet files to a different location"); if (!same_file) { @@ -6394,6 +6792,21 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas } } } + else if (m_background_sync_type == BackgroundSyncCustomPassword && m_background_syncing && !m_is_background_wallet) + { + // We're background syncing, so store the wallet cache as a background cache + // keeping the background sync data + try + { + THROW_WALLET_EXCEPTION_IF(!m_custom_background_key, error::wallet_internal_error, "Custom background key not set"); + store_background_cache(m_custom_background_key.get(), false/*do_reset_background_sync_data*/); + } + catch (const std::exception &e) + { + MERROR("Failed to store background cache while background syncing: " << e.what()); + } + return; + } // get wallet cache data boost::optional cache_file_data = get_cache_file_data(); @@ -6487,6 +6900,22 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas // store should only exist if the MMS is really active m_message_store.write_to_file(get_multisig_wallet_state(), m_mms_file); } + + if (m_background_sync_type == BackgroundSyncCustomPassword && !m_background_syncing && !m_is_background_wallet) + { + // Update the background wallet cache when we store the main wallet cache + // Note: if background syncing when this is called, it means the background + // wallet is open and was already stored above + try + { + THROW_WALLET_EXCEPTION_IF(!m_custom_background_key, error::wallet_internal_error, "Custom background key not set"); + store_background_cache(m_custom_background_key.get(), true/*do_reset_background_sync_data*/); + } + catch (const std::exception &e) + { + MERROR("Failed to update background cache: " << e.what()); + } + } } //---------------------------------------------------------------------------------------------------- boost::optional wallet2::get_cache_file_data() @@ -6504,7 +6933,7 @@ boost::optional wallet2::get_cache_file_data() std::string cipher; cipher.resize(cache_file_data.get().cache_data.size()); cache_file_data.get().iv = crypto::rand(); - crypto::chacha20(cache_file_data.get().cache_data.data(), cache_file_data.get().cache_data.size(), m_cache_key, cache_file_data.get().iv, &cipher[0]); + crypto::chacha20(cache_file_data.get().cache_data.data(), cache_file_data.get().cache_data.size(), get_cache_key(), cache_file_data.get().iv, &cipher[0]); cache_file_data.get().cache_data = cipher; return cache_file_data; } @@ -8580,6 +9009,34 @@ bool wallet2::is_keys_file_locked() const return m_keys_file_locker->locked(); } +bool wallet2::lock_background_keys_file(const std::string &background_keys_file) +{ + if (background_keys_file.empty() || !boost::filesystem::exists(background_keys_file)) + return true; + if (m_background_keys_file_locker && m_background_keys_file_locker->locked()) + return true; + m_background_keys_file_locker.reset(new tools::file_locker(background_keys_file)); + return m_background_keys_file_locker->locked(); +} + +bool wallet2::unlock_background_keys_file() +{ + if (!m_background_keys_file_locker) + { + MDEBUG("background keys file locker is not set"); + return false; + } + m_background_keys_file_locker.reset(); + return true; +} + +bool wallet2::is_background_keys_file_locked() const +{ + if (!m_background_keys_file_locker) + return false; + return m_background_keys_file_locker->locked(); +} + bool wallet2::tx_add_fake_output(std::vector> &outs, uint64_t global_index, const crypto::public_key& output_public_key, const rct::key& mask, uint64_t real_index, bool unlocked, std::unordered_set &valid_public_keys_cache) const { if (!unlocked) // don't add locked outs @@ -13912,6 +14369,413 @@ bool wallet2::import_key_images(signed_tx_set & signed_tx, size_t offset, bool o return import_key_images(signed_tx.key_images, offset, only_selected_transfers ? boost::make_optional(selected_transfers) : boost::none); } +/* + In background sync mode, we use just the view key when the wallet is scanning + to identify all txs where: + + 1. We received an output. + 2. We spent an output. + 3. We *may* have spent a received output but we didn't know for sure because + the spend key was not loaded while background sync was enabled. + + When the user is ready to use the spend key again, we call this function to + process all those background synced transactions with the spend key loaded, + so that we can properly generate key images for the transactions which we + we were not able to do so for while background sync was enabled. This allows + us to determine *all* receives and spends the user completed while the wallet + had background sync enabled. Once this function completes, we can continue + scanning from where the background sync left off. + + Txs of type 3 (txs which we *may* have spent received output(s)) are txs where + 1+ rings contain an output that the user received and the wallet does not know + the associated key image for that output. We don't know if the user spent in + this type of tx or not. This function will generate key images for all outputs + we don't know key images for, and then check if those outputs were spent in + the txs of type 3. + + By storing this type of "plausible spend tx" when scanning in background sync + mode, we avoid the need to query the daemon with key images when background + sync mode is disabled to see if those key images were spent. This would + reveal key images to 3rd party nodes for users who don't run their own. + Although this is not a perfect solution to avoid revealing key images to a 3rd + party node (since tx submission trivially reveals key images to a node), it's + probably better than revealing *unused* key images to a 3rd party node, which + would enable the 3rd party to deduce that a tx is spending an output at least + X old when the key image is included in the chain. +*/ +void wallet2::process_background_cache(const background_sync_data_t &background_sync_data, const hashchain &background_synced_chain, uint64_t last_block_reward) +{ + // We expect the spend key to be in a decrypted state while + // m_processing_background_cache is true + m_processing_background_cache = true; + auto done_processing = epee::misc_utils::create_scope_leave_handler([&, this]() { + m_processing_background_cache = false; + }); + + if (m_background_syncing || m_multisig || m_watch_only || key_on_device()) + return; + + if (!background_sync_data.first_refresh_done) + { + MDEBUG("Skipping processing background cache, background cache has not synced yet"); + return; + } + + // Skip processing if wallet cache is synced higher than background cache + const uint64_t current_height = m_blockchain.size(); + const uint64_t background_height = background_synced_chain.size(); + MDEBUG("Background cache height " << background_height << " , wallet height " << current_height); + if (current_height > background_height) + { + MWARNING("Skipping processing background cache, synced height is higher than background cache"); + return; + } + + if (m_refresh_from_block_height < background_sync_data.wallet_refresh_from_block_height || + m_subaddress_lookahead_major > background_sync_data.subaddress_lookahead_major || + m_subaddress_lookahead_minor > background_sync_data.subaddress_lookahead_minor || + m_refresh_type < background_sync_data.wallet_refresh_type) + { + MWARNING("Skipping processing background cache, background wallet sync settings did not match main wallet's"); + MDEBUG("Wallet settings: " << + ", m_refresh_from_block_height: " << m_refresh_from_block_height << " vs " << background_sync_data.wallet_refresh_from_block_height << + ", m_subaddress_lookahead_major: " << m_subaddress_lookahead_major << " vs " << background_sync_data.subaddress_lookahead_major << + ", m_subaddress_lookahead_minor: " << m_subaddress_lookahead_minor << " vs " << background_sync_data.subaddress_lookahead_minor << + ", m_refresh_type: " << m_refresh_type << " vs " << background_sync_data.wallet_refresh_type); + return; + } + + // Sort background synced txs in the order they appeared in the cache so that + // we process them in the order they appeared in the chain. Thus if tx2 spends + // from tx1, we will know because tx1 is processed before tx2. + std::vector> sorted_bgs_cache(background_sync_data.txs.begin(), background_sync_data.txs.end()); + std::sort(sorted_bgs_cache.begin(), sorted_bgs_cache.end(), + [](const std::pair& l, const std::pair& r) + { + uint64_t left_index = l.second.index_in_background_sync_data; + uint64_t right_index = r.second.index_in_background_sync_data; + THROW_WALLET_EXCEPTION_IF( + (left_index < right_index && l.second.height > r.second.height) || + (left_index > right_index && l.second.height < r.second.height), + error::wallet_internal_error, "Unexpected background sync data order"); + return left_index < right_index; + }); + + // All txs in the background cache should have height >= sync start height, + // but not fatal if not + if (!sorted_bgs_cache.empty() && sorted_bgs_cache[0].second.height < background_sync_data.start_height) + MWARNING("First tx in background cache has height (" << sorted_bgs_cache[0].second.height << ") lower than sync start height (" << background_sync_data.start_height << ")"); + + // We want to process all background synced txs in order to make sure + // the wallet state updates correctly. First we remove all txs from the wallet + // from before the background sync start height, then re-process them in + // chronological order. The background cache should contain a superset of + // *all* the wallet's txs from after the background sync start height. + MDEBUG("Processing " << background_sync_data.txs.size() << " background synced txs starting from height " << background_sync_data.start_height); + detached_blockchain_data dbd = detach_blockchain(background_sync_data.start_height); + + for (const auto &bgs_tx : sorted_bgs_cache) + { + MDEBUG("Processing background synced tx " << bgs_tx.first); + + process_new_transaction(bgs_tx.first, bgs_tx.second.tx, bgs_tx.second.output_indices, bgs_tx.second.height, 0, bgs_tx.second.block_timestamp, + cryptonote::is_coinbase(bgs_tx.second.tx), false/*pool*/, bgs_tx.second.double_spend_seen, {}, {}, true/*ignore_callbacks*/); + + // Re-set destination addresses if they were previously set + if (m_confirmed_txs.find(bgs_tx.first) != m_confirmed_txs.end() && + dbd.detached_confirmed_txs_dests.find(bgs_tx.first) != dbd.detached_confirmed_txs_dests.end()) + { + m_confirmed_txs[bgs_tx.first].m_dests = std::move(dbd.detached_confirmed_txs_dests[bgs_tx.first]); + } + } + + m_blockchain = background_synced_chain; + m_last_block_reward = last_block_reward; + + MDEBUG("Finished processing background sync data"); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::reset_background_sync_data(background_sync_data_t &background_sync_data) +{ + background_sync_data.first_refresh_done = false; + background_sync_data.start_height = get_blockchain_current_height(); + background_sync_data.txs.clear(); + + background_sync_data.wallet_refresh_from_block_height = m_refresh_from_block_height; + background_sync_data.subaddress_lookahead_major = m_subaddress_lookahead_major; + background_sync_data.subaddress_lookahead_minor = m_subaddress_lookahead_minor; + background_sync_data.wallet_refresh_type = m_refresh_type; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::store_background_cache(const crypto::chacha_key &custom_background_key, const bool do_reset_background_sync_data) +{ + MDEBUG("Storing background cache (do_reset_background_sync_data=" << do_reset_background_sync_data << ")"); + + THROW_WALLET_EXCEPTION_IF(m_background_sync_type != BackgroundSyncCustomPassword, error::wallet_internal_error, + "Can only write a background cache when using a custom background password"); + THROW_WALLET_EXCEPTION_IF(m_wallet_file.empty(), error::wallet_internal_error, + "No wallet file known, can't store background cache"); + + std::unique_ptr background_w2(new wallet2(m_nettype)); + background_w2->prepare_file_names(make_background_wallet_file_name(m_wallet_file)); + + // Make sure background wallet is opened by this wallet + THROW_WALLET_EXCEPTION_IF(!lock_background_keys_file(background_w2->m_keys_file), + error::background_wallet_already_open, background_w2->m_wallet_file); + + // Load a background wallet2 instance using this wallet2 instance + std::string this_wallet2; + bool r = ::serialization::dump_binary(*this, this_wallet2); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to serialize wallet cache"); + + background_w2->clear(); + r = ::serialization::parse_binary(this_wallet2, *background_w2); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to deserialize wallet cache"); + + // Clear sensitive data from background cache not needed to sync + background_w2->clear_user_data(); + + background_w2->m_is_background_wallet = true; + if (do_reset_background_sync_data) + reset_background_sync_data(background_w2->m_background_sync_data); + else + background_w2->m_background_sync_data = m_background_sync_data; + background_w2->m_background_syncing = true; + + background_w2->m_custom_background_key = boost::optional(custom_background_key); + background_w2->m_background_sync_type = m_background_sync_type; + background_w2->store(); + + MDEBUG("Background cache stored (" << background_w2->m_transfers.size() << " transfers, " << background_w2->m_background_sync_data.txs.size() << " background synced txs)"); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::store_background_keys(const crypto::chacha_key &custom_background_key) +{ + MDEBUG("Storing background keys"); + + THROW_WALLET_EXCEPTION_IF(m_wallet_file.empty(), error::wallet_internal_error, + "No wallet file known, can't store background keys"); + + const std::string background_keys_file = make_background_keys_file_name(m_wallet_file); + bool r = store_keys(background_keys_file, custom_background_key, false/*watch_only*/, true/*background_keys_file*/); + THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, background_keys_file); + THROW_WALLET_EXCEPTION_IF(!is_background_keys_file_locked(), error::wallet_internal_error, background_keys_file + "\" should be locked"); + + // GUI uses the address file to differentiate non-mainnet wallets in the UI + const std::string background_address_file = make_background_wallet_file_name(m_wallet_file) + ".address.txt"; + if (m_nettype != MAINNET && !boost::filesystem::exists(background_address_file)) + { + r = save_to_file(background_address_file, m_account.get_public_address_str(m_nettype), true); + if (!r) MERROR("String with address text not saved"); + } + + MDEBUG("Background keys stored"); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::write_background_sync_wallet(const epee::wipeable_string &wallet_password, const epee::wipeable_string &background_cache_password) +{ + MDEBUG("Storing background sync wallet"); + THROW_WALLET_EXCEPTION_IF(m_background_sync_type != BackgroundSyncCustomPassword, error::wallet_internal_error, + "Can only write a background sync wallet when using a custom background password"); + THROW_WALLET_EXCEPTION_IF(m_background_syncing || m_is_background_wallet, error::wallet_internal_error, + "Can't write background sync wallet from an existing background cache"); + THROW_WALLET_EXCEPTION_IF(wallet_password == background_cache_password, + error::background_custom_password_same_as_wallet_password); + + // Set the background encryption key + crypto::chacha_key custom_background_key; + get_custom_background_key(background_cache_password, custom_background_key, m_kdf_rounds); + + // Keep the background encryption key in memory so the main wallet can update + // the background cache when it stores the main wallet cache + m_custom_background_key = boost::optional(custom_background_key); + + if (m_wallet_file.empty() || m_keys_file.empty()) + return; + + // Save background keys file, then background cache, then update main wallet settings + store_background_keys(custom_background_key); + store_background_cache(custom_background_key, true/*do_reset_background_sync_data*/); + bool r = store_keys(m_keys_file, wallet_password, false/*watch_only*/); + THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + + MDEBUG("Background sync wallet saved successfully"); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::setup_background_sync(BackgroundSyncType background_sync_type, const epee::wipeable_string &wallet_password, const boost::optional &background_cache_password) +{ + MDEBUG("Setting background sync to type " << background_sync_type); + THROW_WALLET_EXCEPTION_IF(m_background_syncing || m_is_background_wallet, error::wallet_internal_error, + "Can't set background sync type from an existing background cache"); + verify_password_with_cached_key(wallet_password); + + if (background_sync_type != BackgroundSyncOff) + validate_background_cache_password_usage(background_sync_type, background_cache_password, m_multisig, m_watch_only, key_on_device()); + + THROW_WALLET_EXCEPTION_IF(background_sync_type == BackgroundSyncCustomPassword && wallet_password == background_cache_password, + error::background_custom_password_same_as_wallet_password); + + if (m_background_sync_type == background_sync_type && background_sync_type != BackgroundSyncCustomPassword) + return; // No need to make any changes + + if (!m_wallet_file.empty()) + { + // Delete existing background files if they already exist + const std::string old_background_wallet_file = make_background_wallet_file_name(m_wallet_file); + const std::string old_background_keys_file = make_background_keys_file_name(m_wallet_file); + const std::string old_background_address_file = old_background_wallet_file + ".address.txt"; + + // Make sure no other program is using the background wallet + THROW_WALLET_EXCEPTION_IF(!lock_background_keys_file(old_background_keys_file), + error::background_wallet_already_open, old_background_wallet_file); + + if (boost::filesystem::exists(old_background_wallet_file)) + if (!boost::filesystem::remove(old_background_wallet_file)) + LOG_ERROR("Error deleting background wallet file: " << old_background_wallet_file); + + if (boost::filesystem::exists(old_background_keys_file)) + if (!boost::filesystem::remove(old_background_keys_file)) + LOG_ERROR("Error deleting background keys file: " << old_background_keys_file); + + if (boost::filesystem::exists(old_background_address_file)) + if (!boost::filesystem::remove(old_background_address_file)) + LOG_ERROR("Error deleting background address file: " << old_background_address_file); + } + + m_background_sync_type = background_sync_type; + m_custom_background_key = boost::none; + + // Write the new files + switch (background_sync_type) + { + case BackgroundSyncOff: + case BackgroundSyncReusePassword: rewrite(m_wallet_file, wallet_password); break; + case BackgroundSyncCustomPassword: write_background_sync_wallet(wallet_password, background_cache_password.get()); break; + default: THROW_WALLET_EXCEPTION(error::wallet_internal_error, "unknown background sync type"); + } + + MDEBUG("Done setting background sync type"); +} +//---------------------------------------------------------------------------------------------------- +/* + When background syncing, the wallet scans using just the view key, without + keeping the spend key in decrypted state. When a user returns to the wallet + and decrypts the spend key, the wallet processes the background synced txs, + then the wallet picks up scanning normally right where the background sync + left off. +*/ +void wallet2::start_background_sync() +{ + THROW_WALLET_EXCEPTION_IF(m_background_sync_type == BackgroundSyncOff, error::wallet_internal_error, + "must setup background sync first before using background sync"); + THROW_WALLET_EXCEPTION_IF(m_is_background_wallet, error::wallet_internal_error, + "Can't start background syncing from a background wallet (it is always background syncing)"); + + MDEBUG("Starting background sync"); + + if (m_background_syncing) + { + MDEBUG("Already background syncing"); + return; + } + + if (m_background_sync_type == BackgroundSyncCustomPassword && !m_wallet_file.empty()) + { + // Save the current state of the wallet cache. Only necessary when using a + // custom background password which uses distinct background wallet to sync. + // When reusing wallet password to sync we reuse the main wallet cache. + store(); + + // Wipe user data from the background wallet cache not needed to sync. + // Only wipe user data from background cache if wallet cache is stored + // on disk; otherwise we could lose the data. + clear_user_data(); + + // Wipe m_cache_key since it can be used to decrypt main wallet cache + m_cache_key.scrub(); + } + + reset_background_sync_data(m_background_sync_data); + m_background_syncing = true; + + // Wipe the spend key from memory + m_account.forget_spend_key(); + + MDEBUG("Background sync started at height " << m_background_sync_data.start_height); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::stop_background_sync(const epee::wipeable_string &wallet_password, const crypto::secret_key &spend_secret_key) +{ + MDEBUG("Stopping background sync"); + + // Verify provided password and spend secret key. If no spend secret key is + // provided, recover it from the wallet keys file + crypto::secret_key recovered_spend_key = crypto::null_skey; + if (!m_wallet_file.empty()) + { + THROW_WALLET_EXCEPTION_IF(!verify_password(wallet_password, recovered_spend_key), error::invalid_password); + } + else + { + verify_password_with_cached_key(wallet_password); + } + + if (spend_secret_key != crypto::null_skey) + { + THROW_WALLET_EXCEPTION_IF(!m_wallet_file.empty() && spend_secret_key != recovered_spend_key, + error::invalid_spend_key); + MDEBUG("Setting spend secret key with the provided key"); + recovered_spend_key = spend_secret_key; + } + + // Verify private spend key derives to wallet's public spend key + const auto verify_spend_key = [this](crypto::secret_key &recovered_spend_key) -> bool + { + crypto::public_key spend_public_key; + return recovered_spend_key != crypto::null_skey && + crypto::secret_key_to_public_key(recovered_spend_key, spend_public_key) && + m_account.get_keys().m_account_address.m_spend_public_key == spend_public_key; + }; + THROW_WALLET_EXCEPTION_IF(!verify_spend_key(recovered_spend_key), error::invalid_spend_key); + + THROW_WALLET_EXCEPTION_IF(m_background_sync_type == BackgroundSyncOff, error::wallet_internal_error, + "must setup background sync first before using background sync"); + THROW_WALLET_EXCEPTION_IF(m_is_background_wallet, error::wallet_internal_error, + "Can't stop background syncing from a background wallet"); + + if (!m_background_syncing) + return; + + // Copy background cache, we're about to overwrite it + const background_sync_data_t background_sync_data = m_background_sync_data; + const hashchain background_synced_chain = m_blockchain; + const uint64_t last_block_reward = m_last_block_reward; + + if (m_background_sync_type == BackgroundSyncCustomPassword && !m_wallet_file.empty()) + { + // Reload the wallet from disk + load(m_wallet_file, wallet_password); + THROW_WALLET_EXCEPTION_IF(!verify_spend_key(recovered_spend_key), error::invalid_spend_key); + } + m_background_syncing = false; + + // Set the plaintext spend key + m_account.set_spend_key(recovered_spend_key); + + // Encrypt the spend key when done if needed + epee::misc_utils::auto_scope_leave_caller keys_reencryptor; + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]{encrypt_keys(wallet_password);}); + + // Now we can use the decrypted spend key to process background cache + process_background_cache(background_sync_data, background_synced_chain, last_block_reward); + + // Reset the background cache after processing + reset_background_sync_data(m_background_sync_data); + + MDEBUG("Background sync stopped"); +} +//---------------------------------------------------------------------------------------------------- wallet2::payment_container wallet2::export_payments() const { payment_container payments; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index c38d77675..d648cd5d3 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -256,6 +256,20 @@ private: BackgroundMiningNo = 2, }; + enum BackgroundSyncType { + BackgroundSyncOff = 0, + BackgroundSyncReusePassword = 1, + BackgroundSyncCustomPassword = 2, + }; + + static BackgroundSyncType background_sync_type_from_str(const std::string &background_sync_type_str) + { + if (background_sync_type_str == "off") return BackgroundSyncOff; + if (background_sync_type_str == "reuse-wallet-password") return BackgroundSyncReusePassword; + if (background_sync_type_str == "custom-background-password") return BackgroundSyncCustomPassword; + throw std::logic_error("Unknown background sync type"); + }; + enum ExportFormat { Binary = 0, Ascii, @@ -282,7 +296,12 @@ private: //! Just parses variables. static std::unique_ptr make_dummy(const boost::program_options::variables_map& vm, bool unattended, const std::function(const char *, bool)> &password_prompter); - static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds); + static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds) + { + crypto::secret_key spend_key = crypto::null_skey; + return verify_password(keys_file_name, password, no_spend_key, hwdev, kdf_rounds, spend_key); + }; + static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds, crypto::secret_key &spend_key_out); static bool query_device(hw::device::device_type& device_type, const std::string& keys_file_name, const epee::wipeable_string& password, uint64_t kdf_rounds = 1); wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false, std::unique_ptr http_client_factory = std::unique_ptr(new net::http::client_factory())); @@ -792,6 +811,54 @@ private: END_SERIALIZE() }; + struct background_synced_tx_t + { + uint64_t index_in_background_sync_data; + cryptonote::transaction tx; + std::vector output_indices; + uint64_t height; + uint64_t block_timestamp; + bool double_spend_seen; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + VARINT_FIELD(index_in_background_sync_data) + + // prune tx; don't need to keep signature data + if (!tx.serialize_base(ar)) + return false; + + FIELD(output_indices) + VARINT_FIELD(height) + VARINT_FIELD(block_timestamp) + FIELD(double_spend_seen) + END_SERIALIZE() + }; + + struct background_sync_data_t + { + bool first_refresh_done = false; + uint64_t start_height = 0; + serializable_unordered_map txs; + + // Relevant wallet settings + uint64_t wallet_refresh_from_block_height; + size_t subaddress_lookahead_major; + size_t subaddress_lookahead_minor; + RefreshType wallet_refresh_type; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(first_refresh_done) + FIELD(start_height) + FIELD(txs) + FIELD(wallet_refresh_from_block_height) + VARINT_FIELD(subaddress_lookahead_major) + VARINT_FIELD(subaddress_lookahead_minor) + VARINT_FIELD(wallet_refresh_type) + END_SERIALIZE() + }; + typedef std::tuple get_outs_entry; struct parsed_block @@ -974,7 +1041,8 @@ private: /*! * \brief verifies given password is correct for default wallet keys file */ - bool verify_password(const epee::wipeable_string& password); + bool verify_password(const epee::wipeable_string& password) {crypto::secret_key key = crypto::null_skey; return verify_password(password, key);}; + bool verify_password(const epee::wipeable_string& password, crypto::secret_key &spend_key_out); cryptonote::account_base& get_account(){return m_account;} const cryptonote::account_base& get_account()const{return m_account;} @@ -1069,6 +1137,7 @@ private: cryptonote::network_type nettype() const { return m_nettype; } bool watch_only() const { return m_watch_only; } bool multisig(bool *ready = NULL, uint32_t *threshold = NULL, uint32_t *total = NULL) const; + bool is_background_wallet() const { return m_is_background_wallet; } bool has_multisig_partial_key_images() const; bool has_unknown_key_images() const; bool get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase = std::string()) const; @@ -1276,11 +1345,17 @@ private: return; } a & m_has_ever_refreshed_from_node; + if(ver < 31) + { + m_background_sync_data = background_sync_data_t{}; + return; + } + a & m_background_sync_data; } BEGIN_SERIALIZE_OBJECT() MAGIC_FIELD("monero wallet cache") - VERSION_FIELD(1) + VERSION_FIELD(2) FIELD(m_blockchain) FIELD(m_transfers) FIELD(m_account_public_address) @@ -1312,6 +1387,12 @@ private: return true; } FIELD(m_has_ever_refreshed_from_node) + if (version < 2) + { + m_background_sync_data = background_sync_data_t{}; + return true; + } + FIELD(m_background_sync_data) END_SERIALIZE() /*! @@ -1327,6 +1408,8 @@ private: * \return Whether path is valid format */ static bool wallet_valid_path_format(const std::string& file_path); + static std::string make_background_wallet_file_name(const std::string &wallet_file); + static std::string make_background_keys_file_name(const std::string &wallet_file); static bool parse_long_payment_id(const std::string& payment_id_str, crypto::hash& payment_id); static bool parse_short_payment_id(const std::string& payment_id_str, crypto::hash8& payment_id); static bool parse_payment_id(const std::string& payment_id_str, crypto::hash& payment_id); @@ -1375,6 +1458,9 @@ private: void ignore_outputs_below(uint64_t value) { m_ignore_outputs_below = value; } bool track_uses() const { return m_track_uses; } void track_uses(bool value) { m_track_uses = value; } + BackgroundSyncType background_sync_type() const { return m_background_sync_type; } + void setup_background_sync(BackgroundSyncType background_sync_type, const epee::wipeable_string &wallet_password, const boost::optional &background_cache_password); + bool is_background_syncing() const { return m_background_syncing; } bool show_wallet_name_when_locked() const { return m_show_wallet_name_when_locked; } void show_wallet_name_when_locked(bool value) { m_show_wallet_name_when_locked = value; } BackgroundMiningSetupType setup_background_mining() const { return m_setup_background_mining; } @@ -1689,6 +1775,9 @@ private: uint64_t get_bytes_sent() const; uint64_t get_bytes_received() const; + void start_background_sync(); + void stop_background_sync(const epee::wipeable_string &wallet_password, const crypto::secret_key &spend_secret_key = crypto::null_skey); + // MMS ------------------------------------------------------------------------------------------------- mms::message_store& get_message_store() { return m_message_store; }; const mms::message_store& get_message_store() const { return m_message_store; }; @@ -1724,6 +1813,9 @@ private: * \return Whether it was successful. */ bool store_keys(const std::string& keys_file_name, const epee::wipeable_string& password, bool watch_only = false); + bool store_keys(const std::string& keys_file_name, const crypto::chacha_key& key, bool watch_only = false, bool background_keys_file = false); + boost::optional get_keys_file_data(const crypto::chacha_key& key, bool watch_only = false, bool background_keys_file = false); + bool store_keys_file_data(const std::string& keys_file_name, wallet2::keys_file_data &keys_file_data, bool background_keys_file = false); /*! * \brief Load wallet keys information from wallet file. * \param keys_file_name Name of wallet file @@ -1737,6 +1829,7 @@ private: */ bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password); bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional& keys_to_encrypt); + void load_wallet_cache(const bool use_fs, const std::string& cache_buf = ""); void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map, size_t> *output_tracker_cache = NULL, bool ignore_callbacks = false); bool should_skip_block(const cryptonote::block &b, uint64_t height) const; void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector &tx_cache_data, size_t tx_cache_data_offset, std::map, size_t> *output_tracker_cache = NULL); @@ -1745,6 +1838,15 @@ private: void get_short_chain_history(std::list& ids, uint64_t granularity = 1) const; bool clear(); void clear_soft(bool keep_key_images=false); + /* + * clear_user_data clears data created by the user, which is mostly data + * that a view key cannot identify on chain. This function was initially + * added to ensure that a "background" wallet (a wallet that syncs with just + * a view key hot in memory) does not have any sensitive data loaded that it + * does not need in order to sync. Future devs should take care to ensure + * that this function deletes data that is not useful for background syncing + */ + void clear_user_data(); void pull_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::vector &blocks, std::vector &o_indices, uint64_t ¤t_height); void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::vector &hashes); void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list &short_chain_history, bool force = false); @@ -1796,10 +1898,23 @@ private: bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector &outs); crypto::chacha_key get_ringdb_key(); void setup_keys(const epee::wipeable_string &password); + const crypto::chacha_key get_cache_key(); + void verify_password_with_cached_key(const epee::wipeable_string &password); + void verify_password_with_cached_key(const crypto::chacha_key &key); size_t get_transfer_details(const crypto::key_image &ki) const; tx_entry_data get_tx_entries(const std::unordered_set &txids); void sort_scan_tx_entries(std::vector &unsorted_tx_entries); void process_scan_txs(const tx_entry_data &txs_to_scan, const tx_entry_data &txs_to_reprocess, const std::unordered_set &tx_hashes_to_reprocess, detached_blockchain_data &dbd); + void write_background_sync_wallet(const epee::wipeable_string &wallet_password, const epee::wipeable_string &background_cache_password); + void process_background_cache_on_open(); + void process_background_cache(const background_sync_data_t &background_sync_data, const hashchain &background_chain, uint64_t last_block_reward); + void reset_background_sync_data(background_sync_data_t &background_sync_data); + void store_background_cache(const crypto::chacha_key &custom_background_key, const bool do_reset_background_sync_data = true); + void store_background_keys(const crypto::chacha_key &custom_background_key); + + bool lock_background_keys_file(const std::string &background_keys_file); + bool unlock_background_keys_file(); + bool is_background_keys_file_locked() const; void register_devices(); hw::device& lookup_device(const std::string & device_descriptor); @@ -1914,6 +2029,8 @@ private: uint64_t m_ignore_outputs_above; uint64_t m_ignore_outputs_below; bool m_track_uses; + bool m_is_background_wallet; + BackgroundSyncType m_background_sync_type; bool m_show_wallet_name_when_locked; uint32_t m_inactivity_lock_timeout; BackgroundMiningSetupType m_setup_background_mining; @@ -1959,6 +2076,7 @@ private: uint64_t m_last_block_reward; std::unique_ptr m_keys_file_locker; + std::unique_ptr m_background_keys_file_locker; mms::message_store m_message_store; bool m_original_keys_available; @@ -1966,6 +2084,7 @@ private: crypto::secret_key m_original_view_secret_key; crypto::chacha_key m_cache_key; + boost::optional m_custom_background_key = boost::none; std::shared_ptr m_encrypt_keys_after_refresh; bool m_unattended; @@ -1981,9 +2100,13 @@ private: static boost::mutex default_daemon_address_lock; static std::string default_daemon_address; + + bool m_background_syncing; + bool m_processing_background_cache; + background_sync_data_t m_background_sync_data; }; } -BOOST_CLASS_VERSION(tools::wallet2, 30) +BOOST_CLASS_VERSION(tools::wallet2, 31) BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 12) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) @@ -1999,6 +2122,8 @@ BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 1) BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 4) BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3) BOOST_CLASS_VERSION(tools::wallet2::multisig_sig, 1) +BOOST_CLASS_VERSION(tools::wallet2::background_synced_tx_t, 0) +BOOST_CLASS_VERSION(tools::wallet2::background_sync_data_t, 0) namespace boost { @@ -2497,6 +2622,29 @@ namespace boost return; a & x.multisig_sigs; } + + template + inline void serialize(Archive& a, tools::wallet2::background_synced_tx_t &x, const boost::serialization::version_type ver) + { + a & x.index_in_background_sync_data; + a & x.tx; + a & x.output_indices; + a & x.height; + a & x.block_timestamp; + a & x.double_spend_seen; + } + + template + inline void serialize(Archive& a, tools::wallet2::background_sync_data_t &x, const boost::serialization::version_type ver) + { + a & x.first_refresh_done; + a & x.start_height; + a & x.txs.parent(); + a & x.wallet_refresh_from_block_height; + a & x.subaddress_lookahead_major; + a & x.subaddress_lookahead_minor; + a & x.wallet_refresh_type; + } } } diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index c077313d4..c54cd3499 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -63,6 +63,7 @@ namespace tools // invalid_password // invalid_priority // invalid_multisig_seed + // invalid_spend_key // refresh_error * // acc_outs_lookup_error // block_parse_error @@ -97,6 +98,9 @@ namespace tools // wallet_files_doesnt_correspond // scan_tx_error * // wont_reprocess_recent_txs_via_untrusted_daemon + // background_sync_error * + // background_wallet_already_open + // background_custom_password_same_as_wallet_password // // * - class with protected ctor @@ -304,6 +308,16 @@ namespace tools std::string to_string() const { return wallet_logic_error::to_string(); } }; + struct invalid_spend_key : public wallet_logic_error + { + explicit invalid_spend_key(std::string&& loc) + : wallet_logic_error(std::move(loc), "invalid spend key") + { + } + + std::string to_string() const { return wallet_logic_error::to_string(); } + }; + //---------------------------------------------------------------------------------------------------- struct invalid_pregenerated_random : public wallet_logic_error { @@ -947,6 +961,31 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- + struct background_sync_error : public wallet_logic_error + { + protected: + explicit background_sync_error(std::string&& loc, const std::string& message) + : wallet_logic_error(std::move(loc), message) + { + } + }; + //---------------------------------------------------------------------------------------------------- + struct background_wallet_already_open : public background_sync_error + { + explicit background_wallet_already_open(std::string&& loc, const std::string& background_wallet_file) + : background_sync_error(std::move(loc), "background wallet " + background_wallet_file + " is already opened by another wallet program") + { + } + }; + //---------------------------------------------------------------------------------------------------- + struct background_custom_password_same_as_wallet_password : public background_sync_error + { + explicit background_custom_password_same_as_wallet_password(std::string&& loc) + : background_sync_error(std::move(loc), "custom background password must be different than wallet password") + { + } + }; + //---------------------------------------------------------------------------------------------------- #if !defined(_MSC_VER) diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index b1419949f..d24b4c563 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -73,6 +73,54 @@ using namespace epee; } \ } while(0) +#define CHECK_IF_BACKGROUND_SYNCING() \ + do \ + { \ + if (!m_wallet) { return not_open(er); } \ + if (m_wallet->is_background_wallet()) \ + { \ + er.code = WALLET_RPC_ERROR_CODE_IS_BACKGROUND_WALLET; \ + er.message = "This command is disabled for background wallets."; \ + return false; \ + } \ + if (m_wallet->is_background_syncing()) \ + { \ + er.code = WALLET_RPC_ERROR_CODE_IS_BACKGROUND_SYNCING; \ + er.message = "This command is disabled while background syncing. Stop background syncing to use this command."; \ + return false; \ + } \ + } while(0) + +#define PRE_VALIDATE_BACKGROUND_SYNC() \ + do \ + { \ + if (!m_wallet) { return not_open(er); } \ + if (m_restricted) \ + { \ + er.code = WALLET_RPC_ERROR_CODE_DENIED; \ + er.message = "Command unavailable in restricted mode."; \ + return false; \ + } \ + if (m_wallet->key_on_device()) \ + { \ + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; \ + er.message = "Command not supported by HW wallet"; \ + return false; \ + } \ + if (m_wallet->multisig()) \ + { \ + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; \ + er.message = "Multisig wallet cannot enable background sync"; \ + return false; \ + } \ + if (m_wallet->watch_only()) \ + { \ + er.code = WALLET_RPC_ERROR_CODE_WATCH_ONLY; \ + er.message = "Watch-only wallet cannot enable background sync"; \ + return false; \ + } \ + } while (0) + namespace { const command_line::arg_descriptor arg_rpc_bind_port = {"rpc-bind-port", "Sets bind port for server"}; @@ -291,6 +339,9 @@ namespace tools { if (!m_wallet) return; + // Background mining can be toggled from the main wallet + if (m_wallet->is_background_wallet() || m_wallet->is_background_syncing()) + return; tools::wallet2::BackgroundMiningSetupType setup = m_wallet->setup_background_mining(); if (setup == tools::wallet2::BackgroundMiningNo) @@ -582,6 +633,7 @@ namespace tools bool wallet_rpc_server::on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { if (req.count < 1 || req.count > 64) { @@ -619,6 +671,7 @@ namespace tools bool wallet_rpc_server::on_label_address(const wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { m_wallet->set_subaddress_label(req.index, req.label); @@ -681,6 +734,7 @@ namespace tools bool wallet_rpc_server::on_create_account(const wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { m_wallet->add_subaddress_account(req.label); @@ -698,6 +752,7 @@ namespace tools bool wallet_rpc_server::on_label_account(const wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { m_wallet->set_subaddress_label({req.account_index, 0}, req.label); @@ -713,6 +768,7 @@ namespace tools bool wallet_rpc_server::on_get_account_tags(const wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); const std::pair, std::vector> account_tags = m_wallet->get_account_tags(); for (const std::pair& p : account_tags.first) { @@ -732,6 +788,7 @@ namespace tools bool wallet_rpc_server::on_tag_accounts(const wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { m_wallet->set_account_tag(req.accounts, req.tag); @@ -747,6 +804,7 @@ namespace tools bool wallet_rpc_server::on_untag_accounts(const wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { m_wallet->set_account_tag(req.accounts, ""); @@ -762,6 +820,7 @@ namespace tools bool wallet_rpc_server::on_set_account_tag_description(const wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::request& req, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { m_wallet->set_account_tag_description(req.tag, req.description); @@ -792,6 +851,7 @@ namespace tools bool wallet_rpc_server::on_freeze(const wallet_rpc::COMMAND_RPC_FREEZE::request& req, wallet_rpc::COMMAND_RPC_FREEZE::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { if (req.key_image.empty()) @@ -820,6 +880,7 @@ namespace tools bool wallet_rpc_server::on_thaw(const wallet_rpc::COMMAND_RPC_THAW::request& req, wallet_rpc::COMMAND_RPC_THAW::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { if (req.key_image.empty()) @@ -848,6 +909,7 @@ namespace tools bool wallet_rpc_server::on_frozen(const wallet_rpc::COMMAND_RPC_FROZEN::request& req, wallet_rpc::COMMAND_RPC_FROZEN::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { if (req.key_image.empty()) @@ -875,6 +937,8 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::validate_transfer(const std::list& destinations, const std::string& payment_id, std::vector& dsts, std::vector& extra, bool at_least_one_destination, epee::json_rpc::error& er) { + CHECK_IF_BACKGROUND_SYNCING(); + crypto::hash8 integrated_payment_id = crypto::null_hash8; std::string extra_nonce; for (auto it = destinations.begin(); it != destinations.end(); it++) @@ -1204,6 +1268,7 @@ namespace tools } CHECK_MULTISIG_ENABLED(); + CHECK_IF_BACKGROUND_SYNCING(); cryptonote::blobdata blob; if (!epee::string_tools::parse_hexstr_to_binbuff(req.unsigned_txset, blob)) @@ -1285,6 +1350,7 @@ namespace tools er.message = "command not supported by watch-only wallet"; return false; } + CHECK_IF_BACKGROUND_SYNCING(); if(req.unsigned_txset.empty() && req.multisig_txset.empty()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; @@ -1554,6 +1620,7 @@ namespace tools } CHECK_MULTISIG_ENABLED(); + CHECK_IF_BACKGROUND_SYNCING(); try { @@ -2115,6 +2182,7 @@ namespace tools er.message = "The wallet is watch-only. Cannot retrieve seed."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); if (!m_wallet->is_deterministic()) { er.code = WALLET_RPC_ERROR_CODE_NON_DETERMINISTIC; @@ -2143,6 +2211,7 @@ namespace tools er.message = "The wallet is watch-only. Cannot retrieve spend key."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); epee::wipeable_string key = epee::to_hex::wipeable_string(m_wallet->get_account().get_keys().m_spend_secret_key); res.key = std::string(key.data(), key.size()); } @@ -2164,6 +2233,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); try { @@ -2177,6 +2247,79 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_setup_background_sync(const wallet_rpc::COMMAND_RPC_SETUP_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_SETUP_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + try + { + PRE_VALIDATE_BACKGROUND_SYNC(); + const tools::wallet2::BackgroundSyncType background_sync_type = tools::wallet2::background_sync_type_from_str(req.background_sync_type); + boost::optional background_cache_password = boost::none; + if (background_sync_type == tools::wallet2::BackgroundSyncCustomPassword) + background_cache_password = boost::optional(req.background_cache_password); + m_wallet->setup_background_sync(background_sync_type, req.wallet_password, background_cache_password); + } + catch (...) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_start_background_sync(const wallet_rpc::COMMAND_RPC_START_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_START_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + try + { + PRE_VALIDATE_BACKGROUND_SYNC(); + m_wallet->start_background_sync(); + } + catch (...) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_stop_background_sync(const wallet_rpc::COMMAND_RPC_STOP_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_STOP_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + try + { + PRE_VALIDATE_BACKGROUND_SYNC(); + crypto::secret_key spend_secret_key = crypto::null_skey; + + // Load the spend key from seed if seed is provided + if (!req.seed.empty()) + { + crypto::secret_key recovery_key; + std::string language; + + if (!crypto::ElectrumWords::words_to_bytes(req.seed, recovery_key, language)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Electrum-style word list failed verification"; + return false; + } + + if (!req.seed_offset.empty()) + recovery_key = cryptonote::decrypt_key(recovery_key, req.seed_offset); + + // generate spend key + cryptonote::account_base account; + account.generate(recovery_key, true, false); + spend_secret_key = account.get_keys().m_spend_secret_key; + } + + m_wallet->stop_background_sync(req.wallet_password, spend_secret_key); + } + catch (...) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_sign(const wallet_rpc::COMMAND_RPC_SIGN::request& req, wallet_rpc::COMMAND_RPC_SIGN::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2186,6 +2329,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); tools::wallet2::message_signature_type_t signature_type = tools::wallet2::sign_with_spend_key; if (req.signature_type == "spend" || req.signature_type == "") @@ -2278,6 +2422,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); if (req.txids.size() != req.notes.size()) { @@ -2350,6 +2495,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); m_wallet->set_attribute(req.key, req.value); @@ -2377,6 +2523,7 @@ namespace tools bool wallet_rpc_server::on_get_tx_key(const wallet_rpc::COMMAND_RPC_GET_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_GET_TX_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); crypto::hash txid; if (!epee::string_tools::hex_to_pod(req.txid, txid)) @@ -2468,6 +2615,7 @@ namespace tools bool wallet_rpc_server::on_get_tx_proof(const wallet_rpc::COMMAND_RPC_GET_TX_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_TX_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); crypto::hash txid; if (!epee::string_tools::hex_to_pod(req.txid, txid)) @@ -2584,6 +2732,7 @@ namespace tools bool wallet_rpc_server::on_get_reserve_proof(const wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); boost::optional> account_minreserve; if (!req.all) @@ -2826,6 +2975,7 @@ namespace tools er.message = "command not supported by HW wallet"; return false; } + CHECK_IF_BACKGROUND_SYNCING(); try { @@ -2855,6 +3005,7 @@ namespace tools er.message = "command not supported by HW wallet"; return false; } + CHECK_IF_BACKGROUND_SYNCING(); cryptonote::blobdata blob; if (!epee::string_tools::parse_hexstr_to_binbuff(req.outputs_data_hex, blob)) @@ -2880,6 +3031,7 @@ namespace tools bool wallet_rpc_server::on_export_key_images(const wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); try { std::pair>> ski = m_wallet->export_key_images(req.all); @@ -2916,6 +3068,7 @@ namespace tools er.message = "This command requires a trusted daemon."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); try { std::vector> ski; @@ -2984,6 +3137,7 @@ namespace tools bool wallet_rpc_server::on_get_address_book(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); + CHECK_IF_BACKGROUND_SYNCING(); const auto ab = m_wallet->get_address_book(); if (req.entries.empty()) { @@ -3029,6 +3183,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); cryptonote::address_parse_info info; er.message = ""; @@ -3071,6 +3226,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); const auto ab = m_wallet->get_address_book(); if (req.index >= ab.size()) @@ -3133,6 +3289,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); const auto ab = m_wallet->get_address_book(); if (req.index >= ab.size()) @@ -3203,6 +3360,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); std::unordered_set txids; std::list::const_iterator i = req.txids.begin(); @@ -3242,6 +3400,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); try { m_wallet->rescan_spent(); @@ -3506,6 +3665,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } + CHECK_IF_BACKGROUND_SYNCING(); if (m_wallet->verify_password(req.old_password)) { try @@ -4033,6 +4193,7 @@ namespace tools er.message = "wallet is watch-only and cannot be made multisig"; return false; } + CHECK_IF_BACKGROUND_SYNCING(); res.multisig_info = m_wallet->get_multisig_first_kex_msg(); return true; @@ -4060,6 +4221,7 @@ namespace tools er.message = "wallet is watch-only and cannot be made multisig"; return false; } + CHECK_IF_BACKGROUND_SYNCING(); try { diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 3308d1751..c2329aafe 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -160,6 +160,9 @@ namespace tools MAP_JON_RPC_WE("set_log_categories", on_set_log_categories, wallet_rpc::COMMAND_RPC_SET_LOG_CATEGORIES) MAP_JON_RPC_WE("estimate_tx_size_and_weight", on_estimate_tx_size_and_weight, wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT) MAP_JON_RPC_WE("get_version", on_get_version, wallet_rpc::COMMAND_RPC_GET_VERSION) + MAP_JON_RPC_WE("setup_background_sync", on_setup_background_sync, wallet_rpc::COMMAND_RPC_SETUP_BACKGROUND_SYNC) + MAP_JON_RPC_WE("start_background_sync", on_start_background_sync, wallet_rpc::COMMAND_RPC_START_BACKGROUND_SYNC) + MAP_JON_RPC_WE("stop_background_sync", on_stop_background_sync, wallet_rpc::COMMAND_RPC_STOP_BACKGROUND_SYNC) END_JSON_RPC_MAP() END_URI_MAP2() @@ -251,6 +254,9 @@ namespace tools bool on_set_log_categories(const wallet_rpc::COMMAND_RPC_SET_LOG_CATEGORIES::request& req, wallet_rpc::COMMAND_RPC_SET_LOG_CATEGORIES::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_estimate_tx_size_and_weight(const wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT::request& req, wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_get_version(const wallet_rpc::COMMAND_RPC_GET_VERSION::request& req, wallet_rpc::COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_setup_background_sync(const wallet_rpc::COMMAND_RPC_SETUP_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_SETUP_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_start_background_sync(const wallet_rpc::COMMAND_RPC_START_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_START_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_stop_background_sync(const wallet_rpc::COMMAND_RPC_STOP_BACKGROUND_SYNC::request& req, wallet_rpc::COMMAND_RPC_STOP_BACKGROUND_SYNC::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); //json rpc v2 bool on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index b6098d95c..a44b56ed6 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -2696,5 +2696,69 @@ namespace wallet_rpc typedef epee::misc_utils::struct_init response; }; + struct COMMAND_RPC_SETUP_BACKGROUND_SYNC + { + struct request_t + { + std::string background_sync_type; + std::string wallet_password; + std::string background_cache_password; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(background_sync_type) + KV_SERIALIZE(wallet_password) + KV_SERIALIZE_OPT(background_cache_password, (std::string)"") + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init response; + }; + + struct COMMAND_RPC_START_BACKGROUND_SYNC + { + struct request_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init response; + }; + + struct COMMAND_RPC_STOP_BACKGROUND_SYNC + { + struct request_t + { + std::string wallet_password; + std::string seed; + std::string seed_offset; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(wallet_password) + KV_SERIALIZE_OPT(seed, (std::string)"") + KV_SERIALIZE_OPT(seed_offset, (std::string)"") + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init response; + }; } } diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index 541d29f86..4756c191c 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -81,3 +81,5 @@ #define WALLET_RPC_ERROR_CODE_DISABLED -48 #define WALLET_RPC_ERROR_CODE_PROXY_ALREADY_DEFINED -49 #define WALLET_RPC_ERROR_CODE_NONZERO_UNLOCK_TIME -50 +#define WALLET_RPC_ERROR_CODE_IS_BACKGROUND_WALLET -51 +#define WALLET_RPC_ERROR_CODE_IS_BACKGROUND_SYNCING -52 diff --git a/tests/functional_tests/transfer.py b/tests/functional_tests/transfer.py index 4063911f4..60eb09a10 100755 --- a/tests/functional_tests/transfer.py +++ b/tests/functional_tests/transfer.py @@ -30,6 +30,7 @@ from __future__ import print_function import json +import util_resources import pprint from deepdiff import DeepDiff pp = pprint.PrettyPrinter(indent=2) @@ -46,6 +47,17 @@ seeds = [ 'dilute gutter certain antics pamphlet macro enjoy left slid guarded bogeys upload nineteen bomb jubilee enhanced irritate turnip eggs swung jukebox loudly reduce sedan slid', ] +def diff_transfers(actual_transfers, expected_transfers, ignore_order = True): + # The payments containers aren't ordered; re-scanning can lead to diff orders + diff = DeepDiff(actual_transfers, expected_transfers, ignore_order = ignore_order) + if diff != {}: + pp.pprint(diff) + assert diff == {} + +def diff_incoming_transfers(actual_transfers, expected_transfers): + # wallet2 m_transfers container is ordered and order should be the same across rescans + diff_transfers(actual_transfers, expected_transfers, ignore_order = False) + class TransferTest(): def run_test(self): self.reset() @@ -63,6 +75,8 @@ class TransferTest(): self.check_is_key_image_spent() self.check_scan_tx() self.check_subtract_fee_from_outputs() + self.check_background_sync() + self.check_background_sync_reorg_recovery() def reset(self): print('Resetting blockchain') @@ -840,12 +854,6 @@ class TransferTest(): print('Testing scan_tx') - def diff_transfers(actual_transfers, expected_transfers): - diff = DeepDiff(actual_transfers, expected_transfers) - if diff != {}: - pp.pprint(diff) - assert diff == {} - # set up sender_wallet sender_wallet = self.wallet[0] try: sender_wallet.close_wallet() @@ -1127,5 +1135,385 @@ class TransferTest(): except AssertionError: pass + def check_background_sync(self): + daemon = Daemon() + + print('Testing background sync') + + # Some helper functions + def stop_with_wrong_inputs(wallet, wallet_password, seed = ''): + invalid = False + try: wallet.stop_background_sync(wallet_password = wallet_password, seed = seed) + except: invalid = True + assert invalid + + def open_with_wrong_password(wallet, filename, password): + invalid_password = False + try: wallet.open_wallet(filename, password = password) + except: invalid_password = True + assert invalid_password + + def restore_wallet(wallet, seed, filename = '', password = ''): + wallet.close_wallet() + if filename != '': + util_resources.remove_wallet_files(filename) + wallet.restore_deterministic_wallet(seed = seed, filename = filename, password = password) + wallet.auto_refresh(enable = False) + assert wallet.get_transfers() == {} + + def assert_correct_transfers(wallet, expected_transfers, expected_inc_transfers, expected_balance): + diff_transfers(wallet.get_transfers(), expected_transfers) + diff_incoming_transfers(wallet.incoming_transfers(transfer_type = 'all'), expected_inc_transfers) + assert wallet.get_balance().balance == expected_balance + + # Set up sender_wallet. Prepare to sweep single output to receiver. + # We're testing a sweep because it makes sure background sync can + # properly pick up txs which do not have a change output back to sender. + sender_wallet = self.wallet[0] + try: sender_wallet.close_wallet() + except: pass + sender_wallet.restore_deterministic_wallet(seed = seeds[0]) + sender_wallet.auto_refresh(enable = False) + sender_wallet.refresh() + res = sender_wallet.incoming_transfers(transfer_type = 'available') + unlocked = [x for x in res.transfers if x.unlocked and x.amount > 0] + assert len(unlocked) > 0 + ki = unlocked[0].key_image + amount = unlocked[0].amount + spent_txid = unlocked[0].tx_hash + sender_wallet.refresh() + res = sender_wallet.get_transfers() + out_len = 0 if 'out' not in res else len(res.out) + sender_starting_balance = sender_wallet.get_balance().balance + + # Background sync type options + reuse_password = sender_wallet.background_sync_options.reuse_password + custom_password = sender_wallet.background_sync_options.custom_password + + # set up receiver_wallet + receiver_wallet = self.wallet[1] + try: receiver_wallet.close_wallet() + except: pass + receiver_wallet.restore_deterministic_wallet(seed = seeds[1]) + receiver_wallet.auto_refresh(enable = False) + receiver_wallet.refresh() + res = receiver_wallet.get_transfers() + in_len = 0 if 'in' not in res else len(res['in']) + receiver_starting_balance = receiver_wallet.get_balance().balance + + # transfer from sender_wallet to receiver_wallet + dst = '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW' + res = sender_wallet.sweep_single(dst, key_image = ki) + assert len(res.tx_hash) == 32*2 + txid = res.tx_hash + assert res.fee > 0 + fee = res.fee + assert res.amount == amount - fee + + expected_sender_balance = sender_starting_balance - amount + expected_receiver_balance = receiver_starting_balance + (amount - fee) + + print('Checking background sync on outgoing wallet') + sender_wallet.setup_background_sync(background_sync_type = reuse_password) + sender_wallet.start_background_sync() + # Mine block to an uninvolved wallet + daemon.generateblocks('46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', 1) + # sender should still be able to scan the transfer normally because we + # spent an output that had a known key image + sender_wallet.refresh() + transfers = sender_wallet.get_transfers() + assert 'pending' not in transfers or len(transfers.pending) == 0 + assert 'pool' not in transfers or len (transfers.pool) == 0 + assert len(transfers.out) == out_len + 1 + tx = [x for x in transfers.out if x.txid == txid] + assert len(tx) == 1 + tx = tx[0] + assert tx.amount == amount - fee + assert tx.fee == fee + assert len(tx.destinations) == 1 + assert tx.destinations[0].amount == amount - fee + assert tx.destinations[0].address == dst + incoming_transfers = sender_wallet.incoming_transfers(transfer_type = 'all') + assert len([x for x in incoming_transfers.transfers if x.tx_hash == spent_txid and x.key_image == ki and x.spent]) == 1 + assert sender_wallet.get_balance().balance == expected_sender_balance + + # Restore and check background syncing outgoing wallet + restore_wallet(sender_wallet, seeds[0]) + sender_wallet.setup_background_sync(background_sync_type = reuse_password) + sender_wallet.start_background_sync() + sender_wallet.refresh() + for i, out_tx in enumerate(transfers.out): + if 'destinations' in out_tx: + del transfers.out[i]['destinations'] # destinations are not expected after wallet restore + # sender's balance should be higher because can't detect spends while + # background sync enabled, only receives + background_bal = sender_wallet.get_balance().balance + assert background_bal > expected_sender_balance + background_transfers = sender_wallet.get_transfers() + assert 'out' not in background_transfers or len(background_transfers.out) == 0 + assert 'in' in background_transfers and len(background_transfers['in']) > 0 + background_incoming_transfers = sender_wallet.incoming_transfers(transfer_type = 'all') + assert len(background_incoming_transfers) == len(incoming_transfers) + assert len([x for x in background_incoming_transfers.transfers if x.spent or x.key_image != '']) == 0 + assert len([x for x in background_incoming_transfers.transfers if x.tx_hash == spent_txid]) == 1 + + # Try to stop background sync with the wrong seed + stop_with_wrong_inputs(sender_wallet, wallet_password = '', seed = seeds[1]) + + # Stop background sync and check transfers update correctly + sender_wallet.stop_background_sync(wallet_password = '', seed = seeds[0]) + assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance) + + # Check stopping a wallet with wallet files saved to disk + for background_sync_type in [reuse_password, custom_password]: + restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password') + background_cache_password = None if background_sync_type == reuse_password else 'background_password' + sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = 'test_password', background_cache_password = background_cache_password) + sender_wallet.start_background_sync() + sender_wallet.refresh() + assert_correct_transfers(sender_wallet, background_transfers, background_incoming_transfers, background_bal) + stop_with_wrong_inputs(sender_wallet, 'wrong_password') + sender_wallet.stop_background_sync(wallet_password = 'test_password') + assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance) + + # Close wallet while background syncing, then reopen + for background_sync_type in [reuse_password, custom_password]: + restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password') + background_cache_password = None if background_sync_type == reuse_password else 'background_password' + sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = 'test_password', background_cache_password = background_cache_password) + sender_wallet.start_background_sync() + sender_wallet.refresh() + assert_correct_transfers(sender_wallet, background_transfers, background_incoming_transfers, background_bal) + sender_wallet.close_wallet() + open_with_wrong_password(sender_wallet, 'test1', 'wrong_password') + sender_wallet.open_wallet('test1', password = 'test_password') + # It should reopen with spend key loaded and correctly scan all transfers + assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance) + + # Close wallet while syncing normally, then reopen + for background_sync_type in [reuse_password, custom_password]: + restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password') + background_cache_password = None if background_sync_type == reuse_password else 'background_password' + sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = 'test_password', background_cache_password = background_cache_password) + sender_wallet.refresh() + assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance) + sender_wallet.close_wallet() + open_with_wrong_password(sender_wallet, 'test1', 'wrong_password') + sender_wallet.open_wallet('test1', password = 'test_password') + assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance) + + # Create background cache using custom password, then use it to sync, then reopen main wallet + for background_cache_password in ['background_password', '']: + restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password') + assert not util_resources.file_exists('test1.background') + assert not util_resources.file_exists('test1.background.keys') + sender_wallet.setup_background_sync(background_sync_type = custom_password, wallet_password = 'test_password', background_cache_password = background_cache_password) + assert util_resources.file_exists('test1.background') + assert util_resources.file_exists('test1.background.keys') + sender_wallet.close_wallet() + open_with_wrong_password(sender_wallet, 'test1.background', 'test_password') + sender_wallet.open_wallet('test1.background', password = background_cache_password) + sender_wallet.refresh() + assert_correct_transfers(sender_wallet, background_transfers, background_incoming_transfers, background_bal) + sender_wallet.close_wallet() + sender_wallet.open_wallet('test1', password = 'test_password') + assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance) + + # Check that main wallet keeps background cache encrypted with custom password in sync + restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password') + sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = 'test_password', background_cache_password = 'background_password') + sender_wallet.refresh() + assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance) + sender_wallet.close_wallet() + sender_wallet.open_wallet('test1.background', password = 'background_password') + assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance) + + # Try using wallet password as custom background password + restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password') + assert not util_resources.file_exists('test1.background') + assert not util_resources.file_exists('test1.background.keys') + same_password = False + try: sender_wallet.setup_background_sync(background_sync_type = custom_password, wallet_password = 'test_password', background_cache_password = 'test_password') + except: same_password = True + assert same_password + assert not util_resources.file_exists('test1.background') + assert not util_resources.file_exists('test1.background.keys') + + # Turn off background sync + for background_sync_type in [reuse_password, custom_password]: + restore_wallet(sender_wallet, seeds[0], 'test1', 'test_password') + background_cache_password = None if background_sync_type == reuse_password else 'background_password' + sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = 'test_password', background_cache_password = background_cache_password) + if background_sync_type == custom_password: + assert util_resources.file_exists('test1.background') + assert util_resources.file_exists('test1.background.keys') + sender_wallet.close_wallet() + assert util_resources.file_exists('test1.background') + assert util_resources.file_exists('test1.background.keys') + else: + assert not util_resources.file_exists('test1.background') + assert not util_resources.file_exists('test1.background.keys') + sender_wallet.close_wallet() + assert not util_resources.file_exists('test1.background') + assert not util_resources.file_exists('test1.background.keys') + sender_wallet.open_wallet('test1', password = 'test_password') + sender_wallet.setup_background_sync(background_sync_type = sender_wallet.background_sync_options.off, wallet_password = 'test_password') + assert not util_resources.file_exists('test1.background') + assert not util_resources.file_exists('test1.background.keys') + sender_wallet.close_wallet() + assert not util_resources.file_exists('test1.background') + assert not util_resources.file_exists('test1.background.keys') + sender_wallet.open_wallet('test1', password = 'test_password') + + # Sanity check against outgoing wallet restored at height 0 + sender_wallet.close_wallet() + sender_wallet.restore_deterministic_wallet(seed = seeds[0], restore_height = 0) + sender_wallet.refresh() + assert_correct_transfers(sender_wallet, transfers, incoming_transfers, expected_sender_balance) + + print('Checking background sync on incoming wallet') + receiver_wallet.setup_background_sync(background_sync_type = reuse_password) + receiver_wallet.start_background_sync() + receiver_wallet.refresh() + transfers = receiver_wallet.get_transfers() + assert 'pending' not in transfers or len(transfers.pending) == 0 + assert 'pool' not in transfers or len (transfers.pool) == 0 + assert len(transfers['in']) == in_len + 1 + tx = [x for x in transfers['in'] if x.txid == txid] + assert len(tx) == 1 + tx = tx[0] + assert tx.amount == amount - fee + assert tx.fee == fee + incoming_transfers = receiver_wallet.incoming_transfers(transfer_type = 'all') + assert len([x for x in incoming_transfers.transfers if x.tx_hash == txid and x.key_image == '' and not x.spent]) == 1 + assert receiver_wallet.get_balance().balance == expected_receiver_balance + + # Restore and check background syncing incoming wallet + restore_wallet(receiver_wallet, seeds[1]) + receiver_wallet.setup_background_sync(background_sync_type = reuse_password) + receiver_wallet.start_background_sync() + receiver_wallet.refresh() + if 'out' in transfers: + for i, out_tx in enumerate(transfers.out): + if 'destinations' in out_tx: + del transfers.out[i]['destinations'] # destinations are not expected after wallet restore + background_bal = receiver_wallet.get_balance().balance + assert background_bal >= expected_receiver_balance + background_transfers = receiver_wallet.get_transfers() + assert 'out' not in background_transfers or len(background_transfers.out) == 0 + assert 'in' in background_transfers and len(background_transfers['in']) > 0 + background_incoming_transfers = receiver_wallet.incoming_transfers(transfer_type = 'all') + assert len(background_incoming_transfers) == len(incoming_transfers) + assert len([x for x in background_incoming_transfers.transfers if x.spent or x.key_image != '']) == 0 + assert len([x for x in background_incoming_transfers.transfers if x.tx_hash == txid]) == 1 + + # Stop background sync and check transfers update correctly + receiver_wallet.stop_background_sync(wallet_password = '', seed = seeds[1]) + diff_transfers(receiver_wallet.get_transfers(), transfers) + incoming_transfers = receiver_wallet.incoming_transfers(transfer_type = 'all') + assert len(background_incoming_transfers) == len(incoming_transfers) + assert len([x for x in incoming_transfers.transfers if x.tx_hash == txid and x.key_image != '' and not x.spent]) == 1 + assert receiver_wallet.get_balance().balance == expected_receiver_balance + + # Check a fresh incoming wallet with wallet files saved to disk and encrypted with password + restore_wallet(receiver_wallet, seeds[1], 'test2', 'test_password') + receiver_wallet.setup_background_sync(background_sync_type = reuse_password, wallet_password = 'test_password') + receiver_wallet.start_background_sync() + receiver_wallet.refresh() + assert_correct_transfers(receiver_wallet, background_transfers, background_incoming_transfers, background_bal) + stop_with_wrong_inputs(receiver_wallet, 'wrong_password') + receiver_wallet.stop_background_sync(wallet_password = 'test_password') + assert_correct_transfers(receiver_wallet, transfers, incoming_transfers, expected_receiver_balance) + + # Close receiver's wallet while background sync is enabled then reopen + restore_wallet(receiver_wallet, seeds[1], 'test2', 'test_password') + receiver_wallet.setup_background_sync(background_sync_type = reuse_password, wallet_password = 'test_password') + receiver_wallet.start_background_sync() + receiver_wallet.refresh() + diff_transfers(receiver_wallet.get_transfers(), background_transfers) + diff_incoming_transfers(receiver_wallet.incoming_transfers(transfer_type = 'all'), background_incoming_transfers) + assert receiver_wallet.get_balance().balance == background_bal + receiver_wallet.close_wallet() + receiver_wallet.open_wallet('test2', password = 'test_password') + # It should reopen with spend key loaded and correctly scan all transfers + assert_correct_transfers(receiver_wallet, transfers, incoming_transfers, expected_receiver_balance) + + # Sanity check against incoming wallet restored at height 0 + receiver_wallet.close_wallet() + receiver_wallet.restore_deterministic_wallet(seed = seeds[1], restore_height = 0) + receiver_wallet.refresh() + assert_correct_transfers(receiver_wallet, transfers, incoming_transfers, expected_receiver_balance) + + # Clean up + util_resources.remove_wallet_files('test1') + util_resources.remove_wallet_files('test2') + for i in range(2): + self.wallet[i].close_wallet() + self.wallet[i].restore_deterministic_wallet(seed = seeds[i]) + + def check_background_sync_reorg_recovery(self): + daemon = Daemon() + + print('Testing background sync reorg recovery') + + # Disconnect daemon from peers + daemon.out_peers(0) + + # Background sync type options + sender_wallet = self.wallet[0] + reuse_password = sender_wallet.background_sync_options.reuse_password + custom_password = sender_wallet.background_sync_options.custom_password + + for background_sync_type in [reuse_password, custom_password]: + # Set up wallet saved to disk + sender_wallet.close_wallet() + util_resources.remove_wallet_files('test1') + sender_wallet.restore_deterministic_wallet(seed = seeds[0], filename = 'test1', password = '') + sender_wallet.auto_refresh(enable = False) + sender_wallet.refresh() + sender_starting_balance = sender_wallet.get_balance().balance + + # Send tx and mine a block + amount = 1000000000000 + assert sender_starting_balance > amount + dst = {'address': '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', 'amount': amount} + res = sender_wallet.transfer([dst]) + assert len(res.tx_hash) == 32*2 + txid = res.tx_hash + + daemon.generateblocks('46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', 1) + + # Make sure the wallet can see the tx + sender_wallet.refresh() + transfers = sender_wallet.get_transfers() + assert 'pool' not in transfers or len (transfers.pool) == 0 + tx = [x for x in transfers.out if x.txid == txid] + assert len(tx) == 1 + tx = tx[0] + assert sender_wallet.get_balance().balance < (sender_starting_balance - amount) + + # Pop the block while background syncing + background_cache_password = None if background_sync_type == reuse_password else 'background_password' + sender_wallet.setup_background_sync(background_sync_type = background_sync_type, wallet_password = '', background_cache_password = background_cache_password) + sender_wallet.start_background_sync() + daemon.pop_blocks(1) + daemon.flush_txpool() + + daemon.generateblocks('46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', 1) + + # Make sure the wallet can no longer see the tx + sender_wallet.refresh() + sender_wallet.stop_background_sync(wallet_password = '', seed = seeds[0]) + transfers = sender_wallet.get_transfers() + no_tx = [x for x in transfers.out if x.txid == txid] + assert len(no_tx) == 0 + assert sender_wallet.get_balance().balance == sender_starting_balance + + # Clean up + daemon.out_peers(12) + util_resources.remove_wallet_files('test1') + self.wallet[0].close_wallet() + self.wallet[0].restore_deterministic_wallet(seed = seeds[0]) + if __name__ == '__main__': TransferTest().run_test() diff --git a/tests/functional_tests/util_resources.py b/tests/functional_tests/util_resources.py index e030312da..3ca6fdb86 100755 --- a/tests/functional_tests/util_resources.py +++ b/tests/functional_tests/util_resources.py @@ -37,6 +37,8 @@ from __future__ import print_function import subprocess import psutil +import os +import errno def available_ram_gb(): ram_bytes = psutil.virtual_memory().available @@ -51,3 +53,26 @@ def get_time_pi_seconds(cores, app_dir='.'): miliseconds = int(decoded) return miliseconds / 1000.0 + +def remove_file(name): + WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY'] + assert WALLET_DIRECTORY != '' + try: + os.unlink(WALLET_DIRECTORY + '/' + name) + except OSError as e: + if e.errno != errno.ENOENT: + raise + +def get_file_path(name): + WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY'] + assert WALLET_DIRECTORY != '' + return WALLET_DIRECTORY + '/' + name + +def remove_wallet_files(name): + for suffix in ['', '.keys', '.background', '.background.keys', '.address.txt']: + remove_file(name + suffix) + +def file_exists(name): + WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY'] + assert WALLET_DIRECTORY != '' + return os.path.isfile(WALLET_DIRECTORY + '/' + name) diff --git a/tests/functional_tests/wallet.py b/tests/functional_tests/wallet.py index 1ad05c98f..8182cecb2 100755 --- a/tests/functional_tests/wallet.py +++ b/tests/functional_tests/wallet.py @@ -34,8 +34,7 @@ from __future__ import print_function import sys -import os -import errno +import util_resources from framework.wallet import Wallet from framework.daemon import Daemon @@ -54,24 +53,6 @@ class WalletTest(): self.change_password() self.store() - def remove_file(self, name): - WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY'] - assert WALLET_DIRECTORY != '' - try: - os.unlink(WALLET_DIRECTORY + '/' + name) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - def remove_wallet_files(self, name): - for suffix in ['', '.keys']: - self.remove_file(name + suffix) - - def file_exists(self, name): - WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY'] - assert WALLET_DIRECTORY != '' - return os.path.isfile(WALLET_DIRECTORY + '/' + name) - def reset(self): print('Resetting blockchain') daemon = Daemon() @@ -333,7 +314,7 @@ class WalletTest(): try: wallet.close_wallet() except: pass - self.remove_wallet_files('test1') + util_resources.remove_wallet_files('test1') seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted' res = wallet.restore_deterministic_wallet(seed = seed, filename = 'test1') @@ -359,7 +340,7 @@ class WalletTest(): wallet.close_wallet() - self.remove_wallet_files('test1') + util_resources.remove_wallet_files('test1') def store(self): print('Testing store') @@ -369,22 +350,26 @@ class WalletTest(): try: wallet.close_wallet() except: pass - self.remove_wallet_files('test1') + util_resources.remove_wallet_files('test1') seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted' res = wallet.restore_deterministic_wallet(seed = seed, filename = 'test1') assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' assert res.seed == seed - self.remove_file('test1') - assert self.file_exists('test1.keys') - assert not self.file_exists('test1') + util_resources.remove_file('test1') + assert util_resources.file_exists('test1.keys') + assert not util_resources.file_exists('test1') wallet.store() - assert self.file_exists('test1.keys') - assert self.file_exists('test1') + assert util_resources.file_exists('test1.keys') + assert util_resources.file_exists('test1') wallet.close_wallet() - self.remove_wallet_files('test1') + + wallet.open_wallet(filename = 'test1', password = '') + wallet.close_wallet() + + util_resources.remove_wallet_files('test1') if __name__ == '__main__': diff --git a/tests/unit_tests/wipeable_string.cpp b/tests/unit_tests/wipeable_string.cpp index ef6964f9e..25121a02e 100644 --- a/tests/unit_tests/wipeable_string.cpp +++ b/tests/unit_tests/wipeable_string.cpp @@ -211,3 +211,15 @@ TEST(wipeable_string, to_hex) ASSERT_TRUE(epee::to_hex::wipeable_string(epee::span((const uint8_t*)"", 0)) == epee::wipeable_string("")); ASSERT_TRUE(epee::to_hex::wipeable_string(epee::span((const uint8_t*)"abc", 3)) == epee::wipeable_string("616263")); } + +TEST(wipeable_string, to_string) +{ + // Converting a wipeable_string to a string defeats the purpose of wipeable_string, + // but nice to know this works + std::string str; + { + epee::wipeable_string wipeable_str("foo"); + str = std::string(wipeable_str.data(), wipeable_str.size()); + } + ASSERT_TRUE(str == std::string("foo")); +} diff --git a/utils/python-rpc/framework/wallet.py b/utils/python-rpc/framework/wallet.py index 1e10e1f86..bff33a561 100644 --- a/utils/python-rpc/framework/wallet.py +++ b/utils/python-rpc/framework/wallet.py @@ -1138,3 +1138,45 @@ class Wallet(object): 'id': '0' } return self.rpc.send_json_rpc_request(frozen) + + class BackgroundSyncOptions(object): + def __init__(self): + self.off = 'off' + self.reuse_password = 'reuse-wallet-password' + self.custom_password = 'custom-background-password' + background_sync_options = BackgroundSyncOptions() + + def setup_background_sync(self, background_sync_type = background_sync_options.off, wallet_password = '', background_cache_password = ''): + setup_background_sync = { + 'method': 'setup_background_sync', + 'jsonrpc': '2.0', + 'params' : { + 'background_sync_type': background_sync_type, + 'wallet_password': wallet_password, + 'background_cache_password': background_cache_password, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(setup_background_sync) + + def start_background_sync(self): + start_background_sync = { + 'method': 'start_background_sync', + 'jsonrpc': '2.0', + 'params' : {}, + 'id': '0' + } + return self.rpc.send_json_rpc_request(start_background_sync) + + def stop_background_sync(self, wallet_password = '', seed = '', seed_offset = ''): + stop_background_sync = { + 'method': 'stop_background_sync', + 'jsonrpc': '2.0', + 'params' : { + 'wallet_password': wallet_password, + 'seed': seed, + 'seed_offset': seed_offset, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(stop_background_sync)