From 600a3cf0c0ca0a99dbdc91d32138db3c8aa4165c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 26 Mar 2016 14:30:23 +0000 Subject: [PATCH] New RPC and daemon command to get output histogram This is a list of existing output amounts along with the number of outputs of that amount in the blockchain. The daemon command takes: - no parameters: all outputs with at least 3 instances - one parameter: all outputs with at least that many instances - two parameters: all outputs within that many instances The default starts at 3 to avoid massive spamming of all dust outputs in the blockchain, and is the current minimum mixin requirement. An optional vector of amounts may be passed, to request histogram only for those outputs. --- src/blockchain_db/berkeleydb/db_bdb.cpp | 6 +++ src/blockchain_db/berkeleydb/db_bdb.h | 9 ++++ src/blockchain_db/blockchain_db.h | 9 ++++ src/blockchain_db/lmdb/db_lmdb.cpp | 57 +++++++++++++++++++++++++ src/blockchain_db/lmdb/db_lmdb.h | 10 +++++ src/cryptonote_core/blockchain.cpp | 5 +++ src/cryptonote_core/blockchain.h | 9 ++++ src/daemon/command_parser_executor.cpp | 18 ++++++++ src/daemon/command_parser_executor.h | 2 + src/daemon/command_server.cpp | 5 +++ src/daemon/rpc_command_executor.cpp | 36 ++++++++++++++++ src/daemon/rpc_command_executor.h | 2 + src/rpc/core_rpc_server.cpp | 32 ++++++++++++++ src/rpc/core_rpc_server.h | 2 + src/rpc/core_rpc_server_commands_defs.h | 41 ++++++++++++++++++ tests/unit_tests/hardfork.cpp | 1 + 16 files changed, 244 insertions(+) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 1fef9e619..a7fa556bd 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -2180,6 +2180,12 @@ void BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const std::v LOG_PRINT_L3("db3: " << db3); } +std::map::BlockchainBDB::get_output_histogram(const std::vector &amounts) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + throw1(DB_ERROR("Not implemented.")); +} + void BlockchainBDB::set_hard_fork_starting_height(uint8_t version, uint64_t height) { LOG_PRINT_L3("BlockchainBDB::" << __func__); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index d7cbd24e7..5c6bda4eb 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -341,6 +341,15 @@ public: virtual bool can_thread_bulk_indices() const { return false; } #endif + /** + * @brief return a histogram of outputs on the blockchain + * + * @param amounts optional set of amounts to lookup + * + * @return a set of amount/instances + */ + std::map get_output_histogram(const std::vector &amounts) const; + private: virtual void add_block( const block& blk , const size_t& block_size diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 3e0ca141e..3585bd061 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1288,6 +1288,15 @@ public: */ virtual void drop_hard_fork_info() = 0; + /** + * @brief return a histogram of outputs on the blockchain + * + * @param amounts optional set of amounts to lookup + * + * @return a set of amount/instances + */ + virtual std::map get_output_histogram(const std::vector &amounts) const = 0; + /** * @brief is BlockchainDB in read-only mode? * diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index e928ab803..9b99520a1 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2692,6 +2692,63 @@ void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std:: LOG_PRINT_L3("db3: " << db3); } +std::map BlockchainLMDB::get_output_histogram(const std::vector &amounts) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(output_amounts); + + std::map histogram; + MDB_val k; + MDB_val v; + + if (amounts.empty()) + { + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op); + op = MDB_NEXT_NODUP; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate outputs: ", ret).c_str())); + mdb_size_t num_elems = 0; + mdb_cursor_count(m_cur_output_amounts, &num_elems); + uint64_t amount = *(const uint64_t*)k.mv_data; + histogram[amount] = num_elems; + } + } + else + { + for (const auto &amount: amounts) + { + MDB_val_copy k(amount); + int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); + if (ret == MDB_NOTFOUND) + { + histogram[amount] = 0; + } + else if (ret == MDB_SUCCESS) + { + mdb_size_t num_elems = 0; + mdb_cursor_count(m_cur_output_amounts, &num_elems); + histogram[amount] = num_elems; + } + else + { + throw0(DB_ERROR(lmdb_error("Failed to enumerate outputs: ", ret).c_str())); + } + } + } + + TXN_POSTFIX_RDONLY(); + + return histogram; +} + void BlockchainLMDB::check_hard_fork_info() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index a3f32ffaa..6cd3e0e8f 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -280,6 +280,16 @@ public: virtual void pop_block(block& blk, std::vector& txs); virtual bool can_thread_bulk_indices() const { return true; } + + /** + * @brief return a histogram of outputs on the blockchain + * + * @param amounts optional set of amounts to lookup + * + * @return a set of amount/instances + */ + std::map get_output_histogram(const std::vector &amounts) const; + private: void do_resize(uint64_t size_increase=0); diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 2f42e1db5..035afc61e 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -3313,6 +3313,11 @@ bool Blockchain::get_hard_fork_voting_info(uint8_t version, uint32_t &window, ui return m_hardfork->get_voting_info(version, window, votes, threshold, earliest_height, voting); } +std::map Blockchain:: get_output_histogram(const std::vector &amounts) const +{ + return m_db->get_output_histogram(amounts); +} + void Blockchain::load_compiled_in_block_hashes() { if (m_fast_sync && get_blocks_dat_start(m_testnet) != nullptr) diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 94def1aa9..36e0fedfc 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -682,6 +682,15 @@ namespace cryptonote */ bool flush_txes_from_pool(const std::list &txids); + /** + * @brief return a histogram of outputs on the blockchain + * + * @param amounts optional set of amounts to lookup + * + * @return a set of amount/instances + */ + std::map get_output_histogram(const std::vector &amounts) const; + /** * @brief perform a check on all key images in the blockchain * diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 0b42257e7..166fe04ca 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -439,5 +439,23 @@ bool t_command_parser_executor::flush_txpool(const std::vector& arg return m_executor.flush_txpool(txid); } +bool t_command_parser_executor::output_histogram(const std::vector& args) +{ + if (args.size() > 2) return false; + + uint64_t min_count = 3; + uint64_t max_count = 0; + + if (args.size() >= 1) + { + min_count = boost::lexical_cast(args[0]); + } + if (args.size() >= 2) + { + max_count = boost::lexical_cast(args[1]); + } + return m_executor.output_histogram(min_count, max_count); +} + } // namespace daemonize diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index 51c55a8c0..11df92a5e 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -114,6 +114,8 @@ public: bool unban(const std::vector& args); bool flush_txpool(const std::vector& args); + + bool output_histogram(const std::vector& args); }; } // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 206de9d4a..aabc2f09a 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -214,6 +214,11 @@ t_command_server::t_command_server( , std::bind(&t_command_parser_executor::flush_txpool, &m_parser, p::_1) , "Flush a transaction from the tx pool by its txid, or the whole tx pool" ); + m_command_lookup.set_handler( + "output_histogram" + , std::bind(&t_command_parser_executor::output_histogram, &m_parser, p::_1) + , "Print output histogram (amount, instances)" + ); } bool t_command_server::process_command_str(const std::string& cmd) diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 15ddc081f..933c93ed7 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -1203,5 +1203,41 @@ bool t_rpc_command_executor::flush_txpool(const std::string &txid) return true; } +bool t_rpc_command_executor::output_histogram(uint64_t min_count, uint64_t max_count) +{ + cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req; + cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.min_count = min_count; + req.max_count = max_count; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "get_output_histogram", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_output_histogram(req, res, error_resp)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + std::sort(res.histogram.begin(), res.histogram.end(), + [](const cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::entry &e1, const cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::entry &e2)->bool { return e1.instances < e2.instances; }); + for (const auto &e: res.histogram) + { + tools::msg_writer() << e.instances << " " << cryptonote::print_money(e.amount); + } + + return true; +} + }// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index bc3f2d5c5..7e73e7faf 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -132,6 +132,8 @@ public: bool unban(const std::string &ip); bool flush_txpool(const std::string &txid); + + bool output_histogram(uint64_t min_count, uint64_t max_count); }; } // namespace daemonize diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 6b625e77b..5350b6fe0 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1041,6 +1041,38 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp) + { + if(!check_core_busy()) + { + error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; + error_resp.message = "Core is busy."; + return false; + } + + std::map histogram; + try + { + histogram = m_core.get_blockchain_storage().get_output_histogram(req.amounts); + } + catch (const std::exception &e) + { + res.status = "Failed to get output histogram"; + return true; + } + + res.histogram.clear(); + res.histogram.reserve(histogram.size()); + for (const auto &i: histogram) + { + if (i.second >= req.min_count && (i.second <= req.max_count || req.max_count == 0)) + res.histogram.push_back(COMMAND_RPC_GET_OUTPUT_HISTOGRAM::entry(i.first, i.second)); + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res) { cryptonote::core::set_fast_exit(); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index f79087030..5c3707209 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -109,6 +109,7 @@ namespace cryptonote MAP_JON_RPC_WE("setbans", on_set_bans, COMMAND_RPC_SETBANS) MAP_JON_RPC_WE("getbans", on_get_bans, COMMAND_RPC_GETBANS) MAP_JON_RPC_WE("flush_txpool", on_flush_txpool, COMMAND_RPC_FLUSH_TRANSACTION_POOL) + MAP_JON_RPC_WE("get_output_histogram", on_get_output_histogram, COMMAND_RPC_GET_OUTPUT_HISTOGRAM) END_JSON_RPC_MAP() END_URI_MAP2() @@ -149,6 +150,7 @@ namespace cryptonote bool on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp); bool on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp); bool on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp); + bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp); //----------------------- private: diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 72e399ec4..6d4dd1252 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -986,5 +986,46 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; + + struct COMMAND_RPC_GET_OUTPUT_HISTOGRAM + { + struct request + { + std::vector amounts; + uint64_t min_count; + uint64_t max_count; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amounts); + KV_SERIALIZE(min_count); + KV_SERIALIZE(max_count); + END_KV_SERIALIZE_MAP() + }; + + struct entry + { + uint64_t amount; + uint64_t instances; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount); + KV_SERIALIZE(instances); + END_KV_SERIALIZE_MAP() + + entry(uint64_t amount, uint64_t instances): amount(amount), instances(instances) {} + entry() {} + }; + + struct response + { + std::string status; + std::vector histogram; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(histogram) + END_KV_SERIALIZE_MAP() + }; + }; } diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index 50e0e5ae8..c0ee5fff8 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -108,6 +108,7 @@ public: virtual bool for_all_transactions(std::function) const { return true; } virtual bool for_all_outputs(std::function f) const { return true; } virtual bool is_read_only() const { return false; } + virtual std::map get_output_histogram() const { return std::map(); } virtual void add_block( const block& blk , const size_t& block_size