Mirrors
/
wownero
mirror of https://git.wownero.com/wownero/wownero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

daemon: auto public nodes - cache and prioritize most stable nodes

This commit is contained in:
xiphon 2020-02-26 12:37:28 +00:00
parent 6b2b1d6368
commit 42a7a4dd32
9 changed files with 477 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/rpc/CMakeLists.txt
View File

@ -35,6 +35,7 @@ set(rpc_base_sources
set(rpc_sources
bootstrap_daemon.cpp
bootstrap_node_selector.cpp
core_rpc_server.cpp
rpc_payment.cpp
rpc_version_str.cpp

40
src/rpc/bootstrap_daemon.cpp
View File

@ -2,6 +2,8 @@
#include <stdexcept>
#include <boost/thread/locks.hpp>
#include "crypto/crypto.h"
#include "cryptonote_core/cryptonote_core.h"
#include "misc_log_ex.h"
@ -12,15 +14,22 @@
namespace cryptonote
{
bootstrap_daemon::bootstrap_daemon(std::function<boost::optional<std::string>()> get_next_public_node)
: m_get_next_public_node(get_next_public_node)
bootstrap_daemon::bootstrap_daemon(
std::function<std::map<std::string, bool>()> get_public_nodes,
bool rpc_payment_enabled)
: m_selector(new bootstrap_node::selector_auto(std::move(get_public_nodes)))
, m_rpc_payment_enabled(rpc_payment_enabled)
{
}
bootstrap_daemon::bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials)
: bootstrap_daemon(nullptr)
bootstrap_daemon::bootstrap_daemon(
const std::string &address,
boost::optional<epee::net_utils::http::login> credentials,
bool rpc_payment_enabled)
: m_selector(nullptr)
, m_rpc_payment_enabled(rpc_payment_enabled)
{
if (!set_server(address, credentials))
if (!set_server(address, std::move(credentials)))
{
throw std::runtime_error("invalid bootstrap daemon address or credentials");
}
@ -54,11 +63,16 @@ namespace cryptonote
return res.height;
}
bool bootstrap_daemon::handle_result(bool success)
bool bootstrap_daemon::handle_result(bool success, const std::string &status)
{
if (!success && m_get_next_public_node)
const bool failed = !success || (!m_rpc_payment_enabled && status == CORE_RPC_STATUS_PAYMENT_REQUIRED);
if (failed && m_selector)
{
const std::string current_address = address();
m_http_client.disconnect();
const boost::unique_lock<boost::mutex> lock(m_selector_mutex);
m_selector->handle_result(current_address, !failed);
}
return success;
@ -79,14 +93,18 @@ namespace cryptonote
bool bootstrap_daemon::switch_server_if_needed()
{
if (!m_get_next_public_node || m_http_client.is_connected())
if (m_http_client.is_connected() || !m_selector)
{
return true;
}
const boost::optional<std::string> address = m_get_next_public_node();
if (address) {
return set_server(*address);
boost::optional<bootstrap_node::node_info> node;
{
const boost::unique_lock<boost::mutex> lock(m_selector_mutex);
node = m_selector->next_node();
}
if (node) {
return set_server(node->address, node->credentials);
}
return false;

34
src/rpc/bootstrap_daemon.h
View File

@ -1,26 +1,34 @@
#pragma once
#include <functional>
#include <vector>
#include <map>
#include <boost/optional/optional.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/utility/string_ref.hpp>
#include "net/http_client.h"
#include "storages/http_abstract_invoke.h"
#include "bootstrap_node_selector.h"
namespace cryptonote
{
class bootstrap_daemon
{
public:
bootstrap_daemon(std::function<boost::optional<std::string>()> get_next_public_node);
bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials);
bootstrap_daemon(
std::function<std::map<std::string, bool>()> get_public_nodes,
bool rpc_payment_enabled);
bootstrap_daemon(
const std::string &address,
boost::optional<epee::net_utils::http::login> credentials,
bool rpc_payment_enabled);
std::string address() const noexcept;
boost::optional<uint64_t> get_height();
bool handle_result(bool success);
bool handle_result(bool success, const std::string &status);
template <class t_request, class t_response>
bool invoke_http_json(const boost::string_ref uri, const t_request &out_struct, t_response &result_struct)
@ -30,7 +38,8 @@ namespace cryptonote
return false;
}
return handle_result(epee::net_utils::invoke_http_json(uri, out_struct, result_struct, m_http_client));
const bool result = epee::net_utils::invoke_http_json(uri, out_struct, result_struct, m_http_client);
return handle_result(result, result_struct.status);
}
template <class t_request, class t_response>
@ -41,7 +50,8 @@ namespace cryptonote
return false;
}
return handle_result(epee::net_utils::invoke_http_bin(uri, out_struct, result_struct, m_http_client));
const bool result = epee::net_utils::invoke_http_bin(uri, out_struct, result_struct, m_http_client);
return handle_result(result, result_struct.status);
}
template <class t_request, class t_response>
@ -52,7 +62,13 @@ namespace cryptonote
return false;
}
return handle_result(epee::net_utils::invoke_http_json_rpc("/json_rpc", std::string(command_name.begin(), command_name.end()), out_struct, result_struct, m_http_client));
const bool result = epee::net_utils::invoke_http_json_rpc(
"/json_rpc",
std::string(command_name.begin(), command_name.end()),
out_struct,
result_struct,
m_http_client);
return handle_result(result, result_struct.status);
}
private:
@ -61,7 +77,9 @@ namespace cryptonote
private:
epee::net_utils::http::http_simple_client m_http_client;
std::function<boost::optional<std::string>()> m_get_next_public_node;
const bool m_rpc_payment_enabled;
const std::unique_ptr<bootstrap_node::selector> m_selector;
boost::mutex m_selector_mutex;
};
}

117
src/rpc/bootstrap_node_selector.cpp Normal file
View File

@ -0,0 +1,117 @@
// Copyright (c) 2020, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "bootstrap_node_selector.h"
#include "crypto/crypto.h"
namespace cryptonote
{
namespace bootstrap_node
{
void selector_auto::node::handle_result(bool success)
{
if (!success)
{
fails = std::min(std::numeric_limits<size_t>::max() - 2, fails) + 2;
}
else
{
fails = std::max(std::numeric_limits<size_t>::min() + 2, fails) - 2;
}
}
void selector_auto::handle_result(const std::string &address, bool success)
{
auto &nodes_by_address = m_nodes.get<by_address>();
const auto it = nodes_by_address.find(address);
if (it != nodes_by_address.end())
{
nodes_by_address.modify(it, [success](node &entry) {
entry.handle_result(success);
});
}
}
boost::optional<node_info> selector_auto::next_node()
{
if (!has_at_least_one_good_node())
{
append_new_nodes();
}
if (m_nodes.empty())
{
return {};
}
auto node = m_nodes.get<by_fails>().begin();
const size_t count = std::distance(node, m_nodes.get<by_fails>().upper_bound(node->fails));
std::advance(node, crypto::rand_idx(count));
return {{node->address, {}}};
}
bool selector_auto::has_at_least_one_good_node() const
{
return !m_nodes.empty() && m_nodes.get<by_fails>().begin()->fails == 0;
}
void selector_auto::append_new_nodes()
{
bool updated = false;
for (const auto &node : m_get_nodes())
{
const auto &address = node.first;
const auto &white = node.second;
const size_t initial_score = white ? 0 : 1;
updated |= m_nodes.get<by_address>().insert({address, initial_score}).second;
}
if (updated)
{
truncate();
}
}
void selector_auto::truncate()
{
const size_t total = m_nodes.size();
if (total > m_max_nodes)
{
auto &nodes_by_fails = m_nodes.get<by_fails>();
auto from = nodes_by_fails.rbegin();
std::advance(from, total - m_max_nodes);
nodes_by_fails.erase(from.base(), nodes_by_fails.end());
}
}
}
}

103
src/rpc/bootstrap_node_selector.h Normal file
View File

@ -0,0 +1,103 @@
// Copyright (c) 2020, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <functional>
#include <limits>
#include <map>
#include <string>
#include <utility>
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/optional/optional.hpp>
#include "net/http_client.h"
namespace cryptonote
{
namespace bootstrap_node
{
struct node_info
{
std::string address;
boost::optional<epee::net_utils::http::login> credentials;
};
struct selector
{
virtual void handle_result(const std::string &address, bool success) = 0;
virtual boost::optional<node_info> next_node() = 0;
};
class selector_auto : public selector
{
public:
selector_auto(std::function<std::map<std::string, bool>()> get_nodes, size_t max_nodes = 1000)
: m_get_nodes(std::move(get_nodes))
, m_max_nodes(max_nodes)
{}
void handle_result(const std::string &address, bool success) final;
boost::optional<node_info> next_node() final;
private:
bool has_at_least_one_good_node() const;
void append_new_nodes();
void truncate();
private:
struct node
{
std::string address;
size_t fails;
void handle_result(bool success);
};
struct by_address {};
struct by_fails {};
typedef boost::multi_index_container<
node,
boost::multi_index::indexed_by<
boost::multi_index::ordered_unique<boost::multi_index::tag<by_address>, boost::multi_index::member<node, std::string, &node::address>>,
boost::multi_index::ordered_non_unique<boost::multi_index::tag<by_fails>, boost::multi_index::member<node, size_t, &node::fails>>
>
> nodes_list;
const std::function<std::map<std::string, bool>()> m_get_nodes;
const size_t m_max_nodes;
nodes_list m_nodes;
};
}
}

50
src/rpc/core_rpc_server.cpp
View File

@ -170,7 +170,7 @@ namespace cryptonote
return set_bootstrap_daemon(address, credentials);
}
//------------------------------------------------------------------------------------------------------------------------------
boost::optional<std::string> core_rpc_server::get_random_public_node()
std::map<std::string, bool> core_rpc_server::get_public_nodes(uint32_t credits_per_hash_threshold/* = 0*/)
{
COMMAND_RPC_GET_PUBLIC_NODES::request request;
COMMAND_RPC_GET_PUBLIC_NODES::response response;
@ -179,47 +179,51 @@ namespace cryptonote
request.white = true;
if (!on_get_public_nodes(request, response) || response.status != CORE_RPC_STATUS_OK)
{
return boost::none;
return {};
}
const auto get_random_node_address = [](const std::vector<public_node>& public_nodes) -> std::string {
const auto& random_node = public_nodes[crypto::rand_idx(public_nodes.size())];
const auto address = random_node.host + ":" + std::to_string(random_node.rpc_port);
return address;
std::map<std::string, bool> result;
const auto append = [&result, &credits_per_hash_threshold](const std::vector<public_node> &nodes, bool white) {
for (const auto &node : nodes)
{
const bool rpc_payment_enabled = credits_per_hash_threshold > 0;
const bool node_rpc_payment_enabled = node.rpc_credits_per_hash > 0;
if (!node_rpc_payment_enabled ||
(rpc_payment_enabled && node.rpc_credits_per_hash >= credits_per_hash_threshold))
{
result.insert(std::make_pair(node.host + ":" + std::to_string(node.rpc_port), white));
}
}
};
if (!response.white.empty())
{
return get_random_node_address(response.white);
}
append(response.white, true);
append(response.gray, false);
MDEBUG("No white public node found, checking gray peers");
if (!response.gray.empty())
{
return get_random_node_address(response.gray);
}
MERROR("Failed to find any suitable public node");
return boost::none;
return result;
}
//------------------------------------------------------------------------------------------------------------------------------
bool core_rpc_server::set_bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials)
{
boost::unique_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex);
constexpr const uint32_t credits_per_hash_threshold = 0;
constexpr const bool rpc_payment_enabled = credits_per_hash_threshold != 0;
if (address.empty())
{
m_bootstrap_daemon.reset(nullptr);
}
else if (address == "auto")
{
m_bootstrap_daemon.reset(new bootstrap_daemon([this]{ return get_random_public_node(); }));
auto get_nodes = [this, credits_per_hash_threshold]() {
return get_public_nodes(credits_per_hash_threshold);
};
m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled));
}
else
{
m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials));
m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled));
}
m_should_use_bootstrap_daemon = m_bootstrap_daemon.get() != nullptr;
@ -1931,7 +1935,7 @@ namespace cryptonote
if (*bootstrap_daemon_height < target_height)
{
MINFO("Bootstrap daemon is out of sync");
return m_bootstrap_daemon->handle_result(false);
return m_bootstrap_daemon->handle_result(false, {});
}
uint64_t top_height = m_core.get_current_blockchain_height();

2
src/rpc/core_rpc_server.h
View File

@ -265,7 +265,7 @@ private:
//utils
uint64_t get_block_reward(const block& blk);
bool fill_block_header_response(const block& blk, bool orphan_status, uint64_t height, const crypto::hash& hash, block_header_response& response, bool fill_pow_hash);
boost::optional<std::string> get_random_public_node();
std::map<std::string, bool> get_public_nodes(uint32_t credits_per_hash_threshold = 0);
bool set_bootstrap_daemon(const std::string &address, const std::string &username_password);
bool set_bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials);
enum invoke_http_mode { JON, BIN, JON_RPC };

1
tests/unit_tests/CMakeLists.txt
View File

@ -34,6 +34,7 @@ set(unit_tests_sources
blockchain_db.cpp
block_queue.cpp
block_reward.cpp
bootstrap_node_selector.cpp
bulletproofs.cpp
canonical_amounts.cpp
chacha.cpp

172
tests/unit_tests/bootstrap_node_selector.cpp Normal file
View File

@ -0,0 +1,172 @@
// Copyright (c) 2020, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <gtest/gtest.h>
#include "rpc/bootstrap_node_selector.h"
class bootstrap_node_selector : public ::testing::Test
{
protected:
void SetUp() override
{
nodes.insert(white_nodes.begin(), white_nodes.end());
nodes.insert(gray_nodes.begin(), gray_nodes.end());
}
const std::map<std::string, bool> white_nodes = {
{
"white_node_1:18089", true
},
{
"white_node_2:18081", true
}
};
const std::map<std::string, bool> gray_nodes = {
{
"gray_node_1:18081", false
},
{
"gray_node_2:18089", false
}
};
std::map<std::string, bool> nodes;
};
TEST_F(bootstrap_node_selector, selector_auto_empty)
{
cryptonote::bootstrap_node::selector_auto selector([]() {
return std::map<std::string, bool>();
});
EXPECT_FALSE(selector.next_node());
}
TEST_F(bootstrap_node_selector, selector_auto_no_credentials)
{
cryptonote::bootstrap_node::selector_auto selector([this]() {
return nodes;
});
for (size_t fails = 0; fails < nodes.size(); ++fails)
{
const auto current = selector.next_node();
EXPECT_FALSE(current->credentials);
selector.handle_result(current->address, false);
}
}
TEST_F(bootstrap_node_selector, selector_auto_success)
{
cryptonote::bootstrap_node::selector_auto selector([this]() {
return nodes;
});
auto current = selector.next_node();
for (size_t fails = 0; fails < nodes.size(); ++fails)
{
selector.handle_result(current->address, true);
current = selector.next_node();
EXPECT_TRUE(white_nodes.count(current->address) > 0);
}
}
TEST_F(bootstrap_node_selector, selector_auto_failure)
{
cryptonote::bootstrap_node::selector_auto selector([this]() {
return nodes;
});
auto current = selector.next_node();
for (size_t fails = 0; fails < nodes.size(); ++fails)
{
const auto previous = current;
selector.handle_result(current->address, false);
current = selector.next_node();
EXPECT_NE(current->address, previous->address);
}
}
TEST_F(bootstrap_node_selector, selector_auto_white_nodes_first)
{
cryptonote::bootstrap_node::selector_auto selector([this]() {
return nodes;
});
for (size_t iterations = 0; iterations < 2; ++iterations)
{
for (size_t fails = 0; fails < white_nodes.size(); ++fails)
{
const auto current = selector.next_node();
EXPECT_TRUE(white_nodes.count(current->address) > 0);
selector.handle_result(current->address, false);
}
for (size_t fails = 0; fails < gray_nodes.size(); ++fails)
{
const auto current = selector.next_node();
EXPECT_TRUE(gray_nodes.count(current->address) > 0);
selector.handle_result(current->address, false);
}
}
}
TEST_F(bootstrap_node_selector, selector_auto_max_nodes)
{
const size_t max_nodes = nodes.size() / 2;
bool populated_once = false;
cryptonote::bootstrap_node::selector_auto selector([this, &populated_once]() {
if (!populated_once)
{
populated_once = true;
return nodes;
}
return std::map<std::string, bool>();
}, max_nodes);
std::set<std::string> unique_nodes;
for (size_t fails = 0; fails < nodes.size(); ++fails)
{
const auto current = selector.next_node();
unique_nodes.insert(current->address);
selector.handle_result(current->address, false);
}
EXPECT_EQ(unique_nodes.size(), max_nodes);
}