Mirrors
/
monero
mirror of https://github.com/monero-project/monero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

src: dynamic block sync size 

Co-authored-by: nahuhh
This commit is contained in:
0xFFFC0000 2024-10-12 06:35:37 +00:00
parent 9866a0e902
commit 2195b6aaec
7 changed files with 75 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/cryptonote_config.h
View File

@ -98,6 +98,10 @@
#define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT_PRE_V4 100 //by default, blocks count in blocks downloading
#define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT 20 //by default, blocks count in blocks downloading
#define BLOCKS_SYNCHRONIZING_MAX_COUNT 2048 //must be a power of 2, greater than 128, equal to SEEDHASH_EPOCH_BLOCKS
#define BLOCKS_MEDIAN_WINDOW 100 //by default, compute median weights of last 100 blocks
#define BATCH_MAX_WEIGHT 20 //by default, maximum size of batch in [mB]
#define BATCH_MAX_ALLOWED_WEIGHT 50 //maximum allowed size of batch in [mB]
#define BLOCKS_HUGE_THRESHOLD_SIZE ((BATCH_MAX_WEIGHT * 1000000) / 2) //blocks that we consider huge [B]
#define CRYPTONOTE_MEMPOOL_TX_LIVETIME (86400*3) //seconds, three days
#define CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME 604800 //seconds, one week

4
src/cryptonote_core/blockchain.h
View File

@ -1443,7 +1443,11 @@ namespace cryptonote
* @param weights return-by-reference the list of weights
* @param count the number of blocks to get weights for
*/
public:
void get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const;
#ifndef IN_UNIT_TESTS
private:
#endif
/**
* @brief gets block long term weight median

50
src/cryptonote_core/cryptonote_core.cpp
View File

@ -168,6 +168,11 @@ namespace cryptonote
, "How many blocks to sync at once during chain synchronization (0 = adaptive)."
, 0
};
static const command_line::arg_descriptor<size_t> arg_batch_max_weight = {
"batch-max-weight"
, "How many megabytes to sync in one batch during chain synchronization, default is 20 max"
, (BATCH_MAX_WEIGHT)
};
static const command_line::arg_descriptor<std::string> arg_check_updates = {
"check-updates"
, "Check for new versions of monero: [disabled|notify|download|update]"
@ -335,6 +340,7 @@ namespace cryptonote
command_line::add_arg(desc, arg_fast_block_sync);
command_line::add_arg(desc, arg_show_time_stats);
command_line::add_arg(desc, arg_block_sync_size);
command_line::add_arg(desc, arg_batch_max_weight);
command_line::add_arg(desc, arg_check_updates);
command_line::add_arg(desc, arg_no_fluffy_blocks);
command_line::add_arg(desc, arg_test_dbg_lock_sleep);
@ -691,6 +697,17 @@ namespace cryptonote
if (block_sync_size > BLOCKS_SYNCHRONIZING_MAX_COUNT)
MERROR("Error --block-sync-size cannot be greater than " << BLOCKS_SYNCHRONIZING_MAX_COUNT);
if(block_sync_size)
MWARNING("When --block-sync-size defined, the --batch-max-weight is not going to have any effect.");
batch_max_weight = command_line::get_arg(vm, arg_batch_max_weight);
if (batch_max_weight > BATCH_MAX_ALLOWED_WEIGHT) {
MERROR("Error --batch-max-weight cannot be greater than " << BATCH_MAX_ALLOWED_WEIGHT << " [mB]");
batch_max_weight = BATCH_MAX_ALLOWED_WEIGHT;
}
batch_max_weight *= 1000000; // transfer it to byte.
MGINFO("Loading checkpoints");
// load json & DNS checkpoints, and verify them
@ -1214,16 +1231,37 @@ namespace cryptonote
return true;
}
//-----------------------------------------------------------------------------------------------
size_t core::get_block_sync_size(uint64_t height) const
size_t core::get_block_sync_size(uint64_t height, const uint64_t average_blocksize_of_biggest_batch) const
{
static const uint64_t quick_height = m_nettype == TESTNET ? 801219 : m_nettype == MAINNET ? 1220516 : 0;
size_t res = 0;
if (block_sync_size > 0)
res = block_sync_size;
else if (height >= quick_height)
res = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
else
res = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT_PRE_V4;
else {
size_t number_of_blocks = BLOCKS_MEDIAN_WINDOW;
std::vector<uint64_t> last_n_blocks_weights;
m_blockchain_storage.get_last_n_blocks_weights(last_n_blocks_weights, number_of_blocks);
uint64_t median_weight = epee::misc_utils::median(last_n_blocks_weights);
MINFO("Last " << number_of_blocks
<< " blocks median size is " << median_weight
<< " bytes and the max average blocksize in the queue is " << average_blocksize_of_biggest_batch << " bytes");
uint64_t projected_blocksize = (average_blocksize_of_biggest_batch > median_weight) ? average_blocksize_of_biggest_batch : median_weight;
if ((projected_blocksize * BLOCKS_MEDIAN_WINDOW) < batch_max_weight) {
res = BLOCKS_MEDIAN_WINDOW;
MINFO("blocks are tiny, " << projected_blocksize << " bytes, sync " << res << " blocks in next batch");
}
else if (projected_blocksize >= batch_max_weight) {
res = 1;
MINFO("blocks are projected to surpass " << batch_max_weight << " bytes, syncing just a single block in next batch");
}
else if (projected_blocksize > BLOCKS_HUGE_THRESHOLD_SIZE) {
res = 1;
MINFO("blocks are huge, sync just a single block in next batch");
}
else {
res = batch_max_weight / projected_blocksize;
MINFO("projected blocksize is " << projected_blocksize << " bytes, sync " << res << " blocks in next batch");
}
}
static size_t max_block_size = 0;
if (max_block_size == 0)

7
src/cryptonote_core/cryptonote_core.h
View File

@ -798,9 +798,13 @@ namespace cryptonote
/**
* @brief get the number of blocks to sync in one go
*
* @param height the height that we want to get_block_sync_size for
* @param average_blocksize_of_biggest_batch is the average blocksize of the biggest batch
* we are downloading in current active connections.
*
* @return the number of blocks to sync in one go
*/
size_t get_block_sync_size(uint64_t height) const;
size_t get_block_sync_size(uint64_t height, const uint64_t average_blocksize_of_biggest_batch = 0) const;
/**
* @brief get the sum of coinbase tx amounts between blocks
@ -1136,6 +1140,7 @@ namespace cryptonote
bool m_disable_dns_checkpoints;
size_t block_sync_size;
std::uint64_t batch_max_weight;
time_t start_time;

11
src/cryptonote_protocol/cryptonote_protocol_handler.h
View File

@ -50,6 +50,7 @@
#include "net/levin_base.h"
#include "p2p/net_node_common.h"
#include <boost/circular_buffer.hpp>
#include <atomic>
PUSH_WARNINGS
DISABLE_VS_WARNINGS(4355)
@ -112,6 +113,15 @@ namespace cryptonote
void log_connections();
std::list<connection_info> get_connections();
const block_queue &get_block_queue() const { return m_block_queue; }
const std::uint64_t max_average_of_blocksize_in_queue() {
std::vector<std::uint64_t> average_blocksize{0};
m_block_queue.foreach([&](const cryptonote::block_queue::span &span) {
average_blocksize.push_back(span.size / span.nblocks);
return true; // we don't care about the return value
});
MINFO("Maximum average of blocksize for current batches : " << *std::max_element(average_blocksize.begin(), average_blocksize.end()));
return *std::max_element(average_blocksize.begin(), average_blocksize.end());
}
void stop();
void on_connection_close(cryptonote_connection_context &context);
void set_max_out_peers(epee::net_utils::zone zone, unsigned int max) { CRITICAL_REGION_LOCAL(m_max_out_peers_lock); m_max_out_peers[zone] = max; }
@ -191,6 +201,7 @@ namespace cryptonote
uint64_t m_sync_download_chain_size, m_sync_download_objects_size;
size_t m_block_download_max_size;
bool m_sync_pruned_blocks;
std::atomic<size_t> m_bss;
// Values for sync time estimates
boost::posix_time::ptime m_sync_start_time;

10
src/cryptonote_protocol/cryptonote_protocol_handler.inl
View File

@ -87,8 +87,8 @@ namespace cryptonote
m_synchronized(offline),
m_ask_for_txpool_complement(true),
m_stopping(false),
m_no_sync(false)
m_no_sync(false),
m_bss(0)
{
if(!m_p2p)
m_p2p = &m_p2p_stub;
@ -2222,7 +2222,7 @@ skip:
NOTIFY_REQUEST_GET_OBJECTS::request req;
bool is_next = false;
size_t count = 0;
const size_t count_limit = m_core.get_block_sync_size(m_core.get_current_blockchain_height());
size_t l_m_bss = m_bss = m_core.get_block_sync_size(m_core.get_current_blockchain_height(), max_average_of_blocksize_in_queue());
std::pair<uint64_t, uint64_t> span = std::make_pair(0, 0);
if (force_next_span)
{
@ -2272,7 +2272,7 @@ skip:
const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1;
static const uint64_t bp_fork_height = m_core.get_earliest_ideal_height_for_version(8);
bool sync_pruned_blocks = m_sync_pruned_blocks && first_block_height >= bp_fork_height && m_core.get_blockchain_pruning_seed();
span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_remote_address, sync_pruned_blocks, m_core.get_blockchain_pruning_seed(), context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects);
span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, l_m_bss, context.m_connection_id, context.m_remote_address, sync_pruned_blocks, m_core.get_blockchain_pruning_seed(), context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects);
MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second);
if (span.second > 0)
{
@ -2358,7 +2358,7 @@ skip:
context.m_expect_height = span.first;
context.m_expect_response = NOTIFY_RESPONSE_GET_OBJECTS::ID;
MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size()
<< "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front());
<< "requested blocks count=" << count << " / " << l_m_bss << " from " << span.first << ", first hash " << req.blocks.front());
//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
MDEBUG("Asking for " << (req.prune ? "pruned" : "full") << " data, start/end "

2
tests/unit_tests/node_server.cpp
View File

@ -74,7 +74,7 @@ public:
bool cleanup_handle_incoming_blocks(bool force_sync = false) { return true; }
bool update_checkpoints(const bool skip_dns = false) { return true; }
uint64_t get_target_blockchain_height() const { return 1; }
size_t get_block_sync_size(uint64_t height) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; }
size_t get_block_sync_size(uint64_t height, const uint64_t average_blocksize_of_biggest_batch = 0) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; }
virtual void on_transactions_relayed(epee::span<const cryptonote::blobdata> tx_blobs, cryptonote::relay_method tx_relay) {}
cryptonote::network_type get_nettype() const { return cryptonote::MAINNET; }
bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob, cryptonote::relay_category tx_category) const { return false; }