Mirrors
/
wownero
mirror of https://git.wownero.com/wownero/wownero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

2014 network limit 1.0a +utils +toc -doc -drmonero 

commands and options for network limiting
works very well e.g. for 50 KiB/sec up and down
ToS (QoS) flag
peer number limit
TODO some spikes in ingress/download
TODO problems when other up and down limit
added "otshell utils" - simple logging (with colors, text files channels)
This commit is contained in:
rfree2monero 2015-01-05 20:30:17 +01:00
parent e728992803
commit eabb519605
40 changed files with 4016 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
.gitignore vendored
View File

@ -2,6 +2,7 @@
/doc
/build
/tags
log/
# vim swap files
*.swp
@ -20,4 +21,85 @@ cscope.in.out
cscope.po.out
miniupnpcstrings.h
version/
# Created by https://www.gitignore.io
### C++ ###
# Compiled Object files
*.slo
*.lo
*.o
*.obj
# Precompiled Headers
*.gch
*.pch
# Compiled Dynamic libraries
*.so
*.dylib
*.dll
# Fortran module files
*.mod
# Compiled Static libraries
*.lai
*.la
*.a
*.lib
# Executables
*.exe
*.out
*.app
### CMake ###
CMakeCache.txt
CMakeFiles
Makefile
cmake_install.cmake
install_manifest.txt
*.cmake
### Linux ###
*~
# KDE directory preferences
.directory
### Eclipse ###
*.pydevproject
.metadata
.gradle
bin/
tmp/
*.tmp
*.bak
*.swp
*~.nib
local.properties
.settings/
.loadpath
# External tool builders
.externalToolBuilders/
# Locally stored "Eclipse launch configurations"
*.launch
# CDT-specific
.cproject
# PDT-specific
.buildpath
# sbteclipse plugin
.target
# TeXlipse plugin
.texlipse

22
CMakeLists.txt
View File

@ -50,17 +50,24 @@ list(INSERT CMAKE_MODULE_PATH 0
if (NOT DEFINED ENV{DEVELOPER_LOCAL_TOOLS})
message(STATUS "Could not find DEVELOPER_LOCAL_TOOLS in env (not required)")
set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT OFF)
elseif (ENV{DEVELOPER_LOCAL_TOOLS} EQUAL 1)
set(BOOST_IGNORE_SYSTEM_PATHS OFF)
elseif ("$ENV{DEVELOPER_LOCAL_TOOLS}" STREQUAL "1")
message(STATUS "Found: env DEVELOPER_LOCAL_TOOLS = 1")
set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT ON)
set(BOOST_IGNORE_SYSTEM_PATHS ON)
else()
message(STATUS "Found: env DEVELOPER_LOCAL_TOOLS = 0")
set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT OFF)
set(BOOST_IGNORE_SYSTEM_PATHS OFF)
endif()
message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}")
option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT})
#message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}")
#option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT})
message(STATUS "BOOST_IGNORE_SYSTEM_PATHS: ${BOOST_IGNORE_SYSTEM_PATHS}")
# Options (for external/otshell_utils/)
option(WITH_TERMCOLORS "Build with support for unix terminal console colors VT100" ON)
if (WITH_TERMCOLORS)
add_definitions( -DCFG_WITH_TERMCOLORS )
endif ()
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
enable_testing()
@ -234,7 +241,7 @@ else()
endif()
endif()
if (BOOST_IGNORE_SYSTEM_PATHS)
if (${BOOST_IGNORE_SYSTEM_PATHS} STREQUAL "ON")
set(Boost_NO_SYSTEM_PATHS TRUE)
endif()
@ -264,6 +271,7 @@ endif()
include(version.cmake)
add_subdirectory(contrib)
add_subdirectory(src)
if(BUILD_TESTS)

3
contrib/CMakeLists.txt Normal file
View File

@ -0,0 +1,3 @@
add_subdirectory(otshell_utils)

77
contrib/epee/include/net/abstract_tcp_server2.h
View File

@ -1,3 +1,9 @@
/**
@file
@author from CrypoNote (see copyright below; Andrey N. Sabelnikov)
@monero rfree
@brief the connection templated-class for one peer connection
*/
// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
// All rights reserved.
//
@ -26,7 +32,7 @@
#ifndef _ABSTRACT_TCP_SERVER2_H_
#ifndef _ABSTRACT_TCP_SERVER2_H_
#define _ABSTRACT_TCP_SERVER2_H_
@ -36,6 +42,8 @@
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <map>
#include <memory>
#include <boost/asio.hpp>
#include <boost/array.hpp>
@ -46,7 +54,8 @@
#include <boost/thread/thread.hpp>
#include "net_utils_base.h"
#include "syncobj.h"
#include "../../../../src/p2p/connection_basic.hpp"
#include "../../../../contrib/otshell_utils/utils.hpp"
#define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000
@ -61,6 +70,12 @@ namespace net_utils
protected:
virtual ~i_connection_filter(){}
};
enum t_server_role { // type of the server, e.g. so that we will know how to limit it
NET = 0, // default (not used? used for misc connections maybe?) TODO
RPC = 1, // the rpc commands
P2P = 2 // to other p2p node
};
/************************************************************************/
/* */
@ -70,13 +85,18 @@ namespace net_utils
class connection
: public boost::enable_shared_from_this<connection<t_protocol_handler> >,
private boost::noncopyable,
public i_service_endpoint
public i_service_endpoint,
public connection_basic
{
public:
typedef typename t_protocol_handler::connection_context t_connection_context;
/// Construct a connection with the given io_service.
explicit connection(boost::asio::io_service& io_service,
typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter * &pfilter);
explicit connection( boost::asio::io_service& io_service,
typename t_protocol_handler::config_type& config,
std::atomic<long> &ref_sock_count, // the ++/-- counter
std::atomic<long> &sock_number, // the only increasing ++ number generator
i_connection_filter * &pfilter);
virtual ~connection();
/// Get the socket associated with the connection.
@ -90,7 +110,8 @@ namespace net_utils
void call_back_starter();
private:
//----------------- i_service_endpoint ---------------------
virtual bool do_send(const void* ptr, size_t cb);
virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint)
virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data
virtual bool close();
virtual bool call_run_once_service_io();
virtual bool request_callback();
@ -107,29 +128,24 @@ namespace net_utils
/// Handle completion of a write operation.
void handle_write(const boost::system::error_code& e, size_t cb);
/// Strand to ensure the connection's handlers are not called concurrently.
boost::asio::io_service::strand strand_;
/// Socket for the connection.
boost::asio::ip::tcp::socket socket_;
/// Buffer for incoming data.
boost::array<char, 8192> buffer_;
t_connection_context context;
volatile uint32_t m_want_close_connection;
std::atomic<bool> m_was_shutdown;
critical_section m_send_que_lock;
std::list<std::string> m_send_que;
volatile uint32_t& m_ref_sockets_count;
i_connection_filter* &m_pfilter;
volatile bool m_is_multithreaded;
// TODO what do they mean about wait on destructor?? --rfree :
//this should be the last one, because it could be wait on destructor, while other activities possible on other threads
t_protocol_handler m_protocol_handler;
//typename t_protocol_handler::config_type m_dummy_config;
std::list<boost::shared_ptr<connection<t_protocol_handler> > > m_self_refs; // add_ref/release support
critical_section m_self_refs_lock;
critical_section m_chunking_lock; // held while we add small chunks of the big do_send() to small do_send_chunk()
t_server_role m_connection_type;
public:
void setRPcStation();
};
@ -146,9 +162,11 @@ namespace net_utils
/// Construct the server to listen on the specified TCP address and port, and
/// serve up files from the given directory.
boosted_tcp_server();
explicit boosted_tcp_server(boost::asio::io_service& external_io_service);
explicit boosted_tcp_server(boost::asio::io_service& external_io_service, t_server_role s_type);
~boosted_tcp_server();
std::map<std::string, t_server_role> server_type_map;
void create_server_type_map();
bool init_server(uint32_t port, const std::string address = "0.0.0.0");
bool init_server(const std::string port, const std::string& address = "0.0.0.0");
@ -254,22 +272,25 @@ namespace net_utils
/// Acceptor used to listen for incoming connections.
boost::asio::ip::tcp::acceptor acceptor_;
/// The next connection to be accepted.
connection_ptr new_connection_;
std::atomic<bool> m_stop_signal_sent;
uint32_t m_port;
volatile uint32_t m_sockets_count;
std::atomic<long> m_sock_count;
std::atomic<long> m_sock_number;
std::string m_address;
std::string m_thread_name_prefix;
std::string m_thread_name_prefix; //TODO: change to enum server_type, now used
size_t m_threads_count;
i_connection_filter* m_pfilter;
std::vector<boost::shared_ptr<boost::thread> > m_threads;
boost::thread::id m_main_thread_id;
critical_section m_threads_lock;
volatile uint32_t m_thread_index;
};
}
}
volatile uint32_t m_thread_index; // TODO change to std::atomic
t_server_role type;
/// The next connection to be accepted
connection_ptr new_connection_;
}; // class <>boosted_tcp_server
} // namespace
} // namespace
#include "abstract_tcp_server2.inl"

373
contrib/epee/include/net/abstract_tcp_server2.inl
View File

@ -1,3 +1,9 @@
/**
@file
@author from CrypoNote (see copyright below; Andrey N. Sabelnikov)
@monero rfree
@brief the connection templated-class for one peer connection
*/
// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
// All rights reserved.
//
@ -26,7 +32,7 @@
#include "net_utils_base.h"
//#include "net_utils_base.h"
#include <boost/lambda/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/lambda/lambda.hpp>
@ -34,9 +40,20 @@
#include <boost/chrono.hpp>
#include <boost/utility/value_init.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/date_time/posix_time/posix_time.hpp> // TODO
#include <boost/thread/thread.hpp> // TODO
#include "misc_language.h"
#include "pragma_comp_defs.h"
#include <sstream>
#include <iomanip>
#include <algorithm>
#include "../../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal()
#include "../../../../contrib/otshell_utils/utils.hpp"
using namespace nOT::nUtils; // TODO
PRAGMA_WARNING_PUSH
namespace epee
{
@ -48,17 +65,19 @@ namespace net_utils
PRAGMA_WARNING_DISABLE_VS(4355)
template<class t_protocol_handler>
connection<t_protocol_handler>::connection(boost::asio::io_service& io_service,
typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter* &pfilter)
: strand_(io_service),
socket_(io_service),
m_want_close_connection(0),
m_was_shutdown(0),
m_ref_sockets_count(sock_count),
m_pfilter(pfilter),
m_protocol_handler(this, config, context)
connection<t_protocol_handler>::connection( boost::asio::io_service& io_service,
typename t_protocol_handler::config_type& config,
std::atomic<long> &ref_sock_count, // the ++/-- counter
std::atomic<long> &sock_number, // the only increasing ++ number generator
i_connection_filter* &pfilter
)
:
connection_basic(io_service, ref_sock_count, sock_number),
m_protocol_handler(this, config, context),
m_pfilter( pfilter ),
m_connection_type(NET)
{
boost::interprocess::ipcdetail::atomic_inc32(&m_ref_sockets_count);
_info_c("net/sleepRPC", "connection constructor set m_connection_type="<<m_connection_type);
}
PRAGMA_WARNING_DISABLE_VS(4355)
//---------------------------------------------------------------------------------
@ -67,12 +86,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
{
if(!m_was_shutdown)
{
LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown.");
_dbg3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown.");
shutdown();
}
LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed");
boost::interprocess::ipcdetail::atomic_dec32(&m_ref_sockets_count);
_dbg3("[sock " << socket_.native_handle() << "] Socket destroyed");
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
@ -118,13 +136,13 @@ PRAGMA_WARNING_DISABLE_VS(4355)
long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong());
context.set_details(boost::uuids::random_generator()(), ip_, remote_ep.port(), is_income);
LOG_PRINT_L3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) <<
_dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) <<
" to " << local_ep.address().to_string() << ':' << local_ep.port() <<
", total sockets objects " << m_ref_sockets_count);
", total sockets objects " << m_ref_sock_count);
if(m_pfilter && !m_pfilter->is_remote_ip_allowed(context.m_remote_ip))
{
LOG_PRINT_L2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection");
_dbg2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection");
close();
return false;
}
@ -136,7 +154,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
boost::bind(&connection<t_protocol_handler>::handle_read, self,
boost::asio::placeholders::error,
boost::asio::placeholders::bytes_transferred)));
//set ToS flag
int tos = get_tos_flag();
boost::asio::detail::socket_option::integer< IPPROTO_IP, IP_TOS >
optionTos( tos );
socket_.set_option( optionTos );
//_dbg1("Set ToS flag to " << tos);
return true;
CATCH_ENTRY_L0("connection<t_protocol_handler>::start()", false);
@ -146,7 +171,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
bool connection<t_protocol_handler>::request_callback()
{
TRY_ENTRY();
LOG_PRINT_L2("[" << print_connection_context_short(context) << "] request_callback");
_dbg2("[" << print_connection_context_short(context) << "] request_callback");
// Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
auto self = safe_shared_from_this();
if(!self)
@ -167,7 +192,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
bool connection<t_protocol_handler>::add_ref()
{
TRY_ENTRY();
LOG_PRINT_L4("[sock " << socket_.native_handle() << "] add_ref");
//_info("[sock " << socket_.native_handle() << "] add_ref");
CRITICAL_REGION_LOCAL(m_self_refs_lock);
// Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
@ -201,7 +226,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
void connection<t_protocol_handler>::call_back_starter()
{
TRY_ENTRY();
LOG_PRINT_L2("[" << print_connection_context_short(context) << "] fired_callback");
_dbg2("[" << print_connection_context_short(context) << "] fired_callback");
m_protocol_handler.handle_qued_callback();
CATCH_ENTRY_L0("connection<t_protocol_handler>::call_back_starter()", void());
}
@ -211,17 +236,18 @@ PRAGMA_WARNING_DISABLE_VS(4355)
std::size_t bytes_transferred)
{
TRY_ENTRY();
LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async read calledback.");
//_info("[sock " << socket_.native_handle() << "] Async read calledback.");
if (!e)
{
LOG_PRINT("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred, LOG_LEVEL_4);
//_info("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred);
logger_handle_net_read(bytes_transferred);
context.m_last_recv = time(NULL);
context.m_recv_cnt += bytes_transferred;
bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred);
if(!recv_res)
{
LOG_PRINT("[sock " << socket_.native_handle() << "] protocol_want_close", LOG_LEVEL_4);
//_info("[sock " << socket_.native_handle() << "] protocol_want_close");
//some error in protocol, protocol handler ask to close connection
boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1);
@ -239,14 +265,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
boost::bind(&connection<t_protocol_handler>::handle_read, connection<t_protocol_handler>::shared_from_this(),
boost::asio::placeholders::error,
boost::asio::placeholders::bytes_transferred)));
LOG_PRINT_L4("[sock " << socket_.native_handle() << "]Async read requested.");
//_info("[sock " << socket_.native_handle() << "]Async read requested.");
}
}else
{
LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value());
_dbg3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value());
if(e.value() != 2)
{
LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value());
_dbg3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value());
shutdown();
}
}
@ -282,9 +308,88 @@ PRAGMA_WARNING_DISABLE_VS(4355)
return true;
CATCH_ENTRY_L0("connection<t_protocol_handler>::call_run_once_service_io", false);
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb) {
TRY_ENTRY();
// Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
auto self = safe_shared_from_this();
if (!self) return false;
if (m_was_shutdown) return false;
// TODO avoid copy
const double factor = 32; // TODO config
typedef long long signed int t_safe; // my t_size to avoid any overunderflow in arithmetic
const t_safe chunksize_good = (t_safe)( 1024 * std::max(1.0,factor) );
const t_safe chunksize_max = chunksize_good * 2 ;
const bool allow_split = (m_connection_type == RPC) ? false : true; // TODO config
if (allow_split && (cb > chunksize_max)) {
{ // LOCK: chunking
epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical ***
_mark_c("net/out/size", "do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr);
_mark("do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr);
t_safe all = cb; // all bytes to send
t_safe pos = 0; // current sending position
// 01234567890
// ^^^^ (pos=0, len=4) ; pos:=pos+len, pos=4
// ^^^^ (pos=4, len=4) ; pos:=pos+len, pos=8
// ^^^ (pos=8, len=4) ;
// const size_t bufsize = chunksize_good; // TODO safecast
// char* buf = new char[ bufsize ];
bool all_ok = true;
while (pos < all) {
t_safe lenall = all-pos; // length from here to end
t_safe len = std::min( chunksize_good , lenall); // take a smaller part
ASRT(len<=chunksize_good);
// pos=8; len=4; all=10; len=3;
ASRT(len>0); ASRT(len < std::numeric_limits<size_t>::max()); // yeap we want strong < then max size, to be sure
void *chunk_start = ((char*)ptr) + pos;
_fact_c("net/out/size","chunk_start="<<chunk_start<<" ptr="<<ptr<<" pos="<<pos);
ASRT(chunk_start >= ptr); // not wrapped around address?
//std::memcpy( (void*)buf, chunk_start, len);
_dbg1_c("net/out/size", "part of " << lenall << ": pos="<<pos << " len="<<len);
bool ok = do_send_chunk(chunk_start, len); // <====== ***
all_ok = all_ok && ok;
if (!all_ok) {
_mark_c("net/out/size", "do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr);
_mark ( "do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr);
_dbg1("do_send() SEND was aborted in middle of big package - this is mostly harmless "
<< " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << cb);
return false; // partial failure in sending
}
pos = pos+len; ASRT(pos >0);
// (in catch block, or uniq pointer) delete buf;
} // each chunk
_mark_c("net/out/size", "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr);
_mark ( "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr);
_info_c("net/sleepRPC", "do_send() m_connection_type = " << m_connection_type);
return all_ok; // done - e.g. queued - all the chunks of current do_send call
} // LOCK: chunking
} // a big block (to be chunked) - all chunks
else { // small block
return do_send_chunk(ptr,cb); // just send as 1 big chunk
}
CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false);
} // do_send()
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb)
bool connection<t_protocol_handler>::do_send_chunk(const void* ptr, size_t cb)
{
TRY_ENTRY();
// Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
@ -294,49 +399,80 @@ PRAGMA_WARNING_DISABLE_VS(4355)
if(m_was_shutdown)
return false;
LOG_PRINT("[sock " << socket_.native_handle() << "] SEND " << cb, LOG_LEVEL_4);
//_info("[sock " << socket_.native_handle() << "] SEND " << cb);
context.m_last_send = time(NULL);
context.m_send_cnt += cb;
//some data should be wrote to stream
//request complete
epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock);
if(m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT)
do_send_handler_start( ptr , cb ); // (((H)))
epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock); // *** critical ***
long int retry=0;
const long int retry_limit = 5*4;
while (m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT)
{
send_guard.unlock();
LOG_PRINT_L2("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
close();
return false;
retry++;
/* if ( ::cryptonote::core::get_is_stopping() ) { // TODO re-add fast stop
_fact("ABORT queue wait due to stopping");
return false; // aborted
}*/
long int ms = 250 + (rand()%50);
_info_c("net/sleep", "Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep
boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) );
_dbg1("sleep for queue: " << ms);
if (retry > retry_limit) {
send_guard.unlock();
_erro("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
// _mark_c("net/sleep", "send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
close();
return false;
}
}
m_send_que.resize(m_send_que.size()+1);
m_send_que.back().assign((const char*)ptr, cb);
if(m_send_que.size() > 1)
{
//active operation should be in progress, nothing to do, just wait last operation callback
}else
{
//no active operation
if(m_send_que.size()!=1)
{
LOG_ERROR("Looks like no active operations, but send que size != 1!!");
return false;
}
{ // active operation should be in progress, nothing to do, just wait last operation callback
auto size_now = cb;
_info_c("net/out/size", "do_send() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size());
do_send_handler_delayed( ptr , size_now ); // (((H)))
boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()),
//strand_.wrap(
boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2)
//)
);
LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size());
}
else
{ // no active operation
if(m_send_que.size()!=1)
{
_erro("Looks like no active operations, but send que size != 1!!");
return false;
}
auto size_now = m_send_que.front().size();
_mark_c("net/out/size", "do_send() NOW SENSD: packet="<<size_now<<" B");
do_send_handler_write( ptr , size_now ); // (((H)))
ASRT( size_now == m_send_que.front().size() );
boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now ) ,
//strand_.wrap(
boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2)
//)
);
//_dbg3("(chunk): " << size_now);
logger_handle_net_write(size_now);
//_info("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size());
}
do_send_handler_stop( ptr , cb );
return true;
CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false);
}
} // do_send_chunk
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
bool connection<t_protocol_handler>::shutdown()
@ -353,7 +489,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
bool connection<t_protocol_handler>::close()
{
TRY_ENTRY();
LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Que Shutdown called.");
//_info("[sock " << socket_.native_handle() << "] Que Shutdown called.");
size_t send_que_size = 0;
CRITICAL_REGION_BEGIN(m_send_que_lock);
send_que_size = m_send_que.size();
@ -376,7 +512,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
if (e)
{
LOG_PRINT_L1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value());
_dbg1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value());
shutdown();
return;
}
@ -385,7 +521,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
CRITICAL_REGION_BEGIN(m_send_que_lock);
if(m_send_que.empty())
{
LOG_ERROR("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!");
_erro("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!");
return;
}
@ -399,10 +535,17 @@ PRAGMA_WARNING_DISABLE_VS(4355)
}else
{
//have more data to send
boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()),
//strand_.wrap(
boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2));
//);
auto size_now = m_send_que.front().size();
_mark_c("net/out/size", "handle_write() NOW SENDS: packet="<<size_now<<" B" <<", from queue size="<<m_send_que.size());
do_send_handler_write_from_queue(e, m_send_que.front().size() , m_send_que.size()); // (((H)))
ASRT( size_now == m_send_que.front().size() );
boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now) ,
// strand_.wrap(
boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2)
// )
);
//_dbg3("(normal)" << size_now);
logger_handle_net_write(size_now);
}
CRITICAL_REGION_END();
@ -412,6 +555,13 @@ PRAGMA_WARNING_DISABLE_VS(4355)
}
CATCH_ENTRY_L0("connection<t_protocol_handler>::handle_write", void());
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
void connection<t_protocol_handler>::setRPcStation()
{
m_connection_type = RPC;
_fact_c("net/sleepRPC", "set m_connection_type = RPC ");
}
/************************************************************************/
/* */
/************************************************************************/
@ -420,19 +570,27 @@ PRAGMA_WARNING_DISABLE_VS(4355)
m_io_service_local_instance(new boost::asio::io_service()),
io_service_(*m_io_service_local_instance.get()),
acceptor_(io_service_),
new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)),
m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0)
m_stop_signal_sent(false), m_port(0),
m_sock_count(0), m_sock_number(0), m_threads_count(0),
m_pfilter(NULL), m_thread_index(0),
new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter))
{
create_server_type_map();
m_thread_name_prefix = "NET";
type = NET;
}
template<class t_protocol_handler>
boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service):
boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service, t_server_role s_type):
io_service_(extarnal_io_service),
acceptor_(io_service_),
new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)),
m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0)
m_stop_signal_sent(false), m_port(0),
m_sock_count(0), m_sock_number(0), m_threads_count(0),
m_pfilter(NULL), m_thread_index(0),
type(NET),
new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter))
{
create_server_type_map();
m_thread_name_prefix = "NET";
}
//---------------------------------------------------------------------------------
@ -444,6 +602,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
void boosted_tcp_server<t_protocol_handler>::create_server_type_map()
{
server_type_map["NET"] = t_server_role::NET;
server_type_map["RPC"] = t_server_role::RPC;
server_type_map["P2P"] = t_server_role::P2P;
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string address)
{
TRY_ENTRY();
@ -491,6 +657,7 @@ POP_WARNINGS
std::string thread_name = std::string("[") + m_thread_name_prefix;
thread_name += boost::to_string(local_thr_index) + "]";
log_space::log_singletone::set_thread_log_prefix(thread_name);
// _fact("Thread name: " << m_thread_name_prefix);
while(!m_stop_signal_sent)
{
try
@ -499,14 +666,14 @@ POP_WARNINGS
}
catch(const std::exception& ex)
{
LOG_ERROR("Exception at server worker thread, what=" << ex.what());
_erro("Exception at server worker thread, what=" << ex.what());
}
catch(...)
{
LOG_ERROR("Exception at server worker thread, unknown execption");
_erro("Exception at server worker thread, unknown execption");
}
}
LOG_PRINT_L4("Worker thread finished");
//_info("Worker thread finished");
return true;
CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::worker_thread", false);
}
@ -515,6 +682,9 @@ POP_WARNINGS
void boosted_tcp_server<t_protocol_handler>::set_threads_prefix(const std::string& prefix_name)
{
m_thread_name_prefix = prefix_name;
type = server_type_map[m_thread_name_prefix];
_note("Set server type to: " << type);
_note("Set server type to: " << m_thread_name_prefix);
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
@ -539,32 +709,38 @@ POP_WARNINGS
{
boost::shared_ptr<boost::thread> thread(new boost::thread(
attrs, boost::bind(&boosted_tcp_server<t_protocol_handler>::worker_thread, this)));
_note("Run server thread name: " << m_thread_name_prefix);
m_threads.push_back(thread);
}
CRITICAL_REGION_END();
// Wait for all threads in the pool to exit.
if(wait)
if (wait) // && ! ::cryptonote::core::get_is_stopping()) // TODO fast_exit
{
for (std::size_t i = 0; i < m_threads.size(); ++i)
m_threads[i]->join();
_fact("JOINING all threads");
for (std::size_t i = 0; i < m_threads.size(); ++i) {
m_threads[i]->join();
}
_fact("JOINING all threads - almost");
m_threads.clear();
_fact("JOINING all threads - DONE");
}else
{
}
else {
_dbg1("Reiniting OK.");
return true;
}
if(wait && !m_stop_signal_sent)
{
//some problems with the listening socket ?..
LOG_PRINT_L0("Net service stopped without stop request, restarting...");
_dbg1("Net service stopped without stop request, restarting...");
if(!this->init_server(m_port, m_address))
{
LOG_PRINT_L0("Reiniting service failed, exit.");
_dbg1("Reiniting service failed, exit.");
return false;
}else
{
LOG_PRINT_L0("Reiniting OK.");
_dbg1("Reiniting OK.");
}
}
}
@ -597,7 +773,7 @@ POP_WARNINGS
{
if(m_threads[i]->joinable() && !m_threads[i]->try_join_for(ms))
{
LOG_PRINT_L0("Interrupting thread " << m_threads[i]->native_handle());
_dbg1("Interrupting thread " << m_threads[i]->native_handle());
m_threads[i]->interrupt();
}
}
@ -626,19 +802,22 @@ POP_WARNINGS
TRY_ENTRY();
if (!e)
{
connection_ptr conn(std::move(new_connection_));
new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter));
if (type == RPC) {
new_connection_->setRPcStation();
_note("New server for RPC connections");
}
connection_ptr conn(std::move(new_connection_));
new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter));
acceptor_.async_accept(new_connection_->socket(),
boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this,
boost::asio::placeholders::error));
bool r = conn->start(true, 1 < m_threads_count);
if (!r)
LOG_ERROR("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count);
_erro("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count);
}else
{
LOG_ERROR("Some problems at accept: " << e.message() << ", connections_count = " << m_sockets_count);
_erro("Some problems at accept: " << e.message() << ", connections_count = " << m_sock_count);
}
CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::handle_accept", void());
}
@ -648,7 +827,7 @@ POP_WARNINGS
{
TRY_ENTRY();
connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter) );
connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter) );
boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket();
//////////////////////////////////////////////////////////////////////////
@ -658,7 +837,7 @@ POP_WARNINGS
boost::asio::ip::tcp::resolver::iterator end;
if(iterator == end)
{
LOG_ERROR("Failed to resolve " << adr);
_erro("Failed to resolve " << adr);
return false;
}
//////////////////////////////////////////////////////////////////////////
@ -704,7 +883,7 @@ POP_WARNINGS
{
//timeout
sock_.close();
LOG_PRINT_L3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")");
_dbg3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")");
return false;
}
}
@ -712,21 +891,21 @@ POP_WARNINGS
if (ec || !sock_.is_open())
{
LOG_PRINT("Some problems at connect, message: " << ec.message(), LOG_LEVEL_3);
_dbg3("Some problems at connect, message: " << ec.message());
return false;
}
LOG_PRINT_L3("Connected success to " << adr << ':' << port);
_dbg3("Connected success to " << adr << ':' << port);
bool r = new_connection_l->start(false, 1 < m_threads_count);
if (r)
{
new_connection_l->get_context(conn_context);
//new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter));
//new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_pfilter));
}
else
{
LOG_ERROR("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count);
_erro("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count);
}
return r;
@ -738,7 +917,7 @@ POP_WARNINGS
bool boosted_tcp_server<t_protocol_handler>::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_callback cb, const std::string& bind_ip)
{
TRY_ENTRY();
connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter) );
connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter) );
boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket();
//////////////////////////////////////////////////////////////////////////
@ -748,7 +927,7 @@ POP_WARNINGS
boost::asio::ip::tcp::resolver::iterator end;
if(iterator == end)
{
LOG_ERROR("Failed to resolve " << adr);
_erro("Failed to resolve " << adr);
return false;
}
//////////////////////////////////////////////////////////////////////////
@ -768,7 +947,7 @@ POP_WARNINGS
{
if(error != boost::asio::error::operation_aborted)
{
LOG_PRINT_L3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")");
_dbg3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")");
new_connection_l->socket().close();
}
});
@ -785,7 +964,7 @@ POP_WARNINGS
cb(conn_context, boost::asio::error::operation_aborted);//this mean that deadline timer already queued callback with cancel operation, rare situation
}else
{
LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port <<
_dbg3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port <<
" from " << lep.address().to_string() << ':' << lep.port());
bool r = new_connection_l->start(false, 1 < m_threads_count);
if (r)
@ -795,13 +974,13 @@ POP_WARNINGS
}
else
{
LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port);
_dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port);
cb(conn_context, boost::asio::error::fault);
}
}
}else
{
LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port <<
_dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port <<
" from " << lep.address().to_string() << ':' << lep.port() << ": " << ec_.message() << ':' << ec_.value());
cb(conn_context, ec_);
}
@ -809,6 +988,6 @@ POP_WARNINGS
return true;
CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::connect_async", false);
}
}
}
} // namespace
} // namespace
PRAGMA_WARNING_POP

9
contrib/epee/include/net/levin_protocol_handler_async.h
View File

@ -81,6 +81,7 @@ public:
async_protocol_handler_config():m_pcommands_handler(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE)
{}
void del_connections(size_t count);
};
@ -669,6 +670,14 @@ void async_protocol_handler_config<t_connection_context>::del_connection(async_p
}
//------------------------------------------------------------------------------------------
template<class t_connection_context>
void async_protocol_handler_config<t_connection_context>::del_connections(size_t count) // TODO
{
CRITICAL_REGION_BEGIN(m_connects_lock);
m_connects.clear();
CRITICAL_REGION_END();
}
//------------------------------------------------------------------------------------------
template<class t_connection_context>
void async_protocol_handler_config<t_connection_context>::add_connection(async_protocol_handler<t_connection_context>* pconn)
{
CRITICAL_REGION_BEGIN(m_connects_lock);

14
contrib/otshell_utils/CMakeLists.txt Normal file
View File

@ -0,0 +1,14 @@
cmake_minimum_required (VERSION 2.6)
project (otshell CXX)
# Add executable
file(GLOB otshell_utils_sources # All files in directory:
"*.h"
"*.hpp"
"*.cpp"
)
add_library (otshell_utils STATIC ${otshell_utils_sources})
set_target_properties (otshell_utils PROPERTIES OUTPUT_NAME "otshell_utils")
#target_link_libraries (upnpc-static ${LDLIBS}) # to add used libs

21
contrib/otshell_utils/LICENCE.txt Normal file
View File

@ -0,0 +1,21 @@
This are some files also from OpenTransactions / otshell project,
developed thanks to the awesome OpenTransaction project, organization and developers :)
Parts of code here was also developed thanks to the excellent Monero project,
thanks to Monero project, organization and developers :)
[Some] files/code here (in external/otshell_utils) are under licence defined in
src/doc/LICENCE-otshell.txt ;
Others are from monero, with licence in src/doc/LICENCE-monero.txt ;
For me (rfree) the licence seem compatbile so no problem, personally (as author of many parts of the code,
possibly not all) I do not worry who uses it how; I'am not a lawyer.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Please share :-) This licence can be used e.g. for parts of code that are usable in both open-source FOSS project
Monero and Open Transactions, to share and develop both faster.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

116
contrib/otshell_utils/ccolor.cpp Normal file
View File

@ -0,0 +1,116 @@
#include "ccolor.hpp"
#include <cstdarg>
// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal
// from http://wiznet.gr/src/ccolor.zip
// edited by rfree - as part of https://github.com/rfree/Open-Transactions/
using namespace std;
#ifdef _MSC_VER
#define snprintf c99_snprintf
inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) {
int count = -1;
if (size != 0)
count = _vsnprintf_s(str, size, _TRUNCATE, format, ap);
if (count == -1)
count = _vscprintf(format, ap);
return count;
}
inline int c99_snprintf(char* str, size_t size, const char* format, ...) {
int count;
va_list ap;
va_start(ap, format);
count = c99_vsnprintf(str, size, format, ap);
va_end(ap);
return count;
}
#endif // _MSC_VER
#define CC_CONSOLE_COLOR_DEFAULT "\033[0m"
#define CC_FORECOLOR(C) "\033[" #C "m"
#define CC_BACKCOLOR(C) "\033[" #C "m"
#define CC_ATTR(A) "\033[" #A "m"
namespace zkr
{
enum Color
{
Black,
Red,
Green,
Yellow,
Blue,
Magenta,
Cyan,
White,
Default = 9
};
enum Attributes
{
Reset,
Bright,
Dim,
Underline,
Blink,
Reverse,
Hidden
};
char * cc::color(int attr, int fg, int bg)
{
static const int size = 20;
static char command[size];
/* Command is the control command to the terminal */
snprintf(command, size, "%c[%d;%d;%dm", 0x1B, attr, fg + 30, bg + 40);
return command;
}
const char *cc::console = CC_CONSOLE_COLOR_DEFAULT;
const char *cc::underline = CC_ATTR(4);
const char *cc::bold = CC_ATTR(1);
const char *cc::fore::black = CC_FORECOLOR(30);
const char *cc::fore::blue = CC_FORECOLOR(34);
const char *cc::fore::red = CC_FORECOLOR(31);
const char *cc::fore::magenta = CC_FORECOLOR(35);
const char *cc::fore::green = CC_FORECOLOR(92);
const char *cc::fore::cyan = CC_FORECOLOR(36);
const char *cc::fore::yellow = CC_FORECOLOR(33);
const char *cc::fore::white = CC_FORECOLOR(37);
const char *cc::fore::console = CC_FORECOLOR(39);
const char *cc::fore::lightblack = CC_FORECOLOR(90);
const char *cc::fore::lightblue = CC_FORECOLOR(94);
const char *cc::fore::lightred = CC_FORECOLOR(91);
const char *cc::fore::lightmagenta = CC_FORECOLOR(95);
const char *cc::fore::lightgreen = CC_FORECOLOR(92);
const char *cc::fore::lightcyan = CC_FORECOLOR(96);
const char *cc::fore::lightyellow = CC_FORECOLOR(93);
const char *cc::fore::lightwhite = CC_FORECOLOR(97);
const char *cc::back::black = CC_BACKCOLOR(40);
const char *cc::back::blue = CC_BACKCOLOR(44);
const char *cc::back::red = CC_BACKCOLOR(41);
const char *cc::back::magenta = CC_BACKCOLOR(45);
const char *cc::back::green = CC_BACKCOLOR(42);
const char *cc::back::cyan = CC_BACKCOLOR(46);
const char *cc::back::yellow = CC_BACKCOLOR(43);
const char *cc::back::white = CC_BACKCOLOR(47);
const char *cc::back::console = CC_BACKCOLOR(49);
const char *cc::back::lightblack = CC_BACKCOLOR(100);
const char *cc::back::lightblue = CC_BACKCOLOR(104);
const char *cc::back::lightred = CC_BACKCOLOR(101);
const char *cc::back::lightmagenta = CC_BACKCOLOR(105);
const char *cc::back::lightgreen = CC_BACKCOLOR(102);
const char *cc::back::lightcyan = CC_BACKCOLOR(106);
const char *cc::back::lightyellow = CC_BACKCOLOR(103);
const char *cc::back::lightwhite = CC_BACKCOLOR(107);
}

73
contrib/otshell_utils/ccolor.hpp Normal file
View File

@ -0,0 +1,73 @@
// ccolor.hpp
// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal
// from http://wiznet.gr/src/ccolor.zip
// edited by rfree - as part of https://github.com/rfree/Open-Transactions/
#ifndef INCLUDE_OT_ccolor
#define INCLUDE_OT_ccolor
#include <iostream>
#include <stdio.h>
namespace zkr
{
class cc
{
public:
class fore
{
public:
static const char *black;
static const char *blue;
static const char *red;
static const char *magenta;
static const char *green;
static const char *cyan;
static const char *yellow;
static const char *white;
static const char *console;
static const char *lightblack;
static const char *lightblue;
static const char *lightred;
static const char *lightmagenta;
static const char *lightgreen;
static const char *lightcyan;
static const char *lightyellow;
static const char *lightwhite;
};
class back
{
public:
static const char *black;
static const char *blue;
static const char *red;
static const char *magenta;
static const char *green;
static const char *cyan;
static const char *yellow;
static const char *white;
static const char *console;
static const char *lightblack;
static const char *lightblue;
static const char *lightred;
static const char *lightmagenta;
static const char *lightgreen;
static const char *lightcyan;
static const char *lightyellow;
static const char *lightwhite;
};
static char *color(int attr, int fg, int bg);
static const char *console;
static const char *underline;
static const char *bold;
};
}
#endif

51
contrib/otshell_utils/lib_common1.hpp Normal file
View File

@ -0,0 +1,51 @@
/* See other files here for the LICENCE that applies here. */
#ifndef INCLUDE_OT_NEWCLI_COMMON1
#define INCLUDE_OT_NEWCLI_COMMON1
#include <string>
#include <cstring>
#include <vector>
#include <map>
#include <list>
#include <algorithm>
#include <iostream>
#include <fstream>
#include <sstream>
#include <set>
#include <iterator>
#include <stdexcept>
#include <functional>
#include <memory>
#include <thread>
#include <mutex>
// list of thigs from libraries that we pull into namespace nOT::nNewcli
// we might still need to copy/paste it in few places to make IDEs pick it up correctly
#define INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 \
using std::string; \
using std::vector; \
using std::vector; \
using std::list; \
using std::set; \
using std::map; \
using std::ostream; \
using std::istream; \
using std::cin; \
using std::cerr; \
using std::cout; \
using std::cerr; \
using std::endl; \
using std::function; \
using std::unique_ptr; \
using std::shared_ptr; \
using std::weak_ptr; \
using std::enable_shared_from_this; \
using std::mutex; \
using std::lock_guard; \
#endif

69
contrib/otshell_utils/runoptions.cpp Normal file
View File

@ -0,0 +1,69 @@
/* See other files here for the LICENCE that applies here. */
/* See header file .hpp for info */
#include "runoptions.hpp"
#include "lib_common1.hpp"
namespace nOT {
INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces
// (no debug - this is the default)
// +nodebug (no debug)
// +debug ...... --asdf
// +debug +debugcerr .... --asfs
// +debug +debugfile .... --asfs
cRunOptions::cRunOptions()
: mRunMode(eRunModeCurrent), mDebug(false), mDebugSendToFile(false), mDebugSendToCerr(false)
,mDoRunDebugshow(false)
{ }
vector<string> cRunOptions::ExecuteRunoptionsAndRemoveThem(const vector<string> & args) {
vector<string> arg_clear; // will store only the arguments that are not removed
for (auto arg : args) {
bool thisIsRunoption=false;
if (arg.size()>0) {
if (arg.at(0) == '+') thisIsRunoption=true;
}
if (thisIsRunoption) Exec(arg); // ***
if (! thisIsRunoption) arg_clear.push_back(arg);
}
Normalize();
return arg_clear;
}
void cRunOptions::Exec(const string & runoption) { // eg: Exec("+debug");
if (runoption == "+nodebug") { mDebug=false; }
else if (runoption == "+debug") { mDebug=true; }
else if (runoption == "+debugcerr") { mDebug=true; mDebugSendToCerr=true; }
else if (runoption == "+debugfile") { mDebug=true; mDebugSendToFile=true; }
else if (runoption == "+demo") { mRunMode=eRunModeDemo; }
else if (runoption == "+normal") { mRunMode=eRunModeNormal; }
else if (runoption == "+current") { mRunMode=eRunModeCurrent; }
else if (runoption == "+debugshow") { mDebug=true; mDebugSendToCerr=true; mDoRunDebugshow=true; }
else {
cerr << "Unknown runoption in Exec: '" << runoption << "'" << endl;
throw std::runtime_error("Unknown runoption");
}
// cerr<<"debug="<<mDebug<<endl;
}
void cRunOptions::Normalize() {
if (mDebug) {
if (!( mDebugSendToFile || mDebugSendToCerr )) mDebugSendToCerr=true; // if debug is on then send to something, e.g. to cerr
}
}
cRunOptions gRunOptions; // (extern)
}; // namespace OT

58
contrib/otshell_utils/runoptions.hpp Normal file
View File

@ -0,0 +1,58 @@
/* See other files here for the LICENCE that applies here. */
/*
Template for new files, replace word "template" and later delete this line here.
*/
#ifndef INCLUDE_OT_NEWCLI_runoptions_hpp
#define INCLUDE_OT_NEWCLI_runoptions_hpp
#include "lib_common1.hpp"
namespace nOT {
INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces
/** Global options to run this program main() Eg used for developer's special options like +setdemo +setdebug.
This is NOT for all the other options that are parsed and executed by program. */
class cRunOptions {
public:
enum tRunMode { ///< Type of run mode - is this normal, or demonstration etc.
eRunModeCurrent=1, ///< currently developed version
eRunModeDemo, ///< best currently available Demo of something nice
eRunModeNormal, ///< do the normal things that the program should do
};
private:
tRunMode mRunMode; ///< selected run mode
bool mDebug; // turn debug on, Eg: +debug without it probably nothing will be written to debug (maybe just error etc)
bool mDebugSendToFile; // send to file, Eg: for +debugfile ; also turns on debug
bool mDebugSendToCerr; // send to cerr, Eg: for +debugcerr ; also turns on debug
// if debug is set but not any other DebugSend* then we will default to sending to debugcerr
bool mDoRunDebugshow;
public:
tRunMode getTRunMode() const { return mRunMode; }
bool getDebug() const { return mDebug; }
bool getDebugSendToFile() const { return mDebugSendToFile; }
bool getDebugSendToCerr() const { return mDebugSendToCerr; }
bool getDoRunDebugshow() const { return mDoRunDebugshow; }
cRunOptions();
vector<string> ExecuteRunoptionsAndRemoveThem(const vector<string> & args);
void Exec(const string & runoption); // eg: Exec("+debug");
void Normalize();
};
extern cRunOptions gRunOptions;
}; // namespace nOT
#endif

612
contrib/otshell_utils/utils.cpp Normal file
View File

@ -0,0 +1,612 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug
/* See other files here for the LICENCE that applies here. */
/* See header file .hpp for info */
#include <algorithm>
#include <functional>
#include <cctype>
#include <locale>
#include <fstream>
#include <iostream>
#include <iomanip>
#include "utils.hpp"
#include "ccolor.hpp"
#include "lib_common1.hpp"
#include "runoptions.hpp"
#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined (WIN64)
#define OS_TYPE_WINDOWS
#elif defined(__unix__) || defined(__posix) || defined(__linux) || defined(__darwin) || defined(__APPLE__) || defined(__clang__)
#define OS_TYPE_POSIX
#else
#warning "Compiler/OS platform is not recognized"
#warning "Just assuming it will work as POSIX then"
#define OS_TYPE_POSIX
#endif
#if defined(OS_TYPE_WINDOWS)
#include <windows.h>
#elif defined(OS_TYPE_POSIX)
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#else
#error "Compiler/OS platform detection failed - not supported"
#endif
namespace nOT {
namespace nUtils {
INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces
myexception::myexception(const char * what)
: std::runtime_error(what)
{ }
myexception::myexception(const std::string &what)
: std::runtime_error(what)
{ }
void myexception::Report() const {
_erro("Error: " << what());
}
//myexception::~myexception() { }
// ====================================================================
// text trimming
// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
std::string & ltrim(std::string &s) {
s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun<int, int>(std::isspace))));
return s;
}
std::string & rtrim(std::string &s) {
s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end());
return s;
}
std::string & trim(std::string &s) {
return ltrim(rtrim(s));
}
std::string get_current_time()
{
std::stringstream stream;
struct tm * date;
std::chrono::high_resolution_clock::time_point now = std::chrono::high_resolution_clock::now();
time_t time_now;
time_now = std::chrono::high_resolution_clock::to_time_t(now);
date = std::localtime(& time_now);
char date_buff[32];
std::strftime(date_buff, sizeof(date_buff), "%d-%b-%Y %H:%M:%S.", date);
stream << date_buff;
std::chrono::high_resolution_clock::duration duration = now.time_since_epoch();
int64_t micro = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
micro %= 1000000;
stream << std::setfill('0') << std::setw(3) << micro;
return stream.str();
}
cNullstream g_nullstream; // extern a stream that does nothing (eats/discards data)
std::mutex gLoggerGuard; // extern
// ====================================================================
namespace nDetail {
const char* DbgShortenCodeFileName(const char *s) {
const char *p = s;
const char *a = s;
bool inc=1;
while (*p) {
++p;
if (inc && ('\0' != * p)) { a=p; inc=false; } // point to the current character (if valid) becasue previous one was slash
if ((*p)=='/') { a=p; inc=true; } // point at current slash (but set inc to try to point to next character)
}
return a;
}
}
// a workaround for MSVC compiler; e.g. see https://bugs.webkit.org/show_bug.cgi?format=multiple&id=125795
#ifndef _MSC_VER
template<typename T, typename ...Args>
std::unique_ptr<T> make_unique( Args&& ...args )
{
return std::unique_ptr<T>( new T( std::forward<Args>(args)... ) );
}
#else
using std::make_unique;
#endif
// ====================================================================
char cFilesystemUtils::GetDirSeparator() {
// TODO nicer os detection?
#if defined(OS_TYPE_POSIX)
return '/';
#elif defined(OS_TYPE_WINDOWS)
return '\\';
#else
#error "Do not know how to compile this for your platform."
#endif
}
bool cFilesystemUtils::CreateDirTree(const std::string & dir, bool only_below) {
const bool dbg=false;
//struct stat st;
const char dirch = cFilesystemUtils::GetDirSeparator();
std::istringstream iss(dir);
string part, sofar="";
if (dir.size()<1) return false; // illegal name
// dir[0] is valid from here
if (only_below && (dir[0]==dirch)) return false; // no jumping to top (on any os)
while (getline(iss,part,dirch)) {
if (dbg) cout << '['<<part<<']' << endl;
sofar += part;
if (part.size()<1) return false; // bad format?
if ((only_below) && (part=="..")) return false; // going up
if (dbg) cout << "test ["<<sofar<<"]"<<endl;
// TODO nicer os detection?
#if defined(OS_TYPE_POSIX)
struct stat st;
bool exists = stat(sofar.c_str() ,&st) == 0; // *
if (exists) {
if (! S_ISDIR(st.st_mode)) {
// std::cerr << "This exists, but as a file: [" << sofar << "]" << (size_t)st.st_ino << endl;
return false; // exists but is a file nor dir
}
}
#elif defined(OS_TYPE_WINDOWS)
DWORD dwAttrib = GetFileAttributesA(sofar.c_str());
bool exists = (dwAttrib != INVALID_FILE_ATTRIBUTES && (dwAttrib & FILE_ATTRIBUTE_DIRECTORY));
#else
#error "Do not know how to compile this for your platform."
#endif
if (!exists) {
if (dbg) cout << "mkdir ["<<sofar<<"]"<<endl;
#if defined(OS_TYPE_POSIX)
bool ok = 0== mkdir(sofar.c_str(), 0700); // ***
#elif defined(OS_TYPE_WINDOWS)
bool ok = (bool) CreateDirectoryA(sofar.c_str(), NULL); // TODO use -W() after conversion to unicode UTF16
#else
#error "Do not know how to compile this for your platform."
#endif
if (!ok) return false;
}
sofar += cFilesystemUtils::GetDirSeparator();
}
return true;
}
// ====================================================================
namespace nDetail {
cDebugScopeGuard::cDebugScopeGuard() : mLevel(-1) {
}
cDebugScopeGuard::~cDebugScopeGuard() {
if (mLevel != -1) {
gCurrentLogger.write_stream(mLevel,mChan) << mMsg << " ... end" << gCurrentLogger.endline() << std::flush;
}
}
void cDebugScopeGuard::Assign(const string &chan, const int level, const string &msg) {
mChan=chan;
mLevel=level;
mMsg=msg;
}
}; // namespace nDetail
// ====================================================================
cLogger::cLogger() :
mStream(NULL),
mLevel(85),
mThread2Number_Biggest(0) // the CURRENT biggest value (no thread yet in map)
{
mStream = & std::cout;
Thread2Number( std::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1
}
cLogger::~cLogger() {
for (auto pair : mChannels) {
std::ofstream *ptr = pair.second;
delete ptr;
pair.second=NULL;
}
}
std::ostream & cLogger::write_stream(int level) {
return write_stream(level,"");
}
std::ostream & cLogger::write_stream(int level, const std::string & channel ) {
if ((level >= mLevel) && (mStream)) {
ostream & output = SelectOutput(level,channel);
output << icon(level) << ' ';
std::thread::id this_id = std::this_thread::get_id();
output << "{" << Thread2Number(this_id) << "} ";
return output;
}
return g_nullstream;
}
std::string cLogger::GetLogBaseDir() const {
return "log";
}
void cLogger::OpenNewChannel(const std::string & channel) {
size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparator());
// log/test/aaa
// ^----- last_split
string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparator() + channel.substr(0, last_split);
string basefile = channel.substr(last_split+1) + ".log";
string fname = dir + cFilesystemUtils::GetDirSeparator() + cFilesystemUtils::GetDirSeparator() + basefile;
_dbg1("Starting debug to channel file: " + fname + " in directory ["+dir+"]");
bool dirok = cFilesystemUtils::CreateDirTree(dir);
if (!dirok) { const string msg = "In logger failed to open directory (" + dir +")."; _erro(msg); throw std::runtime_error(msg); }
std::ofstream * thefile = new std::ofstream( fname.c_str() );
*thefile << "====== (Log opened: " << fname << ") ======" << endl;
mChannels.insert( std::pair<string,std::ofstream*>(channel , thefile ) );
}
std::ostream & cLogger::SelectOutput(int level, const std::string & channel) {
if (channel=="") return *mStream;
auto obj = mChannels.find(channel);
if (obj == mChannels.end()) { // new channel
OpenNewChannel(channel);
return SelectOutput(level,channel);
}
else { // existing
return * obj->second;
}
}
void cLogger::setOutStreamFile(const string &fname) { // switch to using this file
_mark("WILL SWITCH DEBUG NOW to file: " << fname);
mOutfile = make_unique<std::ofstream>(fname);
mStream = & (*mOutfile);
_mark("Started new debug, to file: " << fname);
}
void cLogger::setOutStreamFromGlobalOptions() {
if ( gRunOptions.getDebug() ) {
if ( gRunOptions.getDebugSendToFile() ) {
mOutfile = make_unique<std::ofstream> ("debuglog.txt");
mStream = & (*mOutfile);
}
else if ( gRunOptions.getDebugSendToCerr() ) {
mStream = & std::cerr;
}
else {
mStream = & g_nullstream;
}
}
else {
mStream = & g_nullstream;
}
}
void cLogger::setDebugLevel(int level) {
bool note_before = (mLevel > level); // report the level change before or after the change? (on higher level)
if (note_before) _note("Setting debug level to "<<level);
mLevel = level;
if (!note_before) _note("Setting debug level to "<<level);
}
std::string cLogger::icon(int level) const {
// TODO replan to avoid needles converting back and forth char*, string etc
using namespace zkr;
if (level >= 100) return cc::back::red + ToStr(cc::fore::black) + ToStr("ERROR ") + ToStr(cc::fore::lightyellow) + " " ;
if (level >= 90) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("Warn ") + ToStr(cc::fore::red)+ " " ;
if (level >= 80) return cc::back::lightmagenta + ToStr(cc::fore::black) + ToStr("MARK "); //+ zkr::cc::console + ToStr(cc::fore::lightmagenta)+ " ";
if (level >= 75) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("FACT ") + zkr::cc::console + ToStr(cc::fore::lightyellow)+ " ";
if (level >= 70) return cc::fore::green + ToStr("Note ");
if (level >= 50) return cc::fore::cyan + ToStr("info ");
if (level >= 40) return cc::fore::lightwhite + ToStr("dbg ");
if (level >= 30) return cc::fore::lightblue + ToStr("dbg ");
if (level >= 20) return cc::fore::blue + ToStr("dbg ");
return " ";
}
std::string cLogger::endline() const {
return ToStr("") + zkr::cc::console + ToStr("\n"); // TODO replan to avoid needles converting back and forth char*, string etc
}
int cLogger::Thread2Number(const std::thread::id id) {
auto found = mThread2Number.find( id );
if (found == mThread2Number.end()) { // new one
mThread2Number_Biggest++;
mThread2Number[id] = mThread2Number_Biggest;
return mThread2Number_Biggest;
// _info("(This is a new thread)"); // recursion!
} else {
return mThread2Number[id];
}
}
// ====================================================================
// object gCurrentLogger is defined later - in global namespace below
// ====================================================================
// vector debug
void DisplayStringEndl(std::ostream & out, const std::string text) {
out << text;
out << std::endl;
}
std::string SpaceFromEscape(const std::string &s) {
std::ostringstream newStr;
for(size_t i = 0; i < s.length();i++) {
if(s[i] == '\\' && s[i+1] ==32)
newStr<<"";
else
newStr<<s[i];
}
return newStr.str();
}
std::string EscapeFromSpace(const std::string &s) {
std::ostringstream newStr;
for(size_t i = 0; i < s.length();i++) {
if(s[i] == 32)
newStr << "\\" << " ";
else
newStr << s[i];
}
return newStr.str();
}
std::string EscapeString(const std::string &s) {
std::ostringstream newStr;
for(size_t i = 0; i < s.length();i++) {
if(s[i] >=32 && s[i] <= 126)
newStr<<s[i];
else
newStr<<"\\"<< (int) s[i];
}
return newStr.str();
}
bool CheckIfBegins(const std::string & beggining, const std::string & all) {
if (all.compare(0, beggining.length(), beggining) == 0) {
return 1;
}
else {
return 0;
}
}
bool CheckIfEnds (std::string const & ending, std::string const & all){
if (all.length() >= ending.length()) {
return (0 == all.compare (all.length() - ending.length(), ending.length(), ending));
} else {
return false;
}
}
vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib) {
vector<string> ret;
for ( auto rec : possib) { // check of possibilities
if (CheckIfBegins(sofar,rec)) {
rec = EscapeFromSpace(rec);
ret.push_back(rec); // this record matches
}
}
return ret;
}
char GetLastChar(const std::string & str) { // TODO unicode?
auto s = str.length();
if (s==0) throw std::runtime_error("Getting last character of empty string (" + ToStr(s) + ")" + OT_CODE_STAMP);
return str.at( s - 1);
}
std::string GetLastCharIf(const std::string & str) { // TODO unicode?
auto s = str.length();
if (s==0) return ""; // empty string signalizes ther is nothing to be returned
return std::string( 1 , str.at( s - 1) );
}
// ====================================================================
// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no.
// Use it like this: ASRT( x>y ); with the semicolon at end, a clever trick forces this syntax :)
void Assert(bool result, const std::string &stamp, const std::string &condition) {
if (!result) {
_erro("Assert failed at "+stamp+": ASSERT( " << condition << ")");
throw std::runtime_error("Assert failed at "+stamp+": ASSERT( " + condition + ")");
}
}
// ====================================================================
// advanced string
const std::string GetMultiline(string endLine) {
std::string result(""); // Taken from OT_CLI_ReadUntilEOF
while (true) {
std::string input_line("");
if (std::getline(std::cin, input_line, '\n'))
{
input_line += "\n";
if (input_line[0] == '~')
break;
result += input_line;
}
if (std::cin.eof() )
{
std::cin.clear();
break;
}
if (std::cin.fail() )
{
std::cin.clear();
break;
}
if (std::cin.bad())
{
std::cin.clear();
break;
}
}
return result;
}
vector<string> SplitString(const string & str){
std::istringstream iss(str);
vector<string> vec { std::istream_iterator<string>{iss}, std::istream_iterator<string>{} };
return vec;
}
bool checkPrefix(const string & str, char prefix) {
if (str.at(0) == prefix)
return true;
return false;
}
// ====================================================================
// operation on files
#ifdef __unix
void cEnvUtils::GetTmpTextFile() {
// TODO make this name configurable (depending on project)
char filename[] = "/tmp/otshellutils_text.XXXXXX";
fd = mkstemp(filename);
if (fd == -1) {
_erro("Can't create the file: " << filename);
return;
}
mFilename = filename;
}
void cEnvUtils::CloseFile() {
close(fd);
unlink( mFilename.c_str() );
}
void cEnvUtils::OpenEditor() {
char* editor = std::getenv("OT_EDITOR"); //TODO Read editor from configuration file
if (editor == NULL)
editor = std::getenv("VISUAL");
if (editor == NULL)
editor = std::getenv("EDITOR");
string command;
if (editor != NULL)
command = ToStr(editor) + " " + mFilename;
else
command = "/usr/bin/editor " + mFilename;
_dbg3("Opening editor with command: " << command);
if ( system( command.c_str() ) == -1 )
_erro("Cannot execute system command: " << command);
}
const string cEnvUtils::ReadFromTmpFile() {
std::ifstream ifs(mFilename);
string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
return msg;
}
const string cEnvUtils::Compose() {
GetTmpTextFile();
OpenEditor();
string input = ReadFromTmpFile();
CloseFile();
return input;
}
#endif
const string cEnvUtils::ReadFromFile(const string path) {
std::ifstream ifs(path);
string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
return msg;
}
void hintingToTxt(std::fstream & file, string command, vector<string> &commands) {
if(file.good()) {
file<<command<<"~"<<endl;
for (auto a: commands) {
file <<a<< " ";
file.flush();
}
file<<endl;
}
}
string stringToColor(const string &hash) {
// Generete vector with all possible light colors
vector <string> lightColors;
using namespace zkr;
lightColors.push_back(cc::fore::lightblue);
lightColors.push_back(cc::fore::lightred);
lightColors.push_back(cc::fore::lightmagenta);
lightColors.push_back(cc::fore::lightgreen);
lightColors.push_back(cc::fore::lightcyan);
lightColors.push_back(cc::fore::lightyellow);
lightColors.push_back(cc::fore::lightwhite);
int sum=0;
for (auto ch : hash) sum+=ch;
auto color = sum%(lightColors.size()-1);
return lightColors.at( color );
}
// ====================================================================
// algorthms
}; // namespace nUtil
}; // namespace OT
// global namespace
const extern int _dbg_ignore = 0; // see description in .hpp
std::string GetObjectName() {
//static std::string * name=nullptr;
//if (!name) name = new std::string("(global)");
return "";
}
// ====================================================================
nOT::nUtils::cLogger gCurrentLogger;

446
contrib/otshell_utils/utils.hpp Normal file
View File

@ -0,0 +1,446 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug
/* See other files here for the LICENCE that applies here. */
#include "ccolor.hpp"
#ifndef INCLUDE_OT_NEWCLI_UTILS
#define INCLUDE_OT_NEWCLI_UTILS
#include "lib_common1.hpp"
#ifdef __unix
#include <unistd.h>
#endif
#ifndef CFG_WITH_TERMCOLORS
#error "You requested to turn off terminal colors (CFG_WITH_TERMCOLORS), however currently they are hardcoded (this option to turn them off is not yet implemented)."
#endif
///Macros related to automatic deduction of class name etc;
#define MAKE_CLASS_NAME(NAME) private: static std::string GetObjectName() { return #NAME; }
#define MAKE_STRUCT_NAME(NAME) private: static std::string GetObjectName() { return #NAME; } public:
namespace nOT {
namespace nUtils {
/// @brief general based for my runtime errors
class myexception : public std::runtime_error {
public:
myexception(const char * what);
myexception(const std::string &what);
//virtual ~myexception();
virtual void Report() const;
};
/// @macro Use this macro INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 as a shortcut for various using std::string etc.
INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces
// ======================================================================================
/// text trimming functions (they do mutate the passes string); they trim based on std::isspace. also return it's reference again
/// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
std::string & trim(std::string &s); ///< trim text http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
std::string & ltrim(std::string &s); ///< left trim
std::string & rtrim(std::string &s); ///< right trim
// ======================================================================================
std::string get_current_time();
// string conversions
template <class T>
std::string ToStr(const T & obj) {
std::ostringstream oss;
oss << obj;
return oss.str();
}
struct cNullstream : std::ostream {
cNullstream() : std::ios(0), std::ostream(0) {}
};
extern cNullstream g_nullstream; // a stream that does nothing (eats/discards data)
// ========== debug ==========
// _dbg_ignore is moved to global namespace (on purpose)
// TODO make _dbg_ignore thread-safe everywhere
extern std::mutex gLoggerGuard;
#define _debug_level_c(CHANNEL,LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \
nOT::nUtils::gLoggerGuard.try_lock(); \
gCurrentLogger.write_stream(LEVEL,CHANNEL) << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \
nOT::nUtils::gLoggerGuard.unlock(); \
} } while(0)
#define _debug_level(LEVEL,VAR) _debug_level_c("",LEVEL,VAR)
#define _dbg3(VAR) _debug_level( 20,VAR)
#define _dbg2(VAR) _debug_level( 30,VAR)
#define _dbg1(VAR) _debug_level( 40,VAR) // details
#define _info(VAR) _debug_level( 50,VAR) // more boring info
#define _note(VAR) _debug_level( 70,VAR) // info
#define _fact(VAR) _debug_level( 75,VAR) // interesting event
#define _mark(VAR) _debug_level( 80,VAR) // marked action
#define _warn(VAR) _debug_level( 90,VAR) // some problem
#define _erro(VAR) _debug_level(100,VAR) // error - report
#define _dbg3_c(C,VAR) _debug_level_c(C, 20,VAR)
#define _dbg2_c(C,VAR) _debug_level_c(C, 30,VAR)
#define _dbg1_c(C,VAR) _debug_level_c(C, 40,VAR) // details
#define _info_c(C,VAR) _debug_level_c(C, 50,VAR) // more boring info
#define _note_c(C,VAR) _debug_level_c(C, 70,VAR) // info
#define _fact_c(C,VAR) _debug_level_c(C, 75,VAR) // interesting event
#define _mark_c(C,VAR) _debug_level_c(C, 80,VAR) // marked action
#define _warn_c(C,VAR) _debug_level_c(C, 90,VAR) // some problem
#define _erro_c(C,VAR) _debug_level_c(C,100,VAR) // error - report
// lock // because od VAR
#define _scope_debug_level_c(CHANNEL,LEVEL,VAR) \
std::ostringstream debug_detail_oss; \
nOT::nUtils::gLoggerGuard.try_lock(); \
debug_detail_oss << OT_CODE_STAMP << ' ' << VAR ; \
nOT::nUtils::nDetail::cDebugScopeGuard debugScopeGuard; \
if (_dbg_ignore<LEVEL) debugScopeGuard.Assign(CHANNEL,LEVEL, debug_detail_oss.str()); \
if (_dbg_ignore<LEVEL) _debug_level_c(CHANNEL,LEVEL,debug_detail_oss.str() + " ... begin"); \
nOT::nUtils::gLoggerGuard.unlock();
#define _scope_debug_level(LEVEL,VAR) _scope_debug_level_c("",LEVEL,VAR)
#define _scope_dbg1(VAR) _scope_debug_level( 20,VAR)
#define _scope_dbg2(VAR) _scope_debug_level( 30,VAR)
#define _scope_dbg3(VAR) _scope_debug_level( 40,VAR) // details
#define _scope_info(VAR) _scope_debug_level( 50,VAR) // more boring info
#define _scope_note(VAR) _scope_debug_level( 70,VAR) // info
#define _scope_fact(VAR) _scope_debug_level( 75,VAR) // interesting event
#define _scope_mark(VAR) _scope_debug_level( 80,VAR) // marked action
#define _scope_warn(VAR) _scope_debug_level( 90,VAR) // some problem
#define _scope_erro(VAR) _scope_debug_level( 100,VAR) // error - report
/***
@brief do not use this namespace directly, it is implementation detail.
*/
namespace nDetail {
/***
@brief a Debug scope-guard, to log a debug message when current scope is left. Do NOT use this directly,
only use it via the macros like _scope_dbg1 etc.
*/
class cDebugScopeGuard {
protected:
string mMsg;
int mLevel;
string mChan;
public:
cDebugScopeGuard();
~cDebugScopeGuard();
void Assign(const string &chan, const int level, const string &msg);
};
const char* DbgShortenCodeFileName(const char *s); ///< Returns a pointer to some part of the string that was given, skipping directory names, for log/debug
}; // namespace nDetail
// ========== logger ==========
/***
@brief Class to write debug into. Used it by calling the debug macros _dbg1(...) _info(...) _erro(...) etc, NOT directly!
@author rfree (maintainer)
*/
class cLogger {
public:
cLogger();
~cLogger();
std::ostream & write_stream(int level); ///< starts a new message on given level (e.g. writes out the icon/tag) and returns stream to output to
std::ostream & write_stream(int level, const std::string & channel); ///< the same but with name of the debug channel
void setOutStreamFromGlobalOptions(); // set debug level, file etc - according to global Options
void setOutStreamFile(const std::string &fname); // switch to using this file
void setDebugLevel(int level); // change the debug level e.g. to mute debug from now
std::string icon(int level) const; ///< returns "icon" for given debug level. It is text, might include color controll characters
std::string endline() const; ///< returns string to be written at end of message
protected:
unique_ptr<std::ofstream> mOutfile;
std::ostream * mStream; ///< pointing only! can point to our own mOutfile, or maye to global null stream
std::map< std::string , std::ofstream * > mChannels; // the ofstream objects are owned by this class
int mLevel; ///< current debug level
std::ostream & SelectOutput(int level, const std::string & channel);
void OpenNewChannel(const std::string & channel);
std::string GetLogBaseDir() const;
std::map< std::thread::id , int > mThread2Number; // change long thread IDs into a short nice number to show
int mThread2Number_Biggest; // current biggest value held there (biggest key) - works as growing-only counter basically
int Thread2Number(const std::thread::id id); // convert the system's thread id into a nice short our id; make one if new thread
};
// ====================================================================
// vector debug
template <class T>
std::string vectorToStr(const T & v) {
std::ostringstream oss;
for(auto rec: v) {
oss << rec <<",";
}
return oss.str();
}
template <class T>
void DisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
std::copy( v.begin(), v.end(), std::ostream_iterator<T>(out, delim.c_str()) );
}
template <class T>
void EndlDisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
out << std::endl;
DisplayVector(out,v,delim);
}
template <class T>
void DisplayVectorEndl(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
DisplayVector(out,v,delim);
out << std::endl;
}
template <class T>
void DbgDisplayVector(const std::vector<T> &v, const std::string &delim=" ") {
std::cerr << "[";
std::copy( v.begin(), v.end(), std::ostream_iterator<T>(std::cerr, delim.c_str()) );
std::cerr << "]";
}
string stringToColor(const string &hash);
template <class T, class T2>
void DisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") {
auto *no_color = zkr::cc::fore::console;
for(auto var : m) {
out << stringToColor(var.first) << var.first << delim << var.second << no_color << endl;
}
}
template <class T, class T2>
void EndlDisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") {
out << endl;
for(auto var : m) {
out << var.first << delim << var.second << endl;
}
}
template <class T, class T2>
void DbgDisplayMap(const std::map<T, T2> &m, const std::string &delim=" ") {
for(auto var : m) {
std::cerr << var.first << delim << var.second << endl;
}
}
template <class T>
void DbgDisplayVectorEndl(const std::vector<T> &v, const std::string &delim=" ") {
DbgDisplayVector(v,delim);
std::cerr << std::endl;
}
void DisplayStringEndl(std::ostream & out, const std::string text);
bool CheckIfBegins(const std::string & beggining, const std::string & all);
bool CheckIfEnds (std::string const & ending, std::string const & all);
std::string SpaceFromEscape(const std::string &s);
std::string EscapeFromSpace(const std::string &s);
vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib);
char GetLastChar(const std::string & str);
std::string GetLastCharIf(const std::string & str); // TODO unicode?
std::string EscapeString(const std::string &s);
template <class T>
std::string DbgVector(const std::vector<T> &v, const std::string &delim="|") {
std::ostringstream oss;
oss << "[";
bool first=true;
for(auto vElement : v) { if (!first) oss<<delim; first=false; oss <<vElement ; }
oss << "]";
//std::copy( v.begin(), v.end(), std::ostream_iterator<T>(oss, delim.c_str()) );
return oss.str();
}
template <class T>
std::ostream & operator<<(std::ostream & os, const map< T, vector<T> > & obj){
os << "[";
for(auto const & elem : obj) {
os << " [" << elem.first << "=" << DbgVector(elem.second) << "] ";
}
os << "]";
return os;
}
template <class T, class T2>
std::string DbgMap(const map<T, T2> & map) {
std::ostringstream oss;
oss << map;
return oss.str();
}
// ====================================================================
// assert
// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no.
// Use it like this: ASRT( x>y ); with the semicolon at end, a clever trick forces this syntax :)
#define ASRT(x) do { if (!(x)) nOT::nUtils::Assert(false, OT_CODE_STAMP, #x); } while(0)
void Assert(bool result, const std::string &stamp, const std::string &condition);
// ====================================================================
// advanced string
const std::string GetMultiline(string endLine = "~");
vector<string> SplitString(const string & str);
bool checkPrefix(const string & str, char prefix = '^');
// ====================================================================
// nUse utils
enum class eSubjectType {Account, Asset, User, Server, Unknown};
string SubjectType2String(const eSubjectType & type);
eSubjectType String2SubjectType(const string & type);
// ====================================================================
// operation on files
/// @brief tools related to filesystem
/// @author rfree (maintainer)
class cFilesystemUtils { // if we do not want to use boost in given project (or we could optionally write boost here later)
public:
static bool CreateDirTree(const std::string & dir, bool only_below=false);
static char GetDirSeparator(); // eg '/' or '\'
};
/// @brief utils to e.g. edit a file from console
/// @author rfree (maintainer)
class cEnvUtils {
int fd;
string mFilename;
void GetTmpTextFile();
void CloseFile();
void OpenEditor();
const string ReadFromTmpFile();
public:
const string Compose();
const string ReadFromFile(const string path);
};
void hintingToTxt(std::fstream & file, string command, vector<string> &commands);
void generateQuestions (std::fstream & file, string command);
void generateAnswers (std::fstream & file, string command, vector<string> &completions);
// ====================================================================
namespace nOper { // nOT::nUtils::nOper
// cool shortcut operators, like vector + vecotr operator working same as string (appending)
// isolated to namespace because it's unorthodox ide to implement this
using namespace std;
// TODO use && and move?
template <class T>
vector<T> operator+(const vector<T> &a, const vector<T> &b) {
vector<T> ret = a;
ret.insert( ret.end() , b.begin(), b.end() );
return ret;
}
template <class T>
vector<T> operator+(const T &a, const vector<T> &b) {
vector<T> ret(1,a);
ret.insert( ret.end() , b.begin(), b.end() );
return ret;
}
template <class T>
vector<T> operator+(const vector<T> &a, const T &b) {
vector<T> b_vector(1,a);
return a + b_vector;
}
template <class T>
vector<T>& operator+=(vector<T> &a, const vector<T> &b) {
a.insert( a.end() , b.begin(), b.end() );
return a;
}
// map
template <class TK,class TV>
map<TK,TV> operator+(const map<TK,TV> &a, const map<TK,TV> &b) {
map<TK,TV> ret = a;
for (const auto & elem : b) {
ret.insert(elem);
}
return ret;
}
} // nOT::nUtils::nOper
// ====================================================================
// ====================================================================
// Algorithms
// ====================================================================
// ====================================================================
/**
@brief Special type that on creation will be initialized to have value INIT given as template argument.
Might be usefull e.g. to express in the declaration of class what will be the default value of member variable
See also http://www.boost.org/doc/libs/1_56_0/libs/utility/value_init.htm
Probably not needed when using boost in your project.
*/
template <class T, T INIT>
class value_init {
private:
T data;
public:
value_init();
T& operator=(const T& v) { data=v; return *this; }
operator T const &() const { return data; }
operator T&() { return data; }
};
template <class T, T INIT>
value_init<T, INIT>::value_init() : data(INIT) { }
}; // namespace nUtils
}; // namespace nOT
// global namespace
extern nOT::nUtils::cLogger gCurrentLogger; ///< The current main logger. Usually do not use it directly, instead use macros like _dbg1 etc
std::string GetObjectName(); ///< Method to return name of current object; To use in debug; Can be shadowed in your classes. (Might be not used currently)
const extern int _dbg_ignore; ///< the global _dbg_ignore, but local code (blocks, classes etc) you could shadow it in your code blocks,
// to override debug compile-time setting for given block/class, e.g. to disable debug in one of your methods or increase it there.
// Or to make it runtime by providing a class normal member and editing it in runtime
#define OT_CODE_STAMP ( nOT::nUtils::ToStr("[") + nOT::nUtils::nDetail::DbgShortenCodeFileName(__FILE__) + nOT::nUtils::ToStr("+") + nOT::nUtils::ToStr(__LINE__) + nOT::nUtils::ToStr(" ") + (GetObjectName()) + nOT::nUtils::ToStr("::") + nOT::nUtils::ToStr(__FUNCTION__) + nOT::nUtils::ToStr("]"))
#endif

2
src/CMakeLists.txt
View File

@ -92,6 +92,8 @@ add_subdirectory(cryptonote_core)
add_subdirectory(mnemonics)
add_subdirectory(rpc)
add_subdirectory(wallet)
add_subdirectory(p2p)
add_subdirectory(cryptonote_protocol)
add_subdirectory(connectivity_tool)
add_subdirectory(miner)

1
src/cryptonote_core/CMakeLists.txt
View File

@ -70,6 +70,7 @@ target_link_libraries(cryptonote_core
LINK_PUBLIC
common
crypto
otshell_utils
${Boost_DATE_TIME_LIBRARY}
${Boost_PROGRAM_OPTIONS_LIBRARY}
${Boost_SERIALIZATION_LIBRARY}

26
src/cryptonote_core/blockchain_storage.cpp
View File

@ -49,6 +49,7 @@
#include "crypto/hash.h"
#include "cryptonote_core/checkpoints_create.h"
//#include "serialization/json_archive.h"
#include "../../contrib/otshell_utils/utils.hpp"
using namespace cryptonote;
@ -1153,6 +1154,31 @@ uint64_t blockchain_storage::block_difficulty(size_t i)
return m_blocks[i].cumulative_difficulty - m_blocks[i-1].cumulative_difficulty;
}
//------------------------------------------------------------------
double blockchain_storage::get_avg_block_size( size_t count)
{
if (count > get_current_blockchain_height()) return 500;
double average = 0;
_dbg1_c("net/blksize", "HEIGHT: " << get_current_blockchain_height());
_dbg1_c("net/blksize", "BLOCK ID BY HEIGHT: " << get_block_id_by_height(get_current_blockchain_height()) );
_dbg1_c("net/blksize", "BLOCK TAIL ID: " << get_tail_id() );
std::vector<size_t> size_vector;
get_backward_blocks_sizes(get_current_blockchain_height() - count, size_vector, count);
std::vector<size_t>::iterator it;
it = size_vector.begin();
while (it != size_vector.end()) {
average += *it;
_dbg2_c("net/blksize", "VECTOR ELEMENT: " << (*it) );
it++;
}
average = average / count;
_dbg1_c("net/blksize", "VECTOR SIZE: " << size_vector.size() << " average=" << average);
return average;
}
//------------------------------------------------------------------
void blockchain_storage::print_blockchain(uint64_t start_index, uint64_t end_index)
{
std::stringstream ss;

1
src/cryptonote_core/blockchain_storage.h
View File

@ -134,6 +134,7 @@ namespace cryptonote
uint64_t get_current_comulative_blocksize_limit();
bool is_storing_blockchain(){return m_is_blockchain_storing;}
uint64_t block_difficulty(size_t i);
double get_avg_block_size( size_t count);
template<class t_ids_container, class t_blocks_container, class t_missed_container>
bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs)

46
src/cryptonote_protocol/CMakeLists.txt Normal file
View File

@ -0,0 +1,46 @@
# Copyright (c) 2014, The Monero Project
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are
# permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
# of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cmake_minimum_required (VERSION 2.6)
project (bitmonero CXX)
file(GLOB CRYPTONOTE_PROTOCOL *)
source_group(cryptonote_protocol FILES ${CRYPTONOTE_PROTOCOL})
#add_library(p2p ${P2P})
#bitmonero_private_headers(p2p ${CRYPTONOTE_PROTOCOL})
bitmonero_add_library(cryptonote_protocol ${CRYPTONOTE_PROTOCOL})
#target_link_libraries(p2p)
# LINK_PRIVATE
# ${Boost_CHRONO_LIBRARY}
# ${Boost_REGEX_LIBRARY}
# ${Boost_SYSTEM_LIBRARY}
# ${Boost_THREAD_LIBRARY}
# ${EXTRA_LIBRARIES})
add_dependencies(cryptonote_protocol
version)

266
src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp Normal file
View File

@ -0,0 +1,266 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief This is the place to implement our handlers for protocol network actions, e.g. for ratelimit for download-requests
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <boost/asio.hpp>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/interprocess/detail/atomic.hpp>
#include <boost/thread/thread.hpp>
#include <memory>
#include "syncobj.h"
#include "../../contrib/epee/include/net/net_utils_base.h"
#include "../../contrib/epee/include/misc_log_ex.h"
#include <boost/lambda/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/chrono.hpp>
#include <boost/utility/value_init.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/thread/thread.hpp>
#include "misc_language.h"
#include "pragma_comp_defs.h"
#include <sstream>
#include <iomanip>
#include <algorithm>
#include <boost/asio/basic_socket.hpp>
#include <boost/asio/ip/unicast.hpp>
#include "../../src/cryptonote_protocol/cryptonote_protocol_handler.h"
#include "../../src/p2p/network_throttle.hpp"
#include "../../contrib/otshell_utils/utils.hpp"
using namespace nOT::nUtils;
#include "../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal()
// ################################################################################################
// ################################################################################################
// the "header part". Not separeted out for .hpp because point of this modification is
// to rebuild just 1 translation unit while working on this code.
// (But maybe common parts will be separated out later though - if needed)
// ################################################################################################
// ################################################################################################
namespace cryptonote {
class cryptonote_protocol_handler_base_pimpl { // placeholer if needed
public:
};
} // namespace
// ################################################################################################
// ################################################################################################
// ################################################################################################
// ################################################################################################
namespace cryptonote {
double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa
const double size_min = 500; // XXX 500
const int history_len = 20; // how many blocks to average over
double avg=0;
try {
avg = get_avg_block_size(history_len);
} catch (...) { }
avg = std::max( size_min , avg);
return avg;
}
cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() {
}
cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() {
}
void cryptonote_protocol_handler_base::handler_request_blocks_now(size_t &count_limit) {
using namespace epee::net_utils;
size_t est_req_size=0; // how much data are we now requesting (to be soon send to us)
const auto count_limit_default = count_limit;
bool allowed_now = false; // are we now allowed to request or are we limited still
// long int size_limit;
while (!allowed_now) {
/* if ( ::cryptonote::core::get_is_stopping() ) { // TODO fast exit
_fact("ABORT sleep (before sending requeset) due to stopping");
break;
}*/
//LOG_PRINT_RED("[DBG]" << get_avg_block_size(1), LOG_LEVEL_0);
//{
long int size_limit1=0, size_limit2=0;
//LOG_PRINT_RED("calculating REQUEST size:", LOG_LEVEL_0);
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in );
network_throttle_manager::get_global_throttle_in().tick();
size_limit1 = network_throttle_manager::get_global_throttle_in().get_recommended_size_of_planned_transport();
}
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq );
network_throttle_manager::get_global_throttle_inreq().tick();
size_limit2 = network_throttle_manager::get_global_throttle_inreq().get_recommended_size_of_planned_transport();
}
long int one_block_estimated_size = estimate_one_block_size();
long int limit_small = std::min( size_limit1 , size_limit2 );
long int size_limit = limit_small/3 + size_limit1/3 + size_limit2/3;
if (limit_small <= 0) size_limit = 0;
const double estimated_peers = 1.2; // how many peers/threads we want to talk to, in order to not grab entire b/w by 1 thread
const double knob = 1.000;
size_limit /= (estimated_peers / estimated_peers) * knob;
_note_c("net/req-calc" , "calculating REQUEST size:" << size_limit1 << " " << size_limit2 << " small=" << limit_small << " final size_limit="<<size_limit);
double L = size_limit / one_block_estimated_size; // calculating item limit (some heuristics)
//LOG_PRINT_RED("L1 = " << L , LOG_LEVEL_0);
//double L2=0; if (L>1) L2=std::log(L);
//L = L/10. + L2*5;
//LOG_PRINT_RED("L2 = " << L , LOG_LEVEL_0);
L = std::min( (double)count_limit_default, (double)L);
//LOG_PRINT_RED("L3 = " << L , LOG_LEVEL_0);
const long int hard_limit = 500; // never get more blocks at once ; TODO depend on speed limit. Must be low or limiting is too bursty.
L = std::min(L, (double) hard_limit);
count_limit = (int)L;
est_req_size = count_limit * one_block_estimated_size ; // how much data did we just requested?
//LOG_PRINT_RED("est_req_size = " << est_req_size , LOG_LEVEL_0);
//LOG_PRINT_RED("count_limit = " << count_limit , LOG_LEVEL_0);
//LOG_PRINT_RED("one_block_estimated_size = " << one_block_estimated_size , LOG_LEVEL_0);
//}
if (count_limit > 0) allowed_now = true;
// XXX if (!allowed_now) { // XXX DOWNLOAD
//long int ms = 3000; // XXX 2000
//LOG_PRINT_RED("size_limit = " << size_limit , LOG_LEVEL_0);
long int ms = network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick(one_block_estimated_size); // XXX too long
//long int ms = network_throttle_manager::get_global_throttle_in().get_sleep_time(count_limit); // XXX
//long int ms = network_throttle_manager::get_global_throttle_in().get_sleep_time(size_limit); // XXX best
//ms /= 100; // XXX
_info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms"); // XXX debug sleep
//LOG_PRINT_RED("ms = " << ms , LOG_LEVEL_0);
boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); // TODO randomize sleeps
//}
}
// done waiting&sleeping ^
// ok we are allowed to send now
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq );
network_throttle_manager::get_global_throttle_inreq().handle_trafic_tcp( est_req_size ); // increase countere of the global requested input
}
// TODO remove debug
LOG_PRINT_YELLOW("*************************************************************************", LOG_LEVEL_0);
LOG_PRINT_RED("### RRRR ### sending request (type 1), CALCULATED limit = " << count_limit << " = estimated " << est_req_size << " b", LOG_LEVEL_0);
LOG_PRINT_YELLOW("*************************************************************************", LOG_LEVEL_0);
LOG_PRINT_RED("\n", LOG_LEVEL_0);
_note_c("net/req", "### RRRR ### sending request (type 1), CALCULATED limit = " << count_limit << " = estimated " << est_req_size << " b");
}
void cryptonote_protocol_handler_base::handler_request_blocks_history(std::list<crypto::hash>& ids) {
using namespace epee::net_utils;
LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size());
LOG_PRINT_RED("RATE LIMIT NOT IMPLEMENTED HERE YET (download at unlimited speed?)" , LOG_LEVEL_0);
_note_c("net/req2", "### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size());
// TODO
}
void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_mark("");
using namespace epee::net_utils;
double delay=0; // will be calculated
_dbg1("Packet size: " << packet_size);
do
{ // rate limiting
//XXX
/*if (::cryptonote::core::get_is_stopping()) {
_dbg1("We are stopping - so abort sleep");
return;
}*/
/*if (m_was_shutdown) {
_dbg2_c("net/netuse/sleep","m_was_shutdown - so abort sleep");
return;
}*/
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global
}
delay *= 0.50;
//delay = 0; // XXX
if (delay > 0) {
//delay += rand2*0.1;
long int ms = (long int)(delay * 1000);
_info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<packet_size); // XXX debug sleep
_dbg1_c("net/sleep/", "sleep in sleep_before_packet");
_dbg2("Sleep for " << ms);
boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); // TODO randomize sleeps
}
} while(delay > 0);
// XXX LATER XXX
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
network_throttle_manager::get_global_throttle_out().handle_trafic_tcp( packet_size ); // increase counter - global
//epee::critical_region_t<decltype(m_throttle_global_lock)> guard(m_throttle_global_lock); // *** critical ***
//m_throttle_global.m_out.handle_trafic_tcp( packet_size ); // increase counter - global
}
}
} // namespace

35
src/cryptonote_protocol/cryptonote_protocol_handler.h
View File

@ -1,3 +1,7 @@
/// @file
/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote)
/// @brief This is the orginal cryptonote protocol network-events handler, modified by us
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
@ -41,6 +45,7 @@
#include "cryptonote_core/connection_context.h"
#include "cryptonote_core/cryptonote_stat_info.h"
#include "cryptonote_core/verification_context.h"
#include <netinet/in.h>
PUSH_WARNINGS
DISABLE_VS_WARNINGS(4355)
@ -48,8 +53,26 @@ DISABLE_VS_WARNINGS(4355)
namespace cryptonote
{
class cryptonote_protocol_handler_base_pimpl;
class cryptonote_protocol_handler_base {
private:
std::unique_ptr<cryptonote_protocol_handler_base_pimpl> mI;
public:
cryptonote_protocol_handler_base();
virtual ~cryptonote_protocol_handler_base();
void handler_request_blocks_now(size_t & count_limit); // before asking for blocks, can adjust the limit of download
void handler_request_blocks_history(std::list<crypto::hash>& ids); // before asking for list of objects, we can change the list still
void handler_response_blocks_now(size_t packet_size);
virtual double get_avg_block_size( size_t count) const = 0;
virtual double estimate_one_block_size() noexcept; // for estimating size of blocks to download
virtual std::ofstream& get_logreq() const =0;
};
template<class t_core>
class t_cryptonote_protocol_handler: public i_cryptonote_protocol
class t_cryptonote_protocol_handler: public i_cryptonote_protocol, cryptonote_protocol_handler_base
{
public:
typedef cryptonote_connection_context connection_context;
@ -107,12 +130,17 @@ namespace cryptonote
std::atomic<uint32_t> m_syncronized_connections_count;
std::atomic<bool> m_synchronized;
// static std::ofstream m_logreq;
double get_avg_block_size(size_t count) const;
template<class t_parametr>
bool post_notify(typename t_parametr::request& arg, cryptonote_connection_context& context)
{
LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parametr).name() << " -->");
std::string blob;
epee::serialization::store_t_to_binary(arg, blob);
//handler_response_blocks_now(blob.size()); // XXX
return m_p2p->invoke_notify_to_peer(t_parametr::ID, blob, context);
}
@ -124,8 +152,11 @@ namespace cryptonote
epee::serialization::store_t_to_binary(arg, arg_buff);
return m_p2p->relay_notify_to_all(t_parametr::ID, arg_buff, exlude_context);
}
virtual std::ofstream& get_logreq() const ;
};
}
} // namespace
#include "cryptonote_protocol_handler.inl"

73
src/cryptonote_protocol/cryptonote_protocol_handler.inl
View File

@ -1,3 +1,7 @@
/// @file
/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote)
/// @brief This is the orginal cryptonote protocol network-events handler, modified by us
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
@ -28,14 +32,26 @@
//
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
// (may contain code and/or modifications by other developers)
// developer rfree: this code is caller of our new network code, and is modded; e.g. for rate limiting
#include <boost/interprocess/detail/atomic.hpp>
#include <list>
#include "cryptonote_core/cryptonote_format_utils.h"
#include "profile_tools.h"
#include "../../contrib/otshell_utils/utils.hpp"
using namespace nOT::nUtils;
namespace cryptonote
{
// static
// template<class t_core> std::ofstream t_cryptonote_protocol_handler<t_core>::m_logreq("logreq.txt"); // static
//-----------------------------------------------------------------------------------------------------------------------
template<class t_core>
t_cryptonote_protocol_handler<t_core>::t_cryptonote_protocol_handler(t_core& rcore, nodetool::i_p2p_endpoint<connection_context>* p_net_layout):m_core(rcore),
@ -100,20 +116,24 @@ namespace cryptonote
{
std::stringstream ss;
ss << std::setw(25) << std::left << "Remote Host"
ss << std::setw(30) << std::left << "Remote Host"
<< std::setw(20) << "Peer id"
<< std::setw(25) << "Recv/Sent (inactive,sec)"
<< std::setw(25) << "State"
<< std::setw(20) << "Livetime(seconds)" << ENDL;
uint32_t ip;
m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id)
{
ss << std::setw(25) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") +
epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port)
ip = ntohl(cntxt.m_remote_ip);
ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") +
epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port)
<< std::setw(20) << std::hex << peer_id
<< std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")"
<< std::setw(25) << get_protocol_state_string(cntxt.m_state)
<< std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) << ENDL;
<< std::setw(20) << std::to_string(time(NULL) - cntxt.m_started)
<< std::setw(10) << (ip > 3232235520 && ip < 3232301055 ? " [LAN]" : "") //TODO: local ip in calss A, B
<< ENDL;
return true;
});
LOG_PRINT_L0("Connections: " << ENDL << ss.str());
@ -234,11 +254,11 @@ namespace cryptonote
block_verification_context bvc = boost::value_initialized<block_verification_context>();
m_core.pause_mine();
m_core.handle_incoming_block(arg.b.block, bvc);
m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block
m_core.resume_mine();
if(bvc.m_verifivation_failed)
{
LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection");
LOG_PRINT_CCONTEXT_L0("Block verification failed, dropping connection");
m_p2p->drop_connection(context);
return 1;
}
@ -304,9 +324,19 @@ namespace cryptonote
LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size()
<< ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height << ", missed_ids.size()=" << rsp.missed_ids.size());
post_notify<NOTIFY_RESPONSE_GET_OBJECTS>(rsp, context);
//handler_response_blocks_now(sizeof(rsp)); // XXX
//handler_response_blocks_now(200);
return 1;
}
//------------------------------------------------------------------------------------------------------------------------
template<class t_core>
double t_cryptonote_protocol_handler<t_core>::get_avg_block_size( size_t count) const {
return m_core.get_blockchain_storage().get_avg_block_size(count);
}
template<class t_core>
int t_cryptonote_protocol_handler<t_core>::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context)
{
@ -378,6 +408,7 @@ namespace cryptonote
epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler(
boost::bind(&t_core::resume_mine, &m_core));
LOG_PRINT_CCONTEXT_YELLOW( "Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() , LOG_LEVEL_0);
BOOST_FOREACH(const block_complete_entry& block_entry, arg.blocks)
{
//process transactions
@ -419,7 +450,8 @@ namespace cryptonote
LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms");
}
}
size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
handler_request_blocks_now(count_limit); // XXX
request_missing_objects(context, true);
return 1;
}
@ -455,6 +487,11 @@ namespace cryptonote
size_t count = 0;
auto it = context.m_needed_objects.begin();
size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
//handler_request_blocks_now( count_limit ); // change the limit, sleep(?) XXX
// XXX
count_limit=200; // XXX
_note_c("net/req-calc" , "Setting count_limit: " << count_limit);
while(it != context.m_needed_objects.end() && count < BLOCKS_SYNCHRONIZING_DEFAULT_COUNT)
{
if( !(check_having_blocks && m_core.have_block(*it)))
@ -465,14 +502,16 @@ namespace cryptonote
}
context.m_needed_objects.erase(it++);
}
LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size());
LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size()
<< "requested blocks count=" << count << " / " << count_limit);
post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context);
}else if(context.m_last_response_height < context.m_remote_blockchain_height-1)
{//we have to fetch more objects ids, request blockchain entry
NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>();
m_core.get_short_chain_history(r.block_ids);
LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
}else
{
@ -575,4 +614,18 @@ namespace cryptonote
{
return relay_post_notify<NOTIFY_NEW_TRANSACTIONS>(arg, exclude_context);
}
}
/// @deprecated
template<class t_core> std::ofstream& t_cryptonote_protocol_handler<t_core>::get_logreq() const {
static std::ofstream * logreq=NULL;
if (!logreq) {
LOG_PRINT_RED("LOG OPENED",LOG_LEVEL_0);
logreq = new std::ofstream("logreq.txt"); // leak mem (singleton)
*logreq << "Opened log" << std::endl;
}
LOG_PRINT_YELLOW("LOG USED",LOG_LEVEL_0);
(*logreq) << "log used" << std::endl;
return *logreq;
}
} // namespace

21
src/daemon/CMakeLists.txt
View File

@ -32,23 +32,7 @@ set(daemon_sources
set(daemon_headers)
set(daemon_private_headers
daemon_commands_handler.h
# cryptonote_protocol
../cryptonote_protocol/blobdatatype.h
../cryptonote_protocol/cryptonote_protocol_defs.h
../cryptonote_protocol/cryptonote_protocol_handler.h
../cryptonote_protocol/cryptonote_protocol_handler.inl
../cryptonote_protocol/cryptonote_protocol_handler_common.h
# p2p
../p2p/net_node.h
../p2p/net_node.inl
../p2p/net_node_common.h
../p2p/net_peerlist.h
../p2p/net_peerlist_boost_serialization.h
../p2p/p2p_protocol_defs.h
../p2p/stdafx.h)
daemon_commands_handler.h)
bitmonero_private_headers(daemon
${daemon_private_headers})
@ -62,6 +46,9 @@ target_link_libraries(daemon
cryptonote_core
crypto
common
otshell_utils
p2p
cryptonote_protocol
${Boost_CHRONO_LIBRARY}
${Boost_FILESYSTEM_LIBRARY}
${Boost_PROGRAM_OPTIONS_LIBRARY}

129
src/daemon/daemon_commands_handler.h
View File

@ -40,6 +40,7 @@
#include "common/util.h"
#include "crypto/hash.h"
#include "version.h"
#include "../../contrib/otshell_utils/utils.hpp"
/*!
* \brief I don't really know right now
@ -74,6 +75,10 @@ public:
m_cmd_binder.set_handler("save", boost::bind(&daemon_cmmands_handler::save, this, _1), "Save blockchain");
m_cmd_binder.set_handler("set_log", boost::bind(&daemon_cmmands_handler::set_log, this, _1), "set_log <level> - Change current log detalization level, <level> is a number 0-4");
m_cmd_binder.set_handler("diff", boost::bind(&daemon_cmmands_handler::diff, this, _1), "Show difficulty");
m_cmd_binder.set_handler("out_peers", boost::bind(&daemon_cmmands_handler::out_peers_limit, this, _1), "Set max limit of out peers");
m_cmd_binder.set_handler("limit_up", boost::bind(&daemon_cmmands_handler::limit_up, this, _1), "Set upload limit [kB/s]");
m_cmd_binder.set_handler("limit_down", boost::bind(&daemon_cmmands_handler::limit_down, this, _1), "Set download limit [kB/s]");
m_cmd_binder.set_handler("limit", boost::bind(&daemon_cmmands_handler::limit, this, _1), "Set download and upload limit [kB/s]");
}
bool start_handling()
@ -398,4 +403,128 @@ private:
m_srv.get_payload_object().get_core().get_miner().stop();
return true;
}
//--------------------------------------------------------------------------------
bool out_peers_limit(const std::vector<std::string>& args) {
if(args.size()!=1) {
std::cout << "Usage: limit_down <speed>" << ENDL;
return true;
}
unsigned int limit;
try {
limit = std::stoi(args[0]);
}
catch(std::invalid_argument& ex) {
_erro("stoi exception");
return false;
}
if (m_srv.m_config.m_net_config.connections_count > limit)
{
int count = m_srv.m_config.m_net_config.connections_count - limit;
m_srv.m_config.m_net_config.connections_count = limit;
m_srv.delete_connections(count);
}
else
m_srv.m_config.m_net_config.connections_count = limit;
return true;
}
//--------------------------------------------------------------------------------
bool limit_up(const std::vector<std::string>& args)
{
if(args.size()!=1) {
std::cout << "Usage: limit_up <speed>" << ENDL;
return false;
}
int limit;
try {
limit = std::stoi(args[0]);
}
catch(std::invalid_argument& ex) {
return false;
}
if (limit==-1) {
limit=128;
//this->islimitup=false;
}
limit *= 1024;
//nodetool::epee::net_utils::connection<epee::levin::async_protocol_handler<nodetool::p2p_connection_context> >::set_rate_up_limit( limit );
epee::net_utils::connection_basic::set_rate_up_limit( limit );
std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl;
return true;
}
//--------------------------------------------------------------------------------
bool limit_down(const std::vector<std::string>& args)
{
if(args.size()!=1) {
std::cout << "Usage: limit_down <speed>" << ENDL;
return true;
}
int limit;
try {
limit = std::stoi(args[0]);
}
catch(std::invalid_argument& ex) {
return false;
}
if (limit==-1) {
limit=128;
//this->islimitup=false;
}
limit *= 1024;
//nodetool::epee::net_utils::connection<epee::levin::async_protocol_handler<nodetool::p2p_connection_context> >::set_rate_up_limit( limit );
epee::net_utils::connection_basic::set_rate_down_limit( limit );
std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl;
return true;
}
//--------------------------------------------------------------------------------
bool limit(const std::vector<std::string>& args)
{
if(args.size()!=1) {
std::cout << "Usage: limit_down <speed>" << ENDL;
return true;
}
int limit;
try {
limit = std::stoi(args[0]);
}
catch(std::invalid_argument& ex) {
return false;
}
if (limit==-1) {
limit=128;
//this->islimitup=false;
}
limit *= 1024;
//nodetool::epee::net_utils::connection<epee::levin::async_protocol_handler<nodetool::p2p_connection_context> >::set_rate_up_limit( limit );
epee::net_utils::connection_basic::set_rate_down_limit( limit );
epee::net_utils::connection_basic::set_rate_up_limit( limit );
std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl;
std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl;
return true;
}
};

46
src/p2p/CMakeLists.txt Normal file
View File

@ -0,0 +1,46 @@
# Copyright (c) 2014, The Monero Project
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are
# permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
# of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cmake_minimum_required (VERSION 2.6)
project (bitmonero CXX)
file(GLOB P2P *)
source_group(p2p FILES ${P2P})
#add_library(p2p ${P2P})
#bitmonero_private_headers(p2p ${P2P})
bitmonero_add_library(p2p ${P2P})
#target_link_libraries(p2p)
# LINK_PRIVATE
# ${Boost_CHRONO_LIBRARY}
# ${Boost_REGEX_LIBRARY}
# ${Boost_SYSTEM_LIBRARY}
# ${Boost_THREAD_LIBRARY}
# ${EXTRA_LIBRARIES})
add_dependencies(p2p
version)

362
src/p2p/connection_basic.cpp Normal file
View File

@ -0,0 +1,362 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief base for connection, contains e.g. the ratelimit hooks
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/* rfree: implementation for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */
#include "connection_basic.hpp"
#include <boost/asio.hpp>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/interprocess/detail/atomic.hpp>
#include <boost/thread/thread.hpp>
#include <memory>
#include "syncobj.h"
#include "../../contrib/epee/include/net/net_utils_base.h"
#include "../../contrib/epee/include/misc_log_ex.h"
#include <boost/lambda/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/chrono.hpp>
#include <boost/utility/value_init.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/thread/thread.hpp>
#include <boost/filesystem.hpp>
#include "misc_language.h"
#include "pragma_comp_defs.h"
#include <fstream>
#include <sstream>
#include <iomanip>
#include <algorithm>
#include <mutex>
#include <boost/asio/basic_socket.hpp>
#include <boost/asio/ip/unicast.hpp>
#include "../../contrib/epee/include/net/abstract_tcp_server2.h"
#include "../../contrib/otshell_utils/utils.hpp"
using namespace nOT::nUtils;
// TODO:
#include "../../src/p2p/network_throttle-detail.hpp"
#include "../../src/cryptonote_core/cryptonote_core.h"
// ################################################################################################
// local (TU local) headers
// ################################################################################################
namespace epee
{
namespace net_utils
{
/* ============================================================================ */
class connection_basic_pimpl {
public:
connection_basic_pimpl(const std::string &name);
static int m_default_tos;
network_throttle_bw m_throttle; // per-perr
critical_section m_throttle_lock;
int m_peer_number; // e.g. for debug/stats
};
} // namespace
} // namespace
// ################################################################################################
// The implementation part
// ################################################################################################
namespace epee
{
namespace net_utils
{
// ================================================================================================
// connection_basic_pimpl
// ================================================================================================
connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_throttle(name) { }
// ================================================================================================
// connection_basic
// ================================================================================================
// static variables:
int connection_basic_pimpl::m_default_tos;
// methods:
connection_basic::connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number)
:
mI( new connection_basic_pimpl("peer") ),
strand_(io_service),
socket_(io_service),
m_want_close_connection(false),
m_was_shutdown(false),
m_ref_sock_count(ref_sock_count)
{
++ref_sock_count; // increase the global counter
mI->m_peer_number = sock_number.fetch_add(1); // use, and increase the generated number
_note("Spawned connection p2p#"<<mI->m_peer_number<<" currently we have sockets count:" << m_ref_sock_count);
boost::filesystem::create_directories("log/dr-monero/net/");
/*boost::asio::SettableSocketOption option;// = new boost::asio::SettableSocketOption();
option.level(IPPROTO_IP);
option.name(IP_TOS);
option.value(&tos);
option.size = sizeof(tos);
socket_.set_option(option);*/
// TODO socket options
}
connection_basic::~connection_basic() {
_note("Destructing connection p2p#"<<mI->m_peer_number);
}
void connection_basic::set_rate_up_limit(uint64_t limit) {
save_limit_to_file(limit);
{
// TODO remove __SCALING_FACTOR...
const double SCALING_FACTOR = 2.25; // to acheve the best performance
limit *= SCALING_FACTOR;
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
network_throttle_manager::get_global_throttle_out().set_target_speed(limit);
}
// connection_basic_pimpl::m_throttle_global.m_out.set_target_speed(limit);
}
void connection_basic::set_rate_down_limit(uint64_t limit) {
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in );
network_throttle_manager::get_global_throttle_in().set_target_speed(limit);
}
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq );
network_throttle_manager::get_global_throttle_inreq().set_target_speed(limit);
}
save_limit_to_file(limit);
}
void connection_basic::set_rate_limit(uint64_t limit) {
// TODO
}
void connection_basic::set_kill_limit (uint64_t limit) {
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in );
network_throttle_manager::get_global_throttle_in().set_target_kill(limit);
}
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
network_throttle_manager::get_global_throttle_out().set_target_kill(limit);
}
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq );
network_throttle_manager::get_global_throttle_inreq().set_target_kill(limit);
}
}
void connection_basic::save_limit_to_file(int limit) {
// saving limit to file
std::ofstream file;
file.open("log/dr-monero/limit.info");
file << limit;
}
void connection_basic::set_rate_autodetect(uint64_t limit) {
// TODO
LOG_PRINT_L0("inside connection_basic we set autodetect (this is additional notification)..");
}
void connection_basic::set_tos_flag(int tos) {
connection_basic_pimpl::m_default_tos = tos;
}
int connection_basic::get_tos_flag() {
return connection_basic_pimpl::m_default_tos;
}
void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q_len) {
double delay=0; // will be calculated
do
{ // rate limiting
//XXX
/*if (::cryptonote::core::get_is_stopping()) {
_dbg1("We are stopping - so abort sleep");
return;
}*/
if (m_was_shutdown) {
_dbg2("m_was_shutdown - so abort sleep");
return;
}
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global
}
delay *= 0.50;
delay = 0; // XXX
if (delay > 0) {
//delay += rand2*0.1;
long int ms = (long int)(delay * 1000);
_info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<packet_size); // XXX debug sleep
_dbg1("sleep in sleep_before_packet");
boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); // TODO randomize sleeps
}
} while(delay > 0);
// XXX LATER XXX
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
network_throttle_manager::get_global_throttle_out().handle_trafic_tcp( packet_size ); // increase counter - global
//epee::critical_region_t<decltype(m_throttle_global_lock)> guard(m_throttle_global_lock); // *** critical ***
//m_throttle_global.m_out.handle_trafic_tcp( packet_size ); // increase counter - global
}
}
void connection_basic::set_start_time() {
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
m_start_time = network_throttle_manager::get_global_throttle_out().get_time_seconds();
}
void connection_basic::do_send_handler_start(const void* ptr , size_t cb ) {
_fact_c("net/out/size", "*** do_sen() called for packet="<<cb<<" B");
sleep_before_packet(cb,1,-1);
// set_start_time();
}
void connection_basic::do_send_handler_delayed(const void* ptr , size_t cb ) {
// CRITICAL_REGION_LOCAL(network_throttle_manager::m_lock_get_global_throttle_out);
// auto sending_time = network_throttle_manager::get_global_throttle_out().get_time_seconds() - m_start_time; // wrong? --r
}
void connection_basic::do_send_handler_write(const void* ptr , size_t cb ) {
sleep_before_packet(cb,1,-1);
_info_c("net/out/size", "handler_write (direct) - before ASIO write, for packet="<<cb<<" B (after sleep)");
set_start_time();
}
void connection_basic::do_send_handler_stop(const void* ptr , size_t cb ) {
}
void connection_basic::do_send_handler_after_write(const boost::system::error_code& e, size_t cb) {
// CRITICAL_REGION_LOCAL(network_throttle_manager::m_lock_get_global_throttle_out);
// auto sending_time = network_throttle_manager::get_global_throttle_out().get_time_seconds() - m_start_time;
// lag: if current sending time > max sending time
//if (sending_time > 0.1) network_throttle_manager::get_global_throttle_out().set_overheat(sending_time); // TODO
}
void connection_basic::do_send_handler_write_from_queue( const boost::system::error_code& e, size_t cb, int q_len ) {
sleep_before_packet(cb,2,q_len);
_info_c("net/out/size", "handler_write (after write, from queue="<<q_len<<") - before ASIO write, for packet="<<cb<<" B (after sleep)");
set_start_time();
}
void connection_basic::do_read_handler_start(const boost::system::error_code& e, std::size_t bytes_transferred) { // from read, after read completion
const size_t packet_size = bytes_transferred;
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in );
// sleep_before_packet(packet_size * __SCALING_FACTOR, 1, -1); // TODO remove __SCALING_FACTOR
network_throttle_manager::get_global_throttle_in().handle_trafic_tcp( packet_size ); // increase counter - global
// epee::critical_region_t<decltype(mI->m_throttle_global_lock)> guard(mI->m_throttle_global_lock); // *** critical ***
// mI->m_throttle_global.m_in.handle_trafic_tcp( packet_size ); // increase counter - global
}
}
void connection_basic::logger_handle_net_peer(size_t size, bool io) { // network data written
// TODO OPTIMIZE! do NOT reopen idiotically :)
std::ostringstream oss;
std::string filename;
if (io) { // write
double time = network_throttle_manager::get_global_throttle_in().get_time_seconds() ;
oss << "log/dr-monero/net/in-peer-" << (mI->m_peer_number) << ".dat" << std::ends;
filename = oss.str();
network_throttle_manager::get_global_throttle_out().logger_handle_net(filename,time,size);
}
else { // read
double time = network_throttle_manager::get_global_throttle_out().get_time_seconds() ;
oss << "log/dr-monero/net/out-peer-" << (mI->m_peer_number) << ".dat" << std::ends;
filename = oss.str();
network_throttle_manager::get_global_throttle_in().logger_handle_net(filename,time,size);
}
}
void connection_basic::logger_handle_net_read(size_t size) { // network data read
std::string filename = "log/dr-monero/net/in-all.data";
double time = network_throttle_manager::get_global_throttle_in().get_time_seconds() ;
network_throttle_manager::get_global_throttle_in().logger_handle_net(filename, time, size);
logger_handle_net_peer(size,0);
}
void connection_basic::logger_handle_net_write(size_t size) {
std::string filename = "log/dr-monero/net/out-all.data";
double time = network_throttle_manager::get_global_throttle_out().get_time_seconds() ;
network_throttle_manager::get_global_throttle_out().logger_handle_net(filename, time, size);
logger_handle_net_peer(size,1);
}
double connection_basic::get_sleep_time(size_t cb) {
auto t = network_throttle_manager::get_global_throttle_out().get_sleep_time(cb);
return t;
}
} // namespace
} // namespace

139
src/p2p/connection_basic.hpp Normal file
View File

@ -0,0 +1,139 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief base for connection, contains e.g. the ratelimit hooks
// ! This file might contain variable names same as in template class connection<>
// ! from files contrib/epee/include/net/abstract_tcp_server2.*
// ! I am not a lawyer; afaik APIs, var names etc are not copyrightable ;)
// ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part)
// ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as:
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
/* rfree: place for hanlers for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */
#ifndef INCLUDED_p2p_connection_basic_hpp
#define INCLUDED_p2p_connection_basic_hpp
#include <boost/asio.hpp>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/interprocess/detail/atomic.hpp>
#include <boost/thread/thread.hpp>
#include <memory>
#include "../../contrib/epee/include/net/net_utils_base.h"
#include "../../contrib/epee/include/syncobj.h"
namespace epee
{
namespace net_utils
{
/************************************************************************/
/* */
/************************************************************************/
/// Represents a single connection from a client.
class connection_basic_pimpl; // PIMPL for this class
class connection_basic { // not-templated base class for rapid developmet of some code parts
public:
std::unique_ptr< connection_basic_pimpl > mI; // my Implementation
// moved here from orginal connecton<> - common member variables that do not depend on template in connection<>
volatile uint32_t m_want_close_connection;
std::atomic<bool> m_was_shutdown;
critical_section m_send_que_lock;
std::list<std::string> m_send_que;
volatile bool m_is_multithreaded;
double m_start_time;
/// Strand to ensure the connection's handlers are not called concurrently.
boost::asio::io_service::strand strand_;
/// Socket for the connection.
boost::asio::ip::tcp::socket socket_;
std::atomic<long> &m_ref_sock_count; // reference to external counter of existing sockets that we will ++/--
public:
// first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator
connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number);
virtual ~connection_basic();
// various handlers to be called from connection class:
void do_send_handler_start(const void * ptr , size_t cb);
void do_send_handler_delayed(const void * ptr , size_t cb);
void do_send_handler_write(const void * ptr , size_t cb);
void do_send_handler_stop(const void * ptr , size_t cb);
void do_send_handler_after_write( const boost::system::error_code& e, size_t cb ); // from handle_write
void do_send_handler_write_from_queue(const boost::system::error_code& e, size_t cb , int q_len); // from handle_write, sending next part
void do_read_handler_start(const boost::system::error_code& e, std::size_t bytes_transferred); // from read, after read completion
void logger_handle_net_write(size_t size); // network data written
void logger_handle_net_read(size_t size); // network data read
void logger_handle_net_peer(size_t size, bool io);
void set_start_time();
// config for rate limit
static void set_rate_up_limit(uint64_t limit);
static void set_rate_down_limit(uint64_t limit);
static void set_rate_limit(uint64_t limit);
static void set_rate_autodetect(uint64_t limit);
static void set_kill_limit (uint64_t limit);
// config misc
static void set_tos_flag(int tos); // ToS / QoS flag
static int get_tos_flag();
// handlers and sleep
void sleep_before_packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?)
static void save_limit_to_file(int limit); ///< for dr-monero
static double get_sleep_time(size_t cb);
};
} // nameserver
} // nameserver
#endif

31
src/p2p/net_node.h
View File

@ -80,14 +80,12 @@ namespace nodetool
public:
typedef t_payload_net_handler payload_net_handler;
node_server(
t_payload_net_handler& payload_handler
, boost::uuids::uuid network_id
)
: m_payload_handler(payload_handler)
, m_allow_local_ip(false)
, m_hide_my_port(false)
, m_network_id(std::move(network_id))
node_server(t_payload_net_handler& payload_handler, boost::uuids::uuid network_id)
:m_payload_handler(payload_handler),
m_allow_local_ip(false),
m_no_igd(false),
m_hide_my_port(false),
m_network_id(std::move(network_id))
{}
static void init_options(boost::program_options::options_description& desc);
@ -111,6 +109,7 @@ namespace nodetool
virtual uint64_t get_connections_count();
size_t get_outgoing_connections_count();
peerlist_manager& get_peerlist_manager(){return m_peerlist;}
void delete_connections(size_t count);
private:
const std::vector<std::string> m_seed_nodes_list =
{ "seeds.moneroseeds.se"
@ -118,6 +117,9 @@ namespace nodetool
, "seeds.moneroseeds.ch"
, "seeds.moneroseeds.li"
};
bool islimitup=false;
bool islimitdown=false;
typedef COMMAND_REQUEST_STAT_INFO_T<typename t_payload_net_handler::stat_info> COMMAND_REQUEST_STAT_INFO;
@ -197,6 +199,13 @@ namespace nodetool
template <class Container>
bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor<std::vector<std::string> > & arg, Container& container);
bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max);
bool set_tos_flag(const boost::program_options::variables_map& vm, int limit);
bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit);
bool set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit);
bool set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit);
//debug functions
std::string print_connections_container();
@ -214,7 +223,10 @@ namespace nodetool
END_KV_SERIALIZE_MAP()
};
config m_config;
public:
config m_config; // TODO was private, add getters?
private:
std::string m_config_folder;
bool m_have_address;
@ -224,6 +236,7 @@ namespace nodetool
uint32_t m_ip_address;
bool m_allow_local_ip;
bool m_hide_my_port;
bool m_no_igd;
//critical_section m_connections_lock;
//connections_indexed_container m_connections;

183
src/p2p/net_node.inl
View File

@ -84,6 +84,14 @@ namespace nodetool
" If this option is given the options add-priority-node and seed-node are ignored"};
const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_seed_node = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"};
const command_line::arg_descriptor<bool> arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true};
const command_line::arg_descriptor<bool> arg_no_igd = {"no-igd", "Disable UPnP port mapping"};
const command_line::arg_descriptor<int64_t> arg_out_peers = {"out-peers", "set max limit of out peers", -1};
const command_line::arg_descriptor<int> arg_tos_flag = {"tos-flag", "set TOS flag", -1};
const command_line::arg_descriptor<int64_t> arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", -1};
const command_line::arg_descriptor<int64_t> arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", -1};
const command_line::arg_descriptor<uint64_t> arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", 128};
}
//-----------------------------------------------------------------------------------
@ -99,7 +107,13 @@ namespace nodetool
command_line::add_arg(desc, arg_p2p_add_priority_node);
command_line::add_arg(desc, arg_p2p_add_exclusive_node);
command_line::add_arg(desc, arg_p2p_seed_node);
command_line::add_arg(desc, arg_p2p_hide_my_port); }
command_line::add_arg(desc, arg_p2p_hide_my_port);
command_line::add_arg(desc, arg_no_igd);
command_line::add_arg(desc, arg_out_peers);
command_line::add_arg(desc, arg_tos_flag);
command_line::add_arg(desc, arg_limit_rate_up);
command_line::add_arg(desc, arg_limit_rate_down);
command_line::add_arg(desc, arg_limit_rate); }
//-----------------------------------------------------------------------------------
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::init_config()
@ -120,7 +134,6 @@ namespace nodetool
//at this moment we have hardcoded config
m_config.m_net_config.handshake_interval = P2P_DEFAULT_HANDSHAKE_INTERVAL;
m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT;
m_config.m_net_config.packet_max_size = P2P_DEFAULT_PACKET_MAX_SIZE; //20 MB limit
m_config.m_net_config.config_id = 0; // initial config
m_config.m_net_config.connection_timeout = P2P_DEFAULT_CONNECTION_TIMEOUT;
@ -165,6 +178,7 @@ namespace nodetool
m_port = command_line::get_arg(vm, p2p_bind_arg);
m_external_port = command_line::get_arg(vm, arg_p2p_external_port);
m_allow_local_ip = command_line::get_arg(vm, arg_p2p_allow_local_ip);
m_no_igd = command_line::get_arg(vm, arg_no_igd);
if (command_line::has_arg(vm, arg_p2p_add_peer))
{
@ -184,11 +198,13 @@ namespace nodetool
if (!parse_peers_and_add_to_container(vm, arg_p2p_add_exclusive_node, m_exclusive_peers))
return false;
}
if (command_line::has_arg(vm, arg_p2p_add_priority_node))
{
if (!parse_peers_and_add_to_container(vm, arg_p2p_add_priority_node, m_priority_peers))
return false;
}
if (command_line::has_arg(vm, arg_p2p_seed_node))
{
if (!parse_peers_and_add_to_container(vm, arg_p2p_seed_node, m_seed_nodes))
@ -197,6 +213,21 @@ namespace nodetool
if(command_line::has_arg(vm, arg_p2p_hide_my_port))
m_hide_my_port = true;
if ( !set_max_out_peers(vm, command_line::get_arg(vm, arg_out_peers) ) )
return false;
if ( !set_tos_flag(vm, command_line::get_arg(vm, arg_tos_flag) ) )
return false;
if ( !set_rate_up_limit(vm, command_line::get_arg(vm, arg_limit_rate_up) ) )
return false;
if ( !set_rate_down_limit(vm, command_line::get_arg(vm, arg_limit_rate_down) ) )
return false;
if ( !set_rate_limit(vm, command_line::get_arg(vm, arg_limit_rate) ) )
return false;
return true;
}
@ -375,42 +406,43 @@ namespace nodetool
LOG_PRINT_L0("External port defined as " << m_external_port);
// Add UPnP port mapping
LOG_PRINT_L0("Attempting to add IGD port mapping.");
int result;
UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result);
UPNPUrls urls;
IGDdatas igdData;
char lanAddress[64];
result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress);
freeUPNPDevlist(deviceList);
if (result != 0) {
if (result == 1) {
std::ostringstream portString;
portString << m_listenning_port;
// Delete the port mapping before we create it, just in case we have dangling port mapping from the daemon not being shut down correctly
UPNP_DeletePortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), "TCP", 0);
if(m_no_igd == false) {
LOG_PRINT_L0("Attempting to add IGD port mapping.");
int result;
UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result);
UPNPUrls urls;
IGDdatas igdData;
char lanAddress[64];
result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress);
freeUPNPDevlist(deviceList);
if (result != 0) {
if (result == 1) {
std::ostringstream portString;
portString << m_listenning_port;
// Delete the port mapping before we create it, just in case we have dangling port mapping from the daemon not being shut down correctly
UPNP_DeletePortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), "TCP", 0);
int portMappingResult;
portMappingResult = UPNP_AddPortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), portString.str().c_str(), lanAddress, CRYPTONOTE_NAME, "TCP", 0, "0");
if (portMappingResult != 0) {
LOG_ERROR("UPNP_AddPortMapping failed, error: " << strupnperror(portMappingResult));
} else {
LOG_PRINT_GREEN("Added IGD port mapping.", LOG_LEVEL_0);
}
} else if (result == 2) {
LOG_PRINT_L0("IGD was found but reported as not connected.");
} else if (result == 3) {
LOG_PRINT_L0("UPnP device was found but not recoginzed as IGD.");
} else {
LOG_ERROR("UPNP_GetValidIGD returned an unknown result code.");
}
FreeUPNPUrls(&urls);
} else {
LOG_PRINT_L0("No IGD was found.");
}
int portMappingResult;
portMappingResult = UPNP_AddPortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), portString.str().c_str(), lanAddress, CRYPTONOTE_NAME, "TCP", 0, "0");
if (portMappingResult != 0) {
LOG_ERROR("UPNP_AddPortMapping failed, error: " << strupnperror(portMappingResult));
} else {
LOG_PRINT_GREEN("Added IGD port mapping.", LOG_LEVEL_0);
}
} else if (result == 2) {
LOG_PRINT_L0("IGD was found but reported as not connected.");
} else if (result == 3) {
LOG_PRINT_L0("UPnP device was found but not recoginzed as IGD.");
} else {
LOG_ERROR("UPNP_GetValidIGD returned an unknown result code.");
}
FreeUPNPUrls(&urls);
} else {
LOG_PRINT_L0("No IGD was found.");
}
}
return res;
}
//-----------------------------------------------------------------------------------
@ -1300,4 +1332,83 @@ namespace nodetool
return true;
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max)
{
if(max == -1) {
m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT;
return true;
}
m_config.m_net_config.connections_count = max;
LOG_PRINT_RED_L0("connections_count: " << m_config.m_net_config.connections_count);
return true;
}
template<class t_payload_net_handler>
void node_server<t_payload_net_handler>::delete_connections(size_t count)
{
m_net_server.get_config_object().del_connections(count);
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_tos_flag(const boost::program_options::variables_map& vm, int flag)
{
if(flag==-1){
return true;
}
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_tos_flag(flag);
_dbg1("Set ToS flag " << flag);
return true;
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit)
{
this->islimitup=true;
if (limit==-1) {
limit=128;
this->islimitup=false;
}
limit *= 1024;
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit );
LOG_PRINT_L0("Set limit-up to " << limit/1024 << " kB/s");
return true;
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit)
{
this->islimitdown=true;
if(limit==-1) {
limit=128;
this->islimitdown=false;
}
limit *= 1024;
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit );
LOG_PRINT_L0("Set limit-down to " << limit/1024 << " kB/s");
return true;
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit)
{
limit *= 1024;
if(this->islimitdown==false && this->islimitup==false) {
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit );
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit );
LOG_PRINT_L0("Set limit to " << limit/1024 << " kB/s");
}
else if(this->islimitdown==false && this->islimitup==true ) {
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit );
}
else if(this->islimitdown==true && this->islimitup==false ) {
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit );
}
return true;
}
}

382
src/p2p/network_throttle-detail.cpp Normal file
View File

@ -0,0 +1,382 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief implementaion for throttling of connection (count and rate-limit speed etc)
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/* rfree: implementation for throttle details */
#include <boost/asio.hpp>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/interprocess/detail/atomic.hpp>
#include <boost/thread/thread.hpp>
#include <memory>
#include "syncobj.h"
#include "../../contrib/epee/include/net/net_utils_base.h"
#include "../../contrib/epee/include/misc_log_ex.h"
#include <boost/lambda/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/chrono.hpp>
#include <boost/utility/value_init.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/thread/thread.hpp>
#include "misc_language.h"
#include "pragma_comp_defs.h"
#include <sstream>
#include <iomanip>
#include <algorithm>
#include <boost/asio/basic_socket.hpp>
#include <boost/asio/ip/unicast.hpp>
#include "../../contrib/epee/include/net/abstract_tcp_server2.h"
// TODO:
#include "../../src/p2p/network_throttle-detail.hpp"
#include "../../contrib/otshell_utils/utils.hpp"
using namespace nOT::nUtils;
// ################################################################################################
// ################################################################################################
// the "header part". Not separeted out for .hpp because point of this modification is
// to rebuild just 1 translation unit while working on this code.
// (But maybe common parts will be separated out later though - if needed)
// ################################################################################################
// ################################################################################################
using namespace nOT::nUtils;
namespace epee
{
namespace net_utils
{
/* ============================================================================ */
class connection_basic_pimpl {
public:
connection_basic_pimpl(const std::string &name);
static int m_default_tos;
network_throttle_bw m_throttle; // per-perr
critical_section m_throttle_lock;
void _packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) could be used for different kinds of sleep e.g. direct/queue write
};
} // namespace
} // namespace
// ################################################################################################
// ################################################################################################
// The implementation part
// ################################################################################################
// ################################################################################################
namespace epee
{
namespace net_utils
{
// ================================================================================================
// network_throttle
// ================================================================================================
network_throttle::~network_throttle() { }
network_throttle::packet_info::packet_info()
: m_size(0)
{
}
network_throttle::network_throttle(const std::string &nameshort, const std::string &name, int window_size)
: m_window_size( (window_size==-1) ? 10 : window_size ),
m_history( m_window_size ), m_nameshort(nameshort)
{
set_name(name);
m_network_add_cost = 128;
m_network_minimal_segment = 256;
m_network_max_segment = 1024*1024;
m_any_packet_yet = false;
m_slot_size = 1.0; // hard coded in few places
m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle
m_target_MB = 0;
}
void network_throttle::set_name(const std::string &name)
{
m_name = name;
}
void network_throttle::set_target_speed( network_speed_kbps target )
{
m_target_speed = target;
_note_c("net/"+m_nameshort, "Setting LIMIT: " << target << " kbps");
}
void network_throttle::set_target_kill( network_MB target )
{
_note_c("net/"+m_nameshort, "Setting KILL: " << target << " MB hard limit");
m_target_MB = target;
}
void network_throttle::tick()
{
double time_now = get_time_seconds();
if (!m_any_packet_yet) m_start_time = time_now; // starting now
network_time_seconds current_sample_time_slot = time_to_slot( time_now ); // T=13.7 --> 13 (for 1-second smallwindow)
network_time_seconds last_sample_time_slot = time_to_slot( m_last_sample_time );
// moving to next position, and filling gaps
// !! during this loop the m_last_sample_time and last_sample_time_slot mean the variable moved in +1
// TODO optimize when moving few slots at once
while ( (!m_any_packet_yet) || (last_sample_time_slot < current_sample_time_slot))
{
LOG_PRINT_L4("Moving counter buffer by 1 second " << last_sample_time_slot << " < " << current_sample_time_slot << " (last time " << m_last_sample_time<<")");
// rotate buffer
for (size_t i=m_history.size()-1; i>=1; --i) m_history[i] = m_history[i-1];
m_history[0] = packet_info();
if (! m_any_packet_yet)
{
m_last_sample_time = time_now;
}
m_last_sample_time += 1; last_sample_time_slot = time_to_slot( m_last_sample_time ); // increase and recalculate time, time slot
m_any_packet_yet=true;
}
m_last_sample_time = time_now; // the real exact last time
}
void network_throttle::handle_trafic_exact(size_t packet_size)
{
_handle_trafic_exact(packet_size, packet_size);
}
void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_size)
{
tick();
calculate_times_struct cts ; calculate_times(packet_size, cts , false, -1);
calculate_times_struct cts2; calculate_times(packet_size, cts2, false, 5);
m_history[0].m_size += packet_size;
std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends;
std::string history_str = oss.str();
logger_handle_net("log/dr-monero/net/inreq-all.data",get_time_seconds(),packet_size);
_info_c( "net/" + m_nameshort , "Throttle " << m_name << ": packet of ~"<<packet_size<<"b " << " (from "<<orginal_size<<" b)"
<< " Speed AVG=" << std::setw(4) << ((long int)(cts .average/1024)) <<"[w="<<cts .window<<"]"
<< " " << std::setw(4) << ((long int)(cts2.average/1024)) <<"[w="<<cts2.window<<"]"
<<" / " << " Limit="<< ((long int)(m_target_speed/1024)) <<" KiB/sec "
<< " " << history_str
);
}
void network_throttle::handle_trafic_tcp(size_t packet_size)
{
size_t all_size = packet_size + m_network_add_cost;
all_size = std::max( m_network_minimal_segment , all_size);
_handle_trafic_exact( all_size , packet_size );
}
void network_throttle::handle_congestion(double overheat) {
// TODO
}
network_time_seconds network_throttle::get_sleep_time_after_tick(size_t packet_size) {
tick();
return get_sleep_time(packet_size);
}
void network_throttle::logger_handle_net(const std::string &filename, double time, size_t size) {
std::mutex mutex;
mutex.lock(); {
std::fstream file;
file.open(filename.c_str(), std::ios::app | std::ios::out );
if(!file.is_open())
_warn("Can't open file " << filename);
file << time << " " << size/1024 << "\n";
file.close();
} mutex.unlock();
}
// fine tune this to decide about sending speed:
network_time_seconds network_throttle::get_sleep_time(size_t packet_size) const
{
//_scope_mark("");
double D2=0;
calculate_times_struct cts = { 0, 0, 0, 0};
//calculate_times(packet_size, cts, false, m_window_size/2); D2=cts.delay;
//calculate_times(packet_size, cts, true, m_window_size/2); D2=cts.delay;
calculate_times(packet_size, cts, true, m_window_size); D2=cts.delay;
return D2;
}
double network_throttle::get_current_overheat() const {
auto now = get_time_seconds();
auto diff = now - m_overheat_time;
auto overheat = m_overheat - diff;
overheat = std::max(m_overheat, 0.);
return overheat;
}
void network_throttle::set_overheat(double lag) {
m_overheat += lag;
m_overheat_time = get_time_seconds();
LOG_PRINT_L0("Lag: " << lag << ", overheat: " << m_overheat );
}
// MAIN LOGIC:
void network_throttle::calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const
{
const double the_window_size = std::max( (double)m_window_size ,
((force_window>0) ? force_window : m_window_size)
);
if (!m_any_packet_yet) {
cts.window=0; cts.average=0; cts.delay=0;
cts.recomendetDataSize = m_network_minimal_segment; // should be overrided by caller anyway
return ; // no packet yet, I can not decide about sleep time
}
network_time_seconds window_len = (the_window_size-1) * m_slot_size ; // -1 since current slot is not finished
window_len += (m_last_sample_time - time_to_slot(m_last_sample_time)); // add the time for current slot e.g. 13.7-13 = 0.7
auto time_passed = get_time_seconds() - m_start_time;
cts.window = std::max( std::min( window_len , time_passed ) , m_slot_size ) ; // window length resulting from size of history but limited by how long ago history was started,
// also at least slot size (e.g. 1 second) to not be ridiculous
// window_len e.g. 5.7 because takes into account current slot time
size_t Epast = 0; // summ of traffic till now
for (auto sample : m_history) Epast += sample.m_size;
const size_t E = Epast;
const size_t Enow = Epast + packet_size ; // including the data we're about to send now
const double M = m_target_speed; // max
const double D1 = (Epast - M*cts.window) / M; // delay - how long to sleep to get back to target speed
const double D2 = (Enow - M*cts.window) / M; // delay - how long to sleep to get back to target speed (including current packet)
auto O = get_current_overheat();
auto Ouse = O * 0 ; // XXX TODO
cts.delay = (D1*0.80 + D2*0.20) + Ouse; // finall sleep depends on both with/without current packet
// update_overheat();
cts.average = Epast/cts.window; // current avg. speed (for info)
if (Epast <= 0) {
if (cts.delay>=0) cts.delay = 0; // no traffic in history so we will not wait
}
double Wgood=-1;
{ // how much data we recommend now to download
Wgood = the_window_size + 1;
cts.recomendetDataSize = M*cts.window - E;
}
if (dbg) {
std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends;
std::string history_str = oss.str();
_dbg1_c( "net/"+m_nameshort+"_c" ,
"dbg " << m_name << ": "
<< "speed is A=" << std::setw(8) <<cts.average<<" vs "
<< "Max=" << std::setw(8) <<M<<" "
<< " so sleep: "
<< "D=" << std::setw(8) <<cts.delay<<" sec "
<< "Overheat=" << std::setw(8) <<O<<" sec "
<< "E="<< std::setw(8) << E << " (Enow="<<std::setw(8)<<Enow<<") "
<< "M=" << std::setw(8) << M <<" W="<< std::setw(8) << cts.window << " "
<< "R=" << std::setw(8) << cts.recomendetDataSize << " Wgood" << std::setw(8) << Wgood << " "
<< "History: " << std::setw(8) << history_str << " "
<< "m_last_sample_time=" << std::setw(8) << m_last_sample_time
);
}
}
double network_throttle::get_time_seconds() const {
using namespace boost::chrono;
auto point = steady_clock::now();
auto time_from_epoh = point.time_since_epoch();
auto ms = duration_cast< milliseconds >( time_from_epoh ).count();
double ms_f = ms;
return ms_f / 1000.;
}
size_t network_throttle::get_recommended_size_of_planned_transport_window(double force_window) const {
calculate_times_struct cts = { 0, 0, 0, 0};
network_throttle::calculate_times(0, cts, true, force_window);
cts.recomendetDataSize += m_network_add_cost;
if (cts.recomendetDataSize<0) cts.recomendetDataSize=0;
if (cts.recomendetDataSize>m_network_max_segment) cts.recomendetDataSize=m_network_max_segment;
size_t RI = (long int)cts.recomendetDataSize;
return RI;
}
size_t network_throttle::get_recommended_size_of_planned_transport() const {
size_t R1=0,R2=0,R3=0;
R1 = get_recommended_size_of_planned_transport_window( -1 );
R2 = get_recommended_size_of_planned_transport_window( m_window_size/2);
R3 = get_recommended_size_of_planned_transport_window( 8 );
auto RM = std::min(R1, std::min(R2,R3));
const double a1=70, a2=10, a3=10, am=10; // weight of the various windows in decisssion
return (R1*a1 + R2*a2 + R3*a3 + RM*am) / (a1+a2+a3+am);
}
} // namespace
} // namespace

133
src/p2p/network_throttle-detail.hpp Normal file
View File

@ -0,0 +1,133 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief implementaion for throttling of connection (count and rate-limit speed etc)
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
/* rfree: throttle details, implementing rate limiting */
#ifndef INCLUDED_src_p2p_throttle_detail_hpp
#define INCLUDED_src_p2p_throttle_detail_hpp
#include "../../src/p2p/network_throttle.hpp"
namespace epee
{
namespace net_utils
{
class network_throttle : public i_network_throttle {
private:
struct packet_info {
size_t m_size; // octets sent. Summary for given small-window (e.g. for all packaged in 1 second)
packet_info();
};
network_speed_kbps m_target_speed;
network_MB m_target_MB;
size_t m_network_add_cost; // estimated add cost of headers
size_t m_network_minimal_segment; // estimated minimal cost of sending 1 byte to round up to
size_t m_network_max_segment; // recommended max size of 1 TCP transmission
const size_t m_window_size; // the number of samples to average over
network_time_seconds m_slot_size; // the size of one slot. TODO: now hardcoded for 1 second e.g. in time_to_slot()
// TODO for big window size, for performance better the substract on change of m_last_sample_time instead of recalculating average of eg >100 elements
std::vector< packet_info > m_history; // the history of bw usage
network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer
network_time_seconds m_start_time; // when we were created
bool m_any_packet_yet; // did we yet got any packet to count
double m_overheat; // last overheat
double m_overheat_time; // time in seconds after epoch
std::string m_name; // my name for debug and logs
std::string m_nameshort; // my name for debug and logs (used in log file name)
// each sample is now 1 second
public:
network_throttle(const std::string &nameshort, const std::string &name, int window_size=-1);
virtual ~network_throttle();
virtual void set_name(const std::string &name);
virtual void set_target_speed( network_speed_kbps target );
virtual void set_target_kill( network_MB target );
// add information about events:
virtual void handle_trafic_exact(size_t packet_size); ///< count the new traffic/packet; the size is exact considering all network costs
virtual void handle_trafic_tcp(size_t packet_size); ///< count the new traffic/packet; the size is as TCP, we will consider MTU etc
virtual void handle_congestion(double overheat); ///< call this when congestion is detected; see example use
virtual void tick(); ///< poke and update timers/history (recalculates, moves the history if needed, checks the real clock etc)
virtual double get_time_seconds() const ; ///< timer that we use, time in seconds, monotionic
virtual double get_current_overheat() const; ///< did we detected congestion now. NOT USED NOW TODO
virtual void set_overheat(double lag); ///< did we detected congestion now. NOT USED NOW TODO. rename to add_overheat ?
// time calculations:
virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const; ///< MAIN LOGIC (see base class for info)
virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size); ///< increase the timer if needed, and get the package size
virtual network_time_seconds get_sleep_time(size_t packet_size) const; ///< gets the Delay (recommended Delay time) from calc. (not safe: only if time didnt change?) TODO
virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported
virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const; ///< ditto, but for given windows time frame
//virtual void add_planned_transport(size_t size);
private:
virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13
virtual void _handle_trafic_exact(size_t packet_size, size_t orginal_size);
virtual void logger_handle_net(const std::string &filename, double time, size_t size);
};
/***
* The complete set of traffic throttle for one typical connection
*/
struct network_throttle_bw {
public:
network_throttle m_in; ///< for incomming traffic (this we can not controll directly as it depends of what others send to us - usually)
network_throttle m_inreq; ///< for requesting incomming traffic (this is exact usually)
network_throttle m_out; ///< for outgoing traffic that we just sent (this is exact usually)
public:
network_throttle_bw(const std::string &name1);
};
} // namespace net_utils
} // namespace epee
#endif

121
src/p2p/network_throttle.cpp Normal file
View File

@ -0,0 +1,121 @@
/**
@file
@author rfree (current maintainer in monero.cc project)
@brief interface for throttling of connection (count and rate-limit speed etc)
@details <PRE>
Throttling work by:
1) taking note of all traffic (hooks added e.g. to connection class) and measuring speed
2) depending on that information we sleep before sending out data (or send smaller portions of data)
3) depending on the information we can also sleep before sending requests or ask for smaller sets of data to download
</PRE>
@image html images/net/rate1-down-1k.png
@image html images/net/rate1-down-full.png
@image html images/net/rate1-up-10k.png
@image html images/net/rate1-up-full.png
@image html images/net/rate2-down-100k.png
@image html images/net/rate2-down-10k.png
@image html images/net/rate2-down-50k.png
@image html images/net/rate2-down-full.png
@image html images/net/rate2-up-100k.png
@image html images/net/rate2-up-10k.png
@image html images/net/rate3-up-10k.png
*/
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
#include "../../src/p2p/network_throttle-detail.hpp"
namespace epee
{
namespace net_utils
{
// ================================================================================================
// network_throttle_manager
// ================================================================================================
// ================================================================================================
// static:
std::mutex network_throttle_manager::m_lock_get_global_throttle_in;
std::mutex network_throttle_manager::m_lock_get_global_throttle_inreq;
std::mutex network_throttle_manager::m_lock_get_global_throttle_out;
int network_throttle_manager::xxx;
// ================================================================================================
// methods:
i_network_throttle & network_throttle_manager::get_global_throttle_in() {
std::call_once(m_once_get_global_throttle_in, [] { m_obj_get_global_throttle_in.reset(new network_throttle("in/all","<<< global-IN",10)); } );
return * m_obj_get_global_throttle_in;
}
std::once_flag network_throttle_manager::m_once_get_global_throttle_in;
std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_in;
i_network_throttle & network_throttle_manager::get_global_throttle_inreq() {
std::call_once(m_once_get_global_throttle_inreq, [] { m_obj_get_global_throttle_inreq.reset(new network_throttle("inreq/all", "<== global-IN-REQ",10)); } );
return * m_obj_get_global_throttle_inreq;
}
std::once_flag network_throttle_manager::m_once_get_global_throttle_inreq;
std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_inreq;
i_network_throttle & network_throttle_manager::get_global_throttle_out() {
std::call_once(m_once_get_global_throttle_out, [] { m_obj_get_global_throttle_out.reset(new network_throttle("out/all", ">>> global-OUT",10)); } );
return * m_obj_get_global_throttle_out;
}
std::once_flag network_throttle_manager::m_once_get_global_throttle_out;
std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_out;
network_throttle_bw::network_throttle_bw(const std::string &name1)
: m_in("in/"+name1, name1+"-DOWNLOAD"), m_inreq("inreq/"+name1, name1+"-DOWNLOAD-REQUESTS"), m_out("out/"+name1, name1+"-UPLOAD")
{ }
} // namespace
} // namespace

187
src/p2p/network_throttle.hpp Normal file
View File

@ -0,0 +1,187 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief interface for throttling of connection (count and rate-limit speed etc)
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
/* rfree: throttle basic interface */
/* rfree: also includes the manager for singeton/global such objects */
#ifndef INCLUDED_p2p_network_throttle_hpp
#define INCLUDED_p2p_network_throttle_hpp
#include <boost/asio.hpp>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/interprocess/detail/atomic.hpp>
#include <boost/thread/thread.hpp>
#include "syncobj.h"
#include "../../contrib/epee/include/net/net_utils_base.h"
#include "../../contrib/epee/include/misc_log_ex.h"
#include <boost/lambda/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/chrono.hpp>
#include <boost/utility/value_init.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/thread/thread.hpp>
#include "misc_language.h"
#include "pragma_comp_defs.h"
#include <sstream>
#include <iomanip>
#include <algorithm>
#include <memory>
#include <mutex>
#include <fstream>
namespace epee
{
namespace net_utils
{
// just typedefs to in code define the units used. TODO later it will be enforced that casts to other numericals are only explicit to avoid mistakes? use boost::chrono?
typedef double network_speed_kbps;
typedef double network_time_seconds;
typedef double network_MB;
class i_network_throttle;
/***
@brief All information about given throttle - speed calculations
*/
struct calculate_times_struct {
double average;
double window;
double delay;
double recomendetDataSize;
};
typedef calculate_times_struct calculate_times_struct;
namespace cryptonote { class cryptonote_protocol_handler_base; }; // a friend class // TODO friend not working
/***
@brief Access to simple throttles, with singlton to access global network limits
*/
class network_throttle_manager {
// provides global (singleton) in/inreq/out throttle access
// [[note1]] see also http://www.nuonsoft.com/blog/2012/10/21/implementing-a-thread-safe-singleton-with-c11/
// [[note2]] _inreq is the requested in traffic - we anticipate we will get in-bound traffic soon as result of what we do (e.g. that we sent network downloads requests)
//protected:
public: // XXX
// [[note1]]
static std::once_flag m_once_get_global_throttle_in;
static std::once_flag m_once_get_global_throttle_inreq; // [[note2]]
static std::once_flag m_once_get_global_throttle_out;
static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_in;
static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_inreq;
static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_out;
static std::mutex m_lock_get_global_throttle_in;
static std::mutex m_lock_get_global_throttle_inreq;
static std::mutex m_lock_get_global_throttle_out;
friend class cryptonote::cryptonote_protocol_handler_base; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS!
friend class connection_basic; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS!
friend class connection_basic_pimpl; // ditto
static int xxx;
public:
static i_network_throttle & get_global_throttle_in(); ///< singleton ; for friend class ; caller MUST use proper locks! like m_lock_get_global_throttle_in
static i_network_throttle & get_global_throttle_inreq(); ///< ditto ; use lock ... use m_lock_get_global_throttle_inreq obviously
static i_network_throttle & get_global_throttle_out(); ///< ditto ; use lock ... use m_lock_get_global_throttle_out obviously
};
/***
@brief interface for the throttle, see the derivated class
*/
class i_network_throttle {
public:
virtual void set_name(const std::string &name)=0;
virtual void set_target_speed( network_speed_kbps target )=0;
virtual void set_target_kill( network_MB target )=0;
virtual void handle_trafic_exact(size_t packet_size) =0; // count the new traffic/packet; the size is exact considering all network costs
virtual void handle_trafic_tcp(size_t packet_size) =0; // count the new traffic/packet; the size is as TCP, we will consider MTU etc
virtual void handle_congestion(double overheat) =0; // call this when congestion is detected; see example use
virtual void tick() =0; // poke and update timers/history
// time calculations:
virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const =0; // assuming sending new package (or 0), calculate:
// Average, Window, Delay, Recommended data size ; also gets dbg=debug flag, and forced widnow size if >0 or -1 for not forcing window size
// Average speed, Window size, recommended Delay to sleep now, Recommended size of data to send now
virtual network_time_seconds get_sleep_time(size_t packet_size) const =0; // gets the D (recommended Delay time) from calc
virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size) =0; // ditto, but first tick the timer
virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future
virtual double get_time_seconds() const =0; // a timer
virtual double get_current_overheat() const =0;
virtual void set_overheat(double lag) =0;
virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0;
};
// ... more in the -advanced.h file
} // namespace net_utils
} // namespace epee
#endif

1
src/simplewallet/CMakeLists.txt
View File

@ -50,6 +50,7 @@ target_link_libraries(simplewallet
crypto
common
mnemonics
p2p
${UNBOUND_LIBRARY}
${UPNP_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}

2
tests/core_proxy/CMakeLists.txt
View File

@ -38,6 +38,8 @@ add_executable(core_proxy
target_link_libraries(core_proxy
LINK_PRIVATE
cryptonote_core
cryptonote_protocol
p2p
${UPNP_LIBRARIES}
${Boost_CHRONO_LIBRARY}
${Boost_FILESYSTEM_LIBRARY}

1
tests/core_proxy/core_proxy.h
View File

@ -81,5 +81,6 @@ namespace tests
bool on_idle(){return true;}
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;}
bool handle_get_objects(cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote::cryptonote_connection_context& context){return true;}
cryptonote::blockchain_storage &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class proxy_core."); }
};
}

4
tests/net_load_tests/CMakeLists.txt
View File

@ -37,6 +37,8 @@ add_executable(net_load_tests_clt
${clt_headers})
target_link_libraries(net_load_tests_clt
LINK_PRIVATE
otshell_utils
p2p
${GTEST_MAIN_LIBRARIES}
${Boost_CHRONO_LIBRARY}
${Boost_DATE_TIME_LIBRARY}
@ -56,6 +58,8 @@ add_executable(net_load_tests_srv
${srv_headers})
target_link_libraries(net_load_tests_srv
LINK_PRIVATE
otshell_utils
p2p
${GTEST_MAIN_LIBRARIES}
${Boost_CHRONO_LIBRARY}
${Boost_DATE_TIME_LIBRARY}

3
tests/net_load_tests/clt.cpp
View File

@ -44,7 +44,10 @@
#include "net_load_tests.h"
#include "../../contrib/otshell_utils/utils.hpp"
using namespace net_load_tests;
using namespace nOT::nUtils;
namespace
{

1
tests/unit_tests/CMakeLists.txt
View File

@ -58,6 +58,7 @@ target_link_libraries(unit_tests
cryptonote_core
rpc
wallet
p2p
${GTEST_MAIN_LIBRARIES}
${Boost_CHRONO_LIBRARY}
${Boost_REGEX_LIBRARY}