Merge pull request #6601

98c151ecb Optimize ZMQ-JSON vector reading; GetBlocksFast reads 24%+ faster (Lee Clagett)
60627c9f2 Switch to insitu parsing for ZMQ-JSON; GetBlocksFast reads 13%+ faster (Lee Clagett)
fe96e66eb Fix pruned tx for ZMQ's GetBlocksFast (Lee Clagett)
This commit is contained in:
Alexander Blair 2020-08-16 12:43:10 -07:00
commit 009ca6fcd3
No known key found for this signature in database
GPG Key ID: C64552D877C32479
10 changed files with 75 additions and 32 deletions

View File

@ -182,6 +182,7 @@ namespace rpc
for (const auto& blob : it->second) for (const auto& blob : it->second)
{ {
bwt.transactions.emplace_back(); bwt.transactions.emplace_back();
bwt.transactions.back().pruned = req.prune;
if (!parse_and_validate_tx_from_blob(blob.second, bwt.transactions.back())) if (!parse_and_validate_tx_from_blob(blob.second, bwt.transactions.back()))
{ {
res.blocks.clear(); res.blocks.clear();
@ -905,13 +906,13 @@ namespace rpc
return true; return true;
} }
epee::byte_slice DaemonHandler::handle(const std::string& request) epee::byte_slice DaemonHandler::handle(std::string&& request)
{ {
MDEBUG("Handling RPC request: " << request); MDEBUG("Handling RPC request: " << request);
try try
{ {
FullMessage req_full(request, true); FullMessage req_full(std::move(request), true);
const std::string request_type = req_full.getRequestType(); const std::string request_type = req_full.getRequestType();
const auto matched_handler = std::lower_bound(std::begin(handlers), std::end(handlers), request_type); const auto matched_handler = std::lower_bound(std::begin(handlers), std::end(handlers), request_type);

View File

@ -133,7 +133,7 @@ class DaemonHandler : public RpcHandler
void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res); void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res);
epee::byte_slice handle(const std::string& request) override final; epee::byte_slice handle(std::string&& request) override final;
private: private:

View File

@ -79,9 +79,12 @@ void Message::fromJson(const rapidjson::Value& val)
GET_FROM_JSON_OBJECT(val, rpc_version, rpc_version); GET_FROM_JSON_OBJECT(val, rpc_version, rpc_version);
} }
FullMessage::FullMessage(const std::string& json_string, bool request) FullMessage::FullMessage(std::string&& json_string, bool request)
: contents(std::move(json_string)), doc()
{ {
doc.Parse(json_string.c_str()); /* Insitu parsing does not copy data from `contents` to DOM,
accelerating string heavy content. */
doc.ParseInsitu(std::addressof(contents[0]));
if (doc.HasParseError() || !doc.IsObject()) if (doc.HasParseError() || !doc.IsObject())
{ {
throw cryptonote::json::PARSE_FAIL(); throw cryptonote::json::PARSE_FAIL();

View File

@ -72,9 +72,7 @@ namespace rpc
public: public:
~FullMessage() { } ~FullMessage() { }
FullMessage(FullMessage&& rhs) noexcept : doc(std::move(rhs.doc)) { } FullMessage(std::string&& json_string, bool request=false);
FullMessage(const std::string& json_string, bool request=false);
std::string getRequestType() const; std::string getRequestType() const;
@ -91,10 +89,13 @@ namespace rpc
private: private:
FullMessage() = default; FullMessage() = default;
FullMessage(const FullMessage&) = delete;
FullMessage& operator=(const FullMessage&) = delete;
FullMessage(const std::string& request, Message* message); FullMessage(const std::string& request, Message* message);
FullMessage(Message* message); FullMessage(Message* message);
std::string contents;
rapidjson::Document doc; rapidjson::Document doc;
}; };

View File

@ -55,7 +55,7 @@ class RpcHandler
RpcHandler() { } RpcHandler() { }
virtual ~RpcHandler() { } virtual ~RpcHandler() { }
virtual epee::byte_slice handle(const std::string& request) = 0; virtual epee::byte_slice handle(std::string&& request) = 0;
static boost::optional<output_distribution_data> static boost::optional<output_distribution_data>
get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height); get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height);

View File

@ -158,9 +158,9 @@ void ZmqServer::serve()
if (!pub || sockets[2].revents) if (!pub || sockets[2].revents)
{ {
const std::string message = MONERO_UNWRAP(net::zmq::receive(rep.get(), read_flags)); std::string message = MONERO_UNWRAP(net::zmq::receive(rep.get(), read_flags));
MDEBUG("Received RPC request: \"" << message << "\""); MDEBUG("Received RPC request: \"" << message << "\"");
epee::byte_slice response = handler.handle(message); epee::byte_slice response = handler.handle(std::move(message));
const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()}; const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()};
MDEBUG("Sending RPC reply: \"" << response_view << "\""); MDEBUG("Sending RPC reply: \"" << response_view << "\"");

View File

@ -42,6 +42,7 @@ monero_add_library(serialization
${serialization_private_headers}) ${serialization_private_headers})
target_link_libraries(serialization target_link_libraries(serialization
LINK_PRIVATE LINK_PRIVATE
cryptonote_basic
cryptonote_core cryptonote_core
cryptonote_protocol cryptonote_protocol
epee epee

View File

@ -33,6 +33,8 @@
#include <limits> #include <limits>
#include <type_traits> #include <type_traits>
#include "cryptonote_basic/cryptonote_basic_impl.h"
// drop macro from windows.h // drop macro from windows.h
#ifdef GetObject #ifdef GetObject
#undef GetObject #undef GetObject
@ -246,7 +248,10 @@ void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::t
INSERT_INTO_JSON_OBJECT(dest, inputs, tx.vin); INSERT_INTO_JSON_OBJECT(dest, inputs, tx.vin);
INSERT_INTO_JSON_OBJECT(dest, outputs, tx.vout); INSERT_INTO_JSON_OBJECT(dest, outputs, tx.vout);
INSERT_INTO_JSON_OBJECT(dest, extra, tx.extra); INSERT_INTO_JSON_OBJECT(dest, extra, tx.extra);
INSERT_INTO_JSON_OBJECT(dest, signatures, tx.signatures); if (!tx.pruned)
{
INSERT_INTO_JSON_OBJECT(dest, signatures, tx.signatures);
}
INSERT_INTO_JSON_OBJECT(dest, ringct, tx.rct_signatures); INSERT_INTO_JSON_OBJECT(dest, ringct, tx.rct_signatures);
dest.EndObject(); dest.EndObject();
@ -265,8 +270,17 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx)
GET_FROM_JSON_OBJECT(val, tx.vin, inputs); GET_FROM_JSON_OBJECT(val, tx.vin, inputs);
GET_FROM_JSON_OBJECT(val, tx.vout, outputs); GET_FROM_JSON_OBJECT(val, tx.vout, outputs);
GET_FROM_JSON_OBJECT(val, tx.extra, extra); GET_FROM_JSON_OBJECT(val, tx.extra, extra);
GET_FROM_JSON_OBJECT(val, tx.signatures, signatures);
GET_FROM_JSON_OBJECT(val, tx.rct_signatures, ringct); GET_FROM_JSON_OBJECT(val, tx.rct_signatures, ringct);
const auto& sigs = val.FindMember("signatures");
if (sigs != val.MemberEnd())
{
fromJsonValue(sigs->value, tx.signatures);
}
const auto& rsig = tx.rct_signatures;
if (!cryptonote::is_coinbase(tx) && rsig.p.bulletproofs.empty() && rsig.p.rangeSigs.empty() && rsig.p.MGs.empty() && rsig.get_pseudo_outs().empty() && sigs == val.MemberEnd())
tx.pruned = true;
} }
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block& b) void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block& b)
@ -1062,6 +1076,7 @@ void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig&
INSERT_INTO_JSON_OBJECT(dest, fee, sig.txnFee); INSERT_INTO_JSON_OBJECT(dest, fee, sig.txnFee);
// prunable // prunable
if (!sig.p.bulletproofs.empty() || !sig.p.rangeSigs.empty() || !sig.p.MGs.empty() || !sig.get_pseudo_outs().empty())
{ {
dest.Key("prunable"); dest.Key("prunable");
dest.StartObject(); dest.StartObject();
@ -1086,35 +1101,39 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig)
throw WRONG_TYPE("json object"); throw WRONG_TYPE("json object");
} }
std::vector<rct::key> commitments;
GET_FROM_JSON_OBJECT(val, sig.type, type); GET_FROM_JSON_OBJECT(val, sig.type, type);
GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, encrypted); GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, encrypted);
GET_FROM_JSON_OBJECT(val, commitments, commitments); GET_FROM_JSON_OBJECT(val, sig.outPk, commitments);
GET_FROM_JSON_OBJECT(val, sig.txnFee, fee); GET_FROM_JSON_OBJECT(val, sig.txnFee, fee);
// prunable // prunable
const auto prunable = val.FindMember("prunable");
if (prunable != val.MemberEnd())
{ {
OBJECT_HAS_MEMBER_OR_THROW(val, "prunable"); rct::keyV pseudo_outs = std::move(sig.get_pseudo_outs());
const auto& prunable = val["prunable"];
rct::keyV pseudo_outs; GET_FROM_JSON_OBJECT(prunable->value, sig.p.rangeSigs, range_proofs);
GET_FROM_JSON_OBJECT(prunable->value, sig.p.bulletproofs, bulletproofs);
GET_FROM_JSON_OBJECT(prunable, sig.p.rangeSigs, range_proofs); GET_FROM_JSON_OBJECT(prunable->value, sig.p.MGs, mlsags);
GET_FROM_JSON_OBJECT(prunable, sig.p.bulletproofs, bulletproofs); GET_FROM_JSON_OBJECT(prunable->value, pseudo_outs, pseudo_outs);
GET_FROM_JSON_OBJECT(prunable, sig.p.MGs, mlsags);
GET_FROM_JSON_OBJECT(prunable, pseudo_outs, pseudo_outs);
sig.get_pseudo_outs() = std::move(pseudo_outs); sig.get_pseudo_outs() = std::move(pseudo_outs);
} }
else
sig.outPk.reserve(commitments.size());
for (rct::key const& commitment : commitments)
{ {
sig.outPk.push_back({{}, commitment}); sig.p.rangeSigs.clear();
sig.p.bulletproofs.clear();
sig.p.MGs.clear();
sig.get_pseudo_outs().clear();
} }
} }
void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key)
{
key.dest = {};
fromJsonValue(val, key.mask);
}
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple) void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple)
{ {
dest.StartObject(); dest.StartObject();

View File

@ -32,6 +32,7 @@
#include <cstring> #include <cstring>
#include <rapidjson/document.h> #include <rapidjson/document.h>
#include <rapidjson/writer.h> #include <rapidjson/writer.h>
#include <vector>
#include "byte_stream.h" #include "byte_stream.h"
#include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/cryptonote_basic.h"
@ -277,6 +278,8 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& i); void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& i);
void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig); void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig);
void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple); void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple);
void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple); void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple);
@ -339,6 +342,7 @@ inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type from
auto itr = val.MemberBegin(); auto itr = val.MemberBegin();
map.clear();
while (itr != val.MemberEnd()) while (itr != val.MemberEnd())
{ {
typename Map::key_type k; typename Map::key_type k;
@ -359,6 +363,19 @@ inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type t
dest.EndArray(); dest.EndArray();
} }
namespace traits
{
template<typename T>
void reserve(const T&, std::size_t)
{}
template<typename T>
void reserve(std::vector<T>& vec, const std::size_t count)
{
vec.reserve(count);
}
}
template <typename Vec> template <typename Vec>
inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type fromJsonValue(const rapidjson::Value& val, Vec& vec) inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type fromJsonValue(const rapidjson::Value& val, Vec& vec)
{ {
@ -367,11 +384,12 @@ inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type f
throw WRONG_TYPE("json array"); throw WRONG_TYPE("json array");
} }
vec.clear();
traits::reserve(vec, val.Size());
for (rapidjson::SizeType i=0; i < val.Size(); i++) for (rapidjson::SizeType i=0; i < val.Size(); i++)
{ {
typename Vec::value_type v; vec.emplace_back();
fromJsonValue(val[i], v); fromJsonValue(val[i], vec.back());
vec.push_back(v);
} }
} }

View File

@ -304,7 +304,7 @@ namespace
: cryptonote::rpc::RpcHandler() : cryptonote::rpc::RpcHandler()
{} {}
virtual epee::byte_slice handle(const std::string& request) override final virtual epee::byte_slice handle(std::string&& request) override final
{ {
throw std::logic_error{"not implemented"}; throw std::logic_error{"not implemented"};
} }