device/trezor: debugging features, trezor tests

Makefile

@@ -63,6 +63,10 @@ debug-test:
 	mkdir -p $(builddir)/debug
 	cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Debug $(topdir) &&  $(MAKE) && $(MAKE) ARGS="-E libwallet_api_tests" test
 
+debug-test-trezor:
+	mkdir -p $(builddir)/debug
+	cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D TREZOR_DEBUG=ON -D CMAKE_BUILD_TYPE=Debug $(topdir) &&  $(MAKE) && $(MAKE) ARGS="-E libwallet_api_tests" test
+
 debug-all:
 	mkdir -p $(builddir)/debug
 	cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D BUILD_SHARED_LIBS=OFF -D CMAKE_BUILD_TYPE=Debug $(topdir) && $(MAKE)

cmake/CheckTrezor.cmake

@@ -1,6 +1,8 @@
 OPTION(USE_DEVICE_TREZOR "Trezor support compilation" ON)
 OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" ON)
 OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" OFF)
+OPTION(USE_DEVICE_TREZOR_DEBUG "Trezor Debugging enabled" OFF)
+OPTION(TREZOR_DEBUG "Main trezor debugging switch" OFF)
 
 # Helper function to fix cmake < 3.6.0 FindProtobuf variables
 function(_trezor_protobuf_fix_vars)
@@ -53,6 +55,14 @@ if (USE_DEVICE_TREZOR)
         set(Protobuf_FOUND 1)  # override found if all rquired info was provided by variables
     endif()
 
+    if(TREZOR_DEBUG)
+        set(USE_DEVICE_TREZOR_DEBUG 1)
+    endif()
+
+    # Compile debugging support (for tests)
+    if (USE_DEVICE_TREZOR_DEBUG)
+        add_definitions(-DWITH_TREZOR_DEBUGGING=1)
+    endif()
 else()
     message(STATUS "Trezor support disabled by USE_DEVICE_TREZOR")
 endif()
@@ -106,7 +116,12 @@ endif()
 if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_TEST_PASSED)
     set(ENV{PROTOBUF_INCLUDE_DIRS} "${Protobuf_INCLUDE_DIR}")
     set(ENV{PROTOBUF_PROTOC_EXECUTABLE} "${Protobuf_PROTOC_EXECUTABLE}")
-    execute_process(COMMAND ${TREZOR_PYTHON} tools/build_protob.py WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../src/device_trezor/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR)
+    set(TREZOR_PROTOBUF_PARAMS "")
+    if (USE_DEVICE_TREZOR_DEBUG)
+        set(TREZOR_PROTOBUF_PARAMS "--debug")
+    endif()
+    
+    execute_process(COMMAND ${TREZOR_PYTHON} tools/build_protob.py ${TREZOR_PROTOBUF_PARAMS} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../src/device_trezor/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR)
     if(RET)
         message(WARNING "Trezor protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})."
                 "OUT: ${OUT}, ERR: ${ERR}."

src/device_trezor/CMakeLists.txt

@@ -67,6 +67,12 @@ set(trezor_private_headers)
 if(DEVICE_TREZOR_READY)
     message(STATUS "Trezor support enabled")
 
+    if(USE_DEVICE_TREZOR_DEBUG)
+        list(APPEND trezor_headers trezor/debug_link.hpp trezor/messages/messages-debug.pb.h)
+        list(APPEND trezor_sources trezor/debug_link.cpp trezor/messages/messages-debug.pb.cc)
+        message(STATUS "Trezor debugging enabled")
+    endif()
+
     monero_private_headers(device_trezor
             ${device_private_headers})
 

src/device_trezor/device_trezor.cpp

@@ -97,7 +97,14 @@ namespace trezor {
         auto res = get_view_key();
         CHECK_AND_ASSERT_MES(res->watch_key().size() == 32, false, "Trezor returned invalid view key");
 
+        // Trezor does not make use of spendkey of the device API.
+        // Ledger loads encrypted spendkey, Trezor loads null key (never leaves device).
+        // In the test (debugging mode) we need to leave this field intact as it is already set by
+        // the debugging code and need to remain same for the testing purposes.
+#ifndef WITH_TREZOR_DEBUGGING
         spendkey = crypto::null_skey; // not given
+#endif
+
         memcpy(viewkey.data, res->watch_key().data(), 32);
 
         return true;
@@ -362,13 +369,9 @@ namespace trezor {
       CHECK_AND_ASSERT_THROW_MES(m_features, "Device state not initialized");  // make sure the caller did not reset features
       const bool nonce_required = init_msg->tsx_data().has_payment_id() && init_msg->tsx_data().payment_id().size() > 0;
 
-      if (nonce_required){
+      if (nonce_required && init_msg->tsx_data().payment_id().size() == 8){
         // Versions 2.0.9 and lower do not support payment ID
-        CHECK_AND_ASSERT_THROW_MES(m_features->has_major_version() && m_features->has_minor_version() && m_features->has_patch_version(), "Invalid Trezor firmware version information");
+        if (get_version() <= pack_version(2, 0, 9)) {
-        const uint32_t vma = m_features->major_version();
-        const uint32_t vmi = m_features->minor_version();
-        const uint32_t vpa = m_features->patch_version();
-        if (vma < 2 || (vma == 2 && vmi == 0 && vpa <= 9)) {
           throw exc::TrezorException("Trezor firmware 2.0.9 and lower does not support transactions with short payment IDs or integrated addresses. Please update.");
         }
       }

src/device_trezor/device_trezor_base.cpp

@@ -44,7 +44,9 @@ namespace trezor {
     const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080};
 
     device_trezor_base::device_trezor_base(): m_callback(nullptr) {
-
+#ifdef WITH_TREZOR_DEBUGGING
+      m_debug = false;
+#endif
     }
 
     device_trezor_base::~device_trezor_base() {
@@ -130,6 +132,10 @@ namespace trezor {
         }
 
         m_transport->open();
+
+#ifdef WITH_TREZOR_DEBUGGING
+        setup_debug();
+#endif
         return true;
 
       } catch(std::exception const& e){
@@ -153,6 +159,13 @@ namespace trezor {
           return false;
         }
       }
+
+#ifdef WITH_TREZOR_DEBUGGING
+      if (m_debug_callback) {
+        m_debug_callback->on_disconnect();
+        m_debug_callback = nullptr;
+      }
+#endif
       return true;
     }
 
@@ -355,6 +368,32 @@ namespace trezor {
       device_state_reset_unsafe();
     }
 
+#ifdef WITH_TREZOR_DEBUGGING
+#define TREZOR_CALLBACK(method, ...) do { \
+  if (m_debug_callback) m_debug_callback->method(__VA_ARGS__); \
+  if (m_callback) m_callback->method(__VA_ARGS__);             \
+}while(0)
+
+    void device_trezor_base::setup_debug(){
+      if (!m_debug){
+        return;
+      }
+
+      if (!m_debug_callback){
+        CHECK_AND_ASSERT_THROW_MES(m_transport, "Transport does not exist");
+        auto debug_transport = m_transport->find_debug();
+        if (debug_transport) {
+          m_debug_callback = std::make_shared<trezor_debug_callback>(debug_transport);
+        } else {
+          MDEBUG("Transport does not have debug link option");
+        }
+      }
+    }
+
+#else
+#define TREZOR_CALLBACK(method, ...) do { if (m_callback) m_callback->method(__VA_ARGS__); } while(0)
+#endif
+
     void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg)
     {
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
@@ -363,10 +402,7 @@ namespace trezor {
       messages::common::ButtonAck ack;
       write_raw(&ack);
 
-      if (m_callback){
+      TREZOR_CALLBACK(on_button_request);
-        m_callback->on_button_request();
-      }
-
       resp = read_raw();
     }
 
@@ -377,9 +413,7 @@ namespace trezor {
 
       epee::wipeable_string pin;
 
-      if (m_callback){
+      TREZOR_CALLBACK(on_pin_request, pin);
-        m_callback->on_pin_request(pin);
-      }
 
       // TODO: remove PIN from memory
       messages::common::PinMatrixAck m;
@@ -393,9 +427,7 @@ namespace trezor {
       MDEBUG("on_passhprase_request, on device: " << msg->on_device());
       epee::wipeable_string passphrase;
 
-      if (m_callback){
+      TREZOR_CALLBACK(on_passphrase_request, msg->on_device(), passphrase);
-        m_callback->on_passphrase_request(msg->on_device(), passphrase);
-      }
 
       messages::common::PassphraseAck m;
       if (!msg->on_device()){
@@ -421,5 +453,67 @@ namespace trezor {
       resp = call_raw(&m);
     }
 
+#ifdef WITH_TREZOR_DEBUGGING
+    void device_trezor_base::wipe_device()
+    {
+      auto msg = std::make_shared<messages::management::WipeDevice>();
+      auto ret = client_exchange<messages::common::Success>(msg);
+      (void)ret;
+      init_device();
+    }
+
+    void device_trezor_base::init_device()
+    {
+      auto msg = std::make_shared<messages::management::Initialize>();
+      m_features = client_exchange<messages::management::Features>(msg);
+    }
+
+    void device_trezor_base::load_device(const std::string & mnemonic, const std::string & pin,
+        bool passphrase_protection, const std::string & label, const std::string & language,
+        bool skip_checksum, bool expand)
+    {
+      if (m_features && m_features->initialized()){
+        throw std::runtime_error("Device is initialized already. Call device.wipe() and try again.");
+      }
+
+      auto msg = std::make_shared<messages::management::LoadDevice>();
+      msg->set_mnemonic(mnemonic);
+      msg->set_pin(pin);
+      msg->set_passphrase_protection(passphrase_protection);
+      msg->set_label(label);
+      msg->set_language(language);
+      msg->set_skip_checksum(skip_checksum);
+      auto ret = client_exchange<messages::common::Success>(msg);
+      (void)ret;
+
+      init_device();
+    }
+
+    trezor_debug_callback::trezor_debug_callback(std::shared_ptr<Transport> & debug_transport){
+      m_debug_link = std::make_shared<DebugLink>();
+      m_debug_link->init(debug_transport);
+    }
+
+    void trezor_debug_callback::on_button_request() {
+      if (m_debug_link) m_debug_link->press_yes();
+    }
+
+    void trezor_debug_callback::on_pin_request(epee::wipeable_string &pin) {
+
+    }
+
+    void trezor_debug_callback::on_passphrase_request(bool on_device, epee::wipeable_string &passphrase) {
+
+    }
+
+    void trezor_debug_callback::on_passphrase_state_request(const std::string &state) {
+
+    }
+
+    void trezor_debug_callback::on_disconnect(){
+      if (m_debug_link) m_debug_link->close();
+    }
+#endif
+
 #endif //WITH_DEVICE_TREZOR
 }}

src/device_trezor/device_trezor_base.hpp

@@ -42,6 +42,10 @@
 #include "cryptonote_config.h"
 #include "trezor.hpp"
 
+#ifdef WITH_TREZOR_DEBUGGING
+#include "trezor/debug_link.hpp"
+#endif
+
 //automatic lock one more level on device ensuring the current thread is allowed to use it
 #define AUTO_LOCK_CMD() \
   /* lock both mutexes without deadlock*/ \
@@ -57,6 +61,23 @@ namespace trezor {
 #ifdef WITH_DEVICE_TREZOR
   class device_trezor_base;
 
+#ifdef WITH_TREZOR_DEBUGGING
+    class trezor_debug_callback {
+    public:
+      trezor_debug_callback()=default;
+      explicit trezor_debug_callback(std::shared_ptr<Transport> & debug_transport);
+
+      void on_button_request();
+      void on_pin_request(epee::wipeable_string &pin);
+      void on_passphrase_request(bool on_device, epee::wipeable_string &passphrase);
+      void on_passphrase_state_request(const std::string &state);
+      void on_disconnect();
+    protected:
+      std::shared_ptr<DebugLink> m_debug_link;
+    };
+
+#endif
+
   /**
    * TREZOR device template with basic functions
    */
@@ -77,6 +98,13 @@ namespace trezor {
 
       cryptonote::network_type network_type;
 
+#ifdef WITH_TREZOR_DEBUGGING
+      std::shared_ptr<trezor_debug_callback> m_debug_callback;
+      bool m_debug;
+
+      void setup_debug();
+#endif
+
       //
       // Internal methods
       //
@@ -103,7 +131,7 @@ namespace trezor {
        * @throws UnexpectedMessageException if the response message type is different than expected.
        * Exception contains message type and the message itself.
        */
-      template<class t_message>
+      template<class t_message=google::protobuf::Message>
       std::shared_ptr<t_message>
       client_exchange(const std::shared_ptr<const google::protobuf::Message> &req,
                       const boost::optional<messages::MessageType> & resp_type = boost::none,
@@ -229,6 +257,12 @@ namespace trezor {
       return m_features;
     }
 
+    uint64_t get_version() const {
+      CHECK_AND_ASSERT_THROW_MES(m_features, "Features not loaded");
+      CHECK_AND_ASSERT_THROW_MES(m_features->has_major_version() && m_features->has_minor_version() && m_features->has_patch_version(), "Invalid Trezor firmware version information");
+      return pack_version(m_features->major_version(), m_features->minor_version(), m_features->patch_version());
+    }
+
     void set_derivation_path(const std::string &deriv_path) override;
 
     /* ======================================================================= */
@@ -268,6 +302,23 @@ namespace trezor {
     void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg);
     void on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg);
     void on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg);
+
+#ifdef WITH_TREZOR_DEBUGGING
+    void set_debug(bool debug){
+      m_debug = debug;
+    }
+
+    void set_debug_callback(std::shared_ptr<trezor_debug_callback> & debug_callback){
+      m_debug_callback = debug_callback;
+    }
+
+    void wipe_device();
+    void init_device();
+    void load_device(const std::string & mnemonic, const std::string & pin="", bool passphrase_protection=false,
+        const std::string & label="test", const std::string & language="english",
+        bool skip_checksum=false, bool expand=false);
+
+#endif
   };
 
 #endif

src/device_trezor/trezor/debug_link.cpp

@@ -0,0 +1,90 @@
+// Copyright (c) 2017-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+#include "debug_link.hpp"
+
+namespace hw{
+namespace trezor{
+
+  DebugLink::DebugLink(){
+
+  }
+
+  DebugLink::~DebugLink(){
+    if (m_transport){
+      close();
+    }
+  }
+
+  void DebugLink::init(std::shared_ptr<Transport> & transport){
+    CHECK_AND_ASSERT_THROW_MES(!m_transport, "Already initialized");
+    m_transport = transport;
+    m_transport->open();
+  }
+
+  void DebugLink::close(){
+    CHECK_AND_ASSERT_THROW_MES(m_transport, "Not initialized");
+    if (m_transport) m_transport->close();
+  }
+
+  std::shared_ptr<messages::debug::DebugLinkState> DebugLink::state(){
+    return call<messages::debug::DebugLinkState>(
+        messages::debug::DebugLinkGetState(),
+        boost::make_optional(messages::MessageType_DebugLinkGetState));
+  }
+
+  void DebugLink::input_word(const std::string & word){
+    messages::debug::DebugLinkDecision decision;
+    decision.set_input(word);
+    call(decision, boost::none, true);
+  }
+
+  void DebugLink::input_button(bool button){
+    messages::debug::DebugLinkDecision decision;
+    decision.set_yes_no(button);
+    call(decision, boost::none, true);
+  }
+
+  void DebugLink::input_swipe(bool swipe){
+    messages::debug::DebugLinkDecision decision;
+    decision.set_up_down(swipe);
+    call(decision, boost::none, true);
+  }
+
+  void DebugLink::stop(){
+    messages::debug::DebugLinkStop msg;
+    call(msg, boost::none, true);
+  }
+
+
+
+
+
+}
+}

src/device_trezor/trezor/debug_link.hpp

@@ -0,0 +1,93 @@
+// Copyright (c) 2017-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+#ifndef MONERO_DEBUG_LINK_H
+#define MONERO_DEBUG_LINK_H
+
+#include "transport.hpp"
+#include "messages/messages-debug.pb.h"
+
+
+namespace hw {
+namespace trezor {
+
+  class DebugLink {
+  public:
+
+    DebugLink();
+    virtual ~DebugLink();
+
+    void init(std::shared_ptr<Transport> & transport);
+    void close();
+
+    std::shared_ptr<messages::debug::DebugLinkState> state();
+    void input_word(const std::string & word);
+    void input_button(bool button);
+    void input_swipe(bool swipe);
+    void press_yes() { input_button(true); }
+    void press_no() { input_button(false); }
+    void stop();
+
+    template<class t_message=messages::debug::DebugLinkState>
+    std::shared_ptr<t_message> call(
+        const google::protobuf::Message & req,
+        const boost::optional<messages::MessageType> &resp_type = boost::none,
+        bool no_wait = false)
+    {
+      BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value);
+
+      m_transport->write(req);
+      if (no_wait){
+        return nullptr;
+      }
+
+      // Read the response
+      std::shared_ptr<google::protobuf::Message> msg_resp;
+      hw::trezor::messages::MessageType msg_resp_type;
+      m_transport->read(msg_resp, &msg_resp_type);
+
+      messages::MessageType required_type = resp_type ? resp_type.get() : MessageMapper::get_message_wire_number<t_message>();
+      if (msg_resp_type == required_type) {
+        return message_ptr_retype<t_message>(msg_resp);
+      } else if (msg_resp_type == messages::MessageType_Failure){
+        throw_failure_exception(dynamic_cast<messages::common::Failure*>(msg_resp.get()));
+      } else {
+        throw exc::UnexpectedMessageException(msg_resp_type, msg_resp);
+      }
+    };
+
+  private:
+    std::shared_ptr<Transport> m_transport;
+
+  };
+
+}
+}
+
+#endif //MONERO_DEBUG_LINK_H

src/device_trezor/trezor/messages_map.cpp

@@ -33,6 +33,10 @@
 #include "messages/messages-management.pb.h"
 #include "messages/messages-monero.pb.h"
 
+#ifdef WITH_TREZOR_DEBUGGING
+#include "messages/messages-debug.pb.h"
+#endif
+
 using namespace std;
 using namespace hw::trezor;
 
@@ -45,6 +49,9 @@ namespace trezor
       "hw.trezor.messages.",
       "hw.trezor.messages.common.",
       "hw.trezor.messages.management.",
+#ifdef WITH_TREZOR_DEBUGGING
+      "hw.trezor.messages.debug.",
+#endif
       "hw.trezor.messages.monero."
   };
 
@@ -68,6 +75,10 @@ namespace trezor
     hw::trezor::messages::management::Cancel::default_instance();
     hw::trezor::messages::monero::MoneroGetAddress::default_instance();
 
+#ifdef WITH_TREZOR_DEBUGGING
+    hw::trezor::messages::debug::DebugLinkDecision::default_instance();
+#endif
+
     google::protobuf::Descriptor const * desc = nullptr;
     for(const string &text : PACKAGES){
       desc = google::protobuf::DescriptorPool::generated_pool()

src/device_trezor/trezor/messages_map.hpp

@@ -62,14 +62,14 @@ namespace trezor {
     static messages::MessageType get_message_wire_number(const google::protobuf::Message & msg);
     static messages::MessageType get_message_wire_number(const std::string & msg_name);
 
-    template<class t_message>
+    template<class t_message=google::protobuf::Message>
     static messages::MessageType get_message_wire_number() {
       BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value);
       return get_message_wire_number(t_message::default_instance().GetDescriptor()->name());
     }
   };
 
-  template<class t_message>
+  template<class t_message=google::protobuf::Message>
   std::shared_ptr<t_message> message_ptr_retype(std::shared_ptr<google::protobuf::Message> & in){
     BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value);
     if (!in){
@@ -79,7 +79,7 @@ namespace trezor {
     return std::dynamic_pointer_cast<t_message>(in);
   }
 
-  template<class t_message>
+  template<class t_message=google::protobuf::Message>
   std::shared_ptr<t_message> message_ptr_retype_static(std::shared_ptr<google::protobuf::Message> & in){
     BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value);
     if (!in){

src/device_trezor/trezor/tools/build_protob.py

@@ -2,6 +2,12 @@
 import os
 import subprocess
 import sys
+import argparse
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-d", "--debug-msg", default=False, action="store_const", const=True, help="Build debug messages")
+args = parser.parse_args()
 
 CWD = os.path.dirname(os.path.realpath(__file__))
 ROOT_DIR = os.path.abspath(os.path.join(CWD, "..", "..", "..", ".."))
@@ -24,6 +30,10 @@ try:
         "messages-management.proto",
         "messages-monero.proto",
     ]
+
+    if args.debug_msg:
+        selected += ["messages-debug.proto"]
+
     proto_srcs = [os.path.join(TREZOR_COMMON, "protob", x) for x in selected]
     exec_args = [
         sys.executable,

src/device_trezor/trezor/transport.cpp

@@ -84,6 +84,17 @@ namespace trezor{
     return std::string(in.GetString());
   }
 
+  uint64_t pack_version(uint32_t major, uint32_t minor, uint32_t patch)
+  {
+    // packing (major, minor, patch) to 64 B: 16 B | 24 B | 24 B
+    const unsigned bits_1 = 16;
+    const unsigned bits_2 = 24;
+    const uint32_t mask_1 = (1 << bits_1) - 1;
+    const uint32_t mask_2 = (1 << bits_2) - 1;
+    CHECK_AND_ASSERT_THROW_MES(major <= mask_1 && minor <= mask_2 && patch <= mask_2, "Version numbers overflow packing scheme");
+    return patch | (((uint64_t)minor) << bits_2) | (((uint64_t)major) << (bits_1 + bits_2));
+  }
+
   //
   // Helpers
   //
@@ -212,6 +223,40 @@ namespace trezor{
     msg = msg_wrap;
   }
 
+  Transport::Transport(): m_open_counter(0) {
+
+  }
+
+  bool Transport::pre_open(){
+    if (m_open_counter > 0){
+      MTRACE("Already opened, count: " << m_open_counter);
+      m_open_counter += 1;
+      return false;
+
+    } else if (m_open_counter < 0){
+      MTRACE("Negative open value: " << m_open_counter);
+
+    }
+
+    // Caller should set m_open_counter to 1 after open
+    m_open_counter = 0;
+    return true;
+  }
+
+  bool Transport::pre_close(){
+    m_open_counter -= 1;
+
+    if (m_open_counter < 0){
+      MDEBUG("Already closed. Counter " << m_open_counter);
+
+    } else if (m_open_counter == 0) {
+      return true;
+
+    }
+
+    return false;
+  }
+
   //
   // Bridge transport
   //
@@ -246,6 +291,10 @@ namespace trezor{
   }
 
   void BridgeTransport::open() {
+    if (!pre_open()){
+      return;
+    }
+
     if (!m_device_path){
       throw exc::CommunicationException("Coud not open, empty device path");
     }
@@ -259,9 +308,15 @@ namespace trezor{
     }
 
     m_session = boost::make_optional(json_get_string(bridge_res["session"]));
+    m_open_counter = 1;
   }
 
   void BridgeTransport::close() {
+    if (!pre_close()){
+      return;
+    }
+
+    MTRACE("Closing Trezor:BridgeTransport");
     if (!m_device_path || !m_session){
       throw exc::CommunicationException("Device not open");
     }
@@ -423,6 +478,10 @@ namespace trezor{
   }
 
   void UdpTransport::open() {
+    if (!pre_open()){
+      return;
+    }
+
     udp::resolver resolver(m_io_service);
     udp::resolver::query query(udp::v4(), m_device_host, std::to_string(m_device_port));
     m_endpoint = *resolver.resolve(query);
@@ -434,9 +493,15 @@ namespace trezor{
     check_deadline();
 
     m_proto->session_begin(*this);
+    m_open_counter = 1;
   }
 
   void UdpTransport::close() {
+    if (!pre_close()){
+      return;
+    }
+
+    MTRACE("Closing Trezor:UdpTransport");
     if (!m_socket) {
       throw exc::CommunicationException("Socket is already closed");
     }
@@ -446,6 +511,19 @@ namespace trezor{
     m_socket = nullptr;
   }
 
+  std::shared_ptr<Transport> UdpTransport::find_debug() {
+#ifdef WITH_TREZOR_DEBUGGING
+    std::shared_ptr<UdpTransport> t = std::make_shared<UdpTransport>();
+    t->m_proto = std::make_shared<ProtocolV1>();
+    t->m_device_host = m_device_host;
+    t->m_device_port = m_device_port + 1;
+    return t;
+#else
+    MINFO("Debug link is disabled in production");
+    return nullptr;
+#endif
+  }
+
   void UdpTransport::write_chunk(const void * buff, size_t size){
     require_socket();
 
@@ -660,8 +738,7 @@ namespace trezor{
   WebUsbTransport::WebUsbTransport(
       boost::optional<libusb_device_descriptor*> descriptor,
       boost::optional<std::shared_ptr<Protocol>> proto
-  ): m_conn_count(0),
+  ): m_usb_session(nullptr), m_usb_device(nullptr), m_usb_device_handle(nullptr),
-     m_usb_session(nullptr), m_usb_device(nullptr), m_usb_device_handle(nullptr),
      m_bus_id(-1), m_device_addr(-1)
   {
     if (descriptor){
@@ -672,7 +749,7 @@ namespace trezor{
 
     m_proto = proto ? proto.get() : std::make_shared<ProtocolV1>();
 
-#ifdef WITH_TREZOR_DEBUG
+#ifdef WITH_TREZOR_DEBUGGING
     m_debug_mode = false;
 #endif
   }
@@ -757,12 +834,10 @@ namespace trezor{
   };
 
   void WebUsbTransport::open() {
-    const int interface = get_interface();
+    if (!pre_open()){
-    if (m_conn_count > 0){
-      MTRACE("Already opened, count: " << m_conn_count);
-      m_conn_count += 1;
       return;
     }
+    const int interface = get_interface();
 
 #define TREZOR_DESTROY_SESSION() do { libusb_exit(m_usb_session); m_usb_session = nullptr; } while(0)
 
@@ -840,21 +915,18 @@ namespace trezor{
       throw exc::DeviceAcquireException("Unable to claim libusb device");
     }
 
-    m_conn_count = 1;
+    m_open_counter = 1;
     m_proto->session_begin(*this);
     
 #undef TREZOR_DESTROY_SESSION
   };
 
   void WebUsbTransport::close() {
-    m_conn_count -= 1;
+    if (!pre_close()){
-
+      return;
-    if (m_conn_count < 0){
+    }
-      MERROR("Close counter is negative: " << m_conn_count);
-
-    } else if (m_conn_count == 0){
-      MTRACE("Closing webusb device");
 
+    MTRACE("Closing Trezor:WebUsbTransport");
     m_proto->session_end(*this);
 
     int r = libusb_release_interface(m_usb_device_handle, get_interface());
@@ -872,13 +944,26 @@ namespace trezor{
       libusb_exit(m_usb_session);
       m_usb_session = nullptr;
     }
-    }
   };
 
+  std::shared_ptr<Transport> WebUsbTransport::find_debug() {
+#ifdef WITH_TREZOR_DEBUGGING
+    require_device();
+    auto t = std::make_shared<WebUsbTransport>(boost::make_optional(m_usb_device_desc.get()));
+    t->m_bus_id = m_bus_id;
+    t->m_device_addr = m_device_addr;
+    t->m_port_numbers = m_port_numbers;
+    t->m_debug_mode = true;
+    return t;
+#else
+      MINFO("Debug link is disabled in production");
+      return nullptr;
+#endif
+    }
 
   int WebUsbTransport::get_interface() const{
     const int INTERFACE_NORMAL = 0;
-#ifdef WITH_TREZOR_DEBUG
+#ifdef WITH_TREZOR_DEBUGGING
     const int INTERFACE_DEBUG = 1;
     return m_debug_mode ? INTERFACE_DEBUG : INTERFACE_NORMAL;
 #else
@@ -888,7 +973,7 @@ namespace trezor{
 
   unsigned char WebUsbTransport::get_endpoint() const{
     const unsigned char ENDPOINT_NORMAL = 1;
-#ifdef WITH_TREZOR_DEBUG
+#ifdef WITH_TREZOR_DEBUGGING
     const unsigned char ENDPOINT_DEBUG = 2;
     return m_debug_mode ? ENDPOINT_DEBUG : ENDPOINT_NORMAL;
 #else
@@ -1047,4 +1132,3 @@ namespace trezor{
 }
 }
 
-

src/device_trezor/trezor/transport.hpp

@@ -62,6 +62,8 @@ namespace trezor {
 
   const std::string DEFAULT_BRIDGE = "127.0.0.1:21325";
 
+  uint64_t pack_version(uint32_t major, uint32_t minor=0, uint32_t patch=0);
+
   // Base HTTP comm serialization.
   bool t_serialize(const std::string & in, std::string & out);
   bool t_serialize(const json_val & in, std::string & out);
@@ -134,7 +136,7 @@ namespace trezor {
 
   class Transport {
   public:
-    Transport() = default;
+    Transport();
     virtual ~Transport() = default;
 
     virtual bool ping() { return false; };
@@ -144,10 +146,16 @@ namespace trezor {
     virtual void close(){};
     virtual void write(const google::protobuf::Message & req) =0;
     virtual void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) =0;
+    virtual std::shared_ptr<Transport> find_debug() { return nullptr; };
 
     virtual void write_chunk(const void * buff, size_t size) { };
     virtual size_t read_chunk(void * buff, size_t size) { return 0; };
     virtual std::ostream& dump(std::ostream& o) const { return o << "Transport<>"; }
+  protected:
+    long m_open_counter;
+
+    virtual bool pre_open();
+    virtual bool pre_close();
   };
 
   // Bridge transport
@@ -212,6 +220,7 @@ namespace trezor {
 
     void open() override;
     void close() override;
+    std::shared_ptr<Transport> find_debug() override;
 
     void write(const google::protobuf::Message &req) override;
     void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override;
@@ -259,6 +268,7 @@ namespace trezor {
 
     void open() override;
     void close() override;
+    std::shared_ptr<Transport> find_debug() override;
 
     void write(const google::protobuf::Message &req) override;
     void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override;
@@ -274,7 +284,6 @@ namespace trezor {
     int get_interface() const;
     unsigned char get_endpoint() const;
 
-    int m_conn_count;
     std::shared_ptr<Protocol> m_proto;
 
     libusb_context        *m_usb_session;
@@ -285,7 +294,7 @@ namespace trezor {
     int m_bus_id;
     int m_device_addr;
 
-#ifdef WITH_TREZOR_DEBUG
+#ifdef WITH_TREZOR_DEBUGGING
     bool m_debug_mode;
 #endif
   };
@@ -309,7 +318,7 @@ namespace trezor {
   /**
    * Transforms path to the particular transport
    */
-  template<class t_transport>
+  template<class t_transport=Transport>
   std::shared_ptr<t_transport> transport_typed(const std::string & path){
     auto t = transport(path);
     if (!t){
@@ -362,7 +371,7 @@ namespace trezor {
    * @throws UnexpectedMessageException if the response message type is different than expected.
    * Exception contains message type and the message itself.
    */
-  template<class t_message>
+  template<class t_message=google::protobuf::Message>
   std::shared_ptr<t_message>
       exchange_message(Transport & transport, const google::protobuf::Message & req,
                        boost::optional<messages::MessageType> resp_type = boost::none)

src/net/CMakeLists.txt

@@ -30,5 +30,5 @@ set(net_sources error.cpp parse.cpp socks.cpp tor_address.cpp)
 set(net_headers error.h parse.h socks.h tor_address.h)
 
 monero_add_library(net ${net_sources} ${net_headers})
-target_link_libraries(net epee ${Boost_ASIO_LIBRARY})
+target_link_libraries(net common epee ${Boost_ASIO_LIBRARY})
 

src/wallet/wallet2.h

@@ -67,6 +67,7 @@
 #define MONERO_DEFAULT_LOG_CATEGORY "wallet.wallet2"
 
 class Serialization_portability_wallet_Test;
+class wallet_accessor_test;
 
 namespace tools
 {
@@ -171,6 +172,7 @@ namespace tools
   class wallet2
   {
     friend class ::Serialization_portability_wallet_Test;
+    friend class ::wallet_accessor_test;
     friend class wallet_keys_unlocker;
     friend class wallet_device_callback;
   public:

tests/CMakeLists.txt

@@ -91,6 +91,10 @@ if (BUILD_GUI_DEPS)
   add_subdirectory(libwallet_api_tests)
 endif()
 
+if (TREZOR_DEBUG)
+  add_subdirectory(trezor)
+endif()
+
 # add_subdirectory(daemon_tests)
 
 set(hash_targets_sources

tests/core_tests/CMakeLists.txt

@@ -42,7 +42,8 @@ set(core_tests_sources
   tx_validation.cpp
   v2_tests.cpp
   rct.cpp
-  bulletproofs.cpp)
+  bulletproofs.cpp
+  wallet_tools.cpp)
 
 set(core_tests_headers
   block_reward.h
@@ -60,7 +61,8 @@ set(core_tests_headers
   tx_validation.h
   v2_tests.h
   rct.h
-  bulletproofs.h)
+  bulletproofs.h
+  wallet_tools.h)
 
 add_executable(core_tests
   ${core_tests_sources}
@@ -73,6 +75,7 @@ target_link_libraries(core_tests
     version
     epee
     device
+    wallet
     ${CMAKE_THREAD_LIBS_INIT}
     ${EXTRA_LIBRARIES})
 enable_stack_trace(core_tests)

tests/core_tests/chaingen.cpp

@@ -31,6 +31,11 @@
 #include <vector>
 #include <iostream>
 #include <sstream>
+#include <algorithm>
+#include <array>
+#include <random>
+#include <sstream>
+#include <fstream>
 
 #include "include_base_utils.h"
 
@@ -105,10 +110,11 @@ void test_generator::add_block(const cryptonote::block& blk, size_t txs_weight,
 
 bool test_generator::construct_block(cryptonote::block& blk, uint64_t height, const crypto::hash& prev_id,
                                      const cryptonote::account_base& miner_acc, uint64_t timestamp, uint64_t already_generated_coins,
-                                     std::vector<size_t>& block_weights, const std::list<cryptonote::transaction>& tx_list)
+                                     std::vector<size_t>& block_weights, const std::list<cryptonote::transaction>& tx_list,
+                                     const boost::optional<uint8_t>& hf_ver)
 {
-  blk.major_version = CURRENT_BLOCK_MAJOR_VERSION;
+  blk.major_version = hf_ver ? hf_ver.get() : CURRENT_BLOCK_MAJOR_VERSION;
-  blk.minor_version = CURRENT_BLOCK_MINOR_VERSION;
+  blk.minor_version = hf_ver ? hf_ver.get() : CURRENT_BLOCK_MINOR_VERSION;
   blk.timestamp = timestamp;
   blk.prev_id = prev_id;
 
@@ -135,7 +141,7 @@ bool test_generator::construct_block(cryptonote::block& blk, uint64_t height, co
   size_t target_block_weight = txs_weight + get_transaction_weight(blk.miner_tx);
   while (true)
   {
-    if (!construct_miner_tx(height, misc_utils::median(block_weights), already_generated_coins, target_block_weight, total_fee, miner_acc.get_keys().m_account_address, blk.miner_tx, blobdata(), 10))
+    if (!construct_miner_tx(height, misc_utils::median(block_weights), already_generated_coins, target_block_weight, total_fee, miner_acc.get_keys().m_account_address, blk.miner_tx, blobdata(), 10, hf_ver ? hf_ver.get() : 1))
       return false;
 
     size_t actual_block_weight = txs_weight + get_transaction_weight(blk.miner_tx);
@@ -180,10 +186,10 @@ bool test_generator::construct_block(cryptonote::block& blk, uint64_t height, co
 
   // Nonce search...
   blk.nonce = 0;
-  while (!miner::find_nonce_for_given_block(blk, get_test_difficulty(), height))
+  while (!miner::find_nonce_for_given_block(blk, get_test_difficulty(hf_ver), height))
     blk.timestamp++;
 
-  add_block(blk, txs_weight, block_weights, already_generated_coins);
+  add_block(blk, txs_weight, block_weights, already_generated_coins, hf_ver ? hf_ver.get() : 1);
 
   return true;
 }
@@ -197,17 +203,18 @@ bool test_generator::construct_block(cryptonote::block& blk, const cryptonote::a
 
 bool test_generator::construct_block(cryptonote::block& blk, const cryptonote::block& blk_prev,
                                      const cryptonote::account_base& miner_acc,
-                                     const std::list<cryptonote::transaction>& tx_list/* = std::list<cryptonote::transaction>()*/)
+                                     const std::list<cryptonote::transaction>& tx_list/* = std::list<cryptonote::transaction>()*/,
+                                     const boost::optional<uint8_t>& hf_ver)
 {
   uint64_t height = boost::get<txin_gen>(blk_prev.miner_tx.vin.front()).height + 1;
   crypto::hash prev_id = get_block_hash(blk_prev);
   // Keep difficulty unchanged
-  uint64_t timestamp = blk_prev.timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN;
+  uint64_t timestamp = blk_prev.timestamp + current_difficulty_window(hf_ver); // DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN;
   uint64_t already_generated_coins = get_already_generated_coins(prev_id);
   std::vector<size_t> block_weights;
   get_last_n_block_weights(block_weights, prev_id, CRYPTONOTE_REWARD_BLOCKS_WINDOW);
 
-  return construct_block(blk, height, prev_id, miner_acc, timestamp, already_generated_coins, block_weights, tx_list);
+  return construct_block(blk, height, prev_id, miner_acc, timestamp, already_generated_coins, block_weights, tx_list, hf_ver);
 }
 
 bool test_generator::construct_block_manually(block& blk, const block& prev_block, const account_base& miner_acc,
@@ -244,7 +251,7 @@ bool test_generator::construct_block_manually(block& blk, const block& prev_bloc
 
   //blk.tree_root_hash = get_tx_tree_hash(blk);
 
-  difficulty_type a_diffic = actual_params & bf_diffic ? diffic : get_test_difficulty();
+  difficulty_type a_diffic = actual_params & bf_diffic ? diffic : get_test_difficulty(hf_version);
   fill_nonce(blk, a_diffic, height);
 
   add_block(blk, txs_weight, block_weights, already_generated_coins, hf_version);
@@ -259,49 +266,6 @@ bool test_generator::construct_block_manually_tx(cryptonote::block& blk, const c
   return construct_block_manually(blk, prev_block, miner_acc, bf_tx_hashes, 0, 0, 0, crypto::hash(), 0, transaction(), tx_hashes, txs_weight);
 }
 
-
-struct output_index {
-    const cryptonote::txout_target_v out;
-    uint64_t amount;
-    size_t blk_height; // block height
-    size_t tx_no; // index of transaction in block
-    size_t out_no; // index of out in transaction
-    size_t idx;
-    bool spent;
-    const cryptonote::block *p_blk;
-    const cryptonote::transaction *p_tx;
-
-    output_index(const cryptonote::txout_target_v &_out, uint64_t _a, size_t _h, size_t tno, size_t ono, const cryptonote::block *_pb, const cryptonote::transaction *_pt)
-        : out(_out), amount(_a), blk_height(_h), tx_no(tno), out_no(ono), idx(0), spent(false), p_blk(_pb), p_tx(_pt) { }
-
-    output_index(const output_index &other)
-        : out(other.out), amount(other.amount), blk_height(other.blk_height), tx_no(other.tx_no), out_no(other.out_no), idx(other.idx), spent(other.spent), p_blk(other.p_blk), p_tx(other.p_tx) {  }
-
-    const std::string toString() const {
-        std::stringstream ss;
-
-        ss << "output_index{blk_height=" << blk_height
-           << " tx_no=" << tx_no
-           << " out_no=" << out_no
-           << " amount=" << amount
-           << " idx=" << idx
-           << " spent=" << spent
-           << "}";
-
-        return ss.str();
-    }
-
-    output_index& operator=(const output_index& other)
-    {
-      new(this) output_index(other);
-      return *this;
-    }
-};
-
-typedef std::map<uint64_t, std::vector<size_t> > map_output_t;
-typedef std::map<uint64_t, std::vector<output_index> > map_output_idx_t;
-typedef pair<uint64_t, size_t>  outloc_t;
-
 namespace
 {
   uint64_t get_inputs_amount(const vector<tx_source_entry> &s)
@@ -339,6 +303,9 @@ bool init_output_indices(map_output_idx_t& outs, std::map<uint64_t, std::vector<
                 const tx_out &out = tx.vout[j];
 
                 output_index oi(out.target, out.amount, boost::get<txin_gen>(*blk.miner_tx.vin.begin()).height, i, j, &blk, vtx[i]);
+                oi.set_rct(tx.version == 2);
+                oi.unlock_time = tx.unlock_time;
+                oi.is_coin_base = i == 0;
 
                 if (2 == out.target.which()) { // out_to_key
                     outs[out.amount].push_back(oi);
@@ -416,8 +383,9 @@ bool fill_output_entries(std::vector<output_index>& out_indices, size_t sender_o
 
     if (append)
     {
+      rct::key comm = oi.commitment();
       const txout_to_key& otk = boost::get<txout_to_key>(oi.out);
-      output_entries.push_back(tx_source_entry::output_entry(oi.idx, rct::ctkey({rct::pk2rct(otk.key), rct::identity()})));
+      output_entries.push_back(tx_source_entry::output_entry(oi.idx, rct::ctkey({rct::pk2rct(otk.key), comm})));
     }
   }
 
@@ -452,6 +420,8 @@ bool fill_tx_sources(std::vector<tx_source_entry>& sources, const std::vector<te
             const output_index& oi = outs[o.first][sender_out];
             if (oi.spent)
                 continue;
+            if (oi.rct)
+                continue;
 
             cryptonote::tx_source_entry ts;
             ts.amount = oi.amount;
@@ -463,6 +433,11 @@ bool fill_tx_sources(std::vector<tx_source_entry>& sources, const std::vector<te
 
             ts.real_output = realOutput;
             ts.rct = false;
+            ts.mask = rct::identity();  // non-rct has identity mask by definition
+
+            rct::key comm = rct::zeroCommit(ts.amount);
+            for(auto & ot : ts.outputs)
+              ot.second.mask = comm;
 
             sources.push_back(ts);
 
@@ -477,14 +452,327 @@ bool fill_tx_sources(std::vector<tx_source_entry>& sources, const std::vector<te
     return sources_found;
 }
 
-bool fill_tx_destination(tx_destination_entry &de, const cryptonote::account_base &to, uint64_t amount) {
+bool fill_tx_destination(tx_destination_entry &de, const cryptonote::account_public_address &to, uint64_t amount) {
-    de.addr = to.get_keys().m_account_address;
+    de.addr = to;
     de.amount = amount;
     return true;
 }
 
+map_txid_output_t::iterator block_tracker::find_out(const crypto::hash &txid, size_t out)
+{
+  return find_out(std::make_pair(txid, out));
+}
+
+map_txid_output_t::iterator block_tracker::find_out(const output_hasher &id)
+{
+  return m_map_outs.find(id);
+}
+
+void block_tracker::process(const std::vector<cryptonote::block>& blockchain, const map_hash2tx_t& mtx)
+{
+  std::vector<const cryptonote::block*> blks;
+  blks.reserve(blockchain.size());
+
+  BOOST_FOREACH (const block& blk, blockchain) {
+    auto hsh = get_block_hash(blk);
+    auto it = m_blocks.find(hsh);
+    if (it == m_blocks.end()){
+      m_blocks[hsh] = blk;
+    }
+
+    blks.push_back(&m_blocks[hsh]);
+  }
+
+  process(blks, mtx);
+}
+
+void block_tracker::process(const std::vector<const cryptonote::block*>& blockchain, const map_hash2tx_t& mtx)
+{
+  BOOST_FOREACH (const block* blk, blockchain) {
+    vector<const transaction*> vtx;
+    vtx.push_back(&(blk->miner_tx));
+
+    BOOST_FOREACH(const crypto::hash &h, blk->tx_hashes) {
+      const map_hash2tx_t::const_iterator cit = mtx.find(h);
+      CHECK_AND_ASSERT_THROW_MES(mtx.end() != cit, "block contains an unknown tx hash");
+      vtx.push_back(cit->second);
+    }
+
+    for (size_t i = 0; i < vtx.size(); i++) {
+      process(blk, vtx[i], i);
+    }
+  }
+}
+
+void block_tracker::process(const block* blk, const transaction * tx, size_t i)
+{
+  for (size_t j = 0; j < tx->vout.size(); ++j) {
+    const tx_out &out = tx->vout[j];
+
+    if (typeid(cryptonote::txout_to_key) != out.target.type()) { // out_to_key
+      continue;
+    }
+
+    const uint64_t rct_amount = tx->version == 2 ? 0 : out.amount;
+    const output_hasher hid = std::make_pair(tx->hash, j);
+    auto it = find_out(hid);
+    if (it != m_map_outs.end()){
+      continue;
+    }
+
+    output_index oi(out.target, out.amount, boost::get<txin_gen>(blk->miner_tx.vin.front()).height, i, j, blk, tx);
+    oi.set_rct(tx->version == 2);
+    oi.idx = m_outs[rct_amount].size();
+    oi.unlock_time = tx->unlock_time;
+    oi.is_coin_base = tx->vin.size() == 1 && tx->vin.back().type() == typeid(cryptonote::txin_gen);
+
+    m_outs[rct_amount].push_back(oi);
+    m_map_outs.insert({hid, oi});
+  }
+}
+
+void block_tracker::global_indices(const cryptonote::transaction *tx, std::vector<uint64_t> &indices)
+{
+  indices.clear();
+
+  for(size_t j=0; j < tx->vout.size(); ++j){
+    auto it = find_out(tx->hash, j);
+    if (it != m_map_outs.end()){
+      indices.push_back(it->second.idx);
+    }
+  }
+}
+
+void block_tracker::get_fake_outs(size_t num_outs, uint64_t amount, uint64_t global_index, uint64_t cur_height, std::vector<get_outs_entry> &outs){
+  auto & vct = m_outs[amount];
+  const size_t n_outs = vct.size();
+
+  std::set<size_t> used;
+  std::vector<size_t> choices;
+  choices.resize(n_outs);
+  for(size_t i=0; i < n_outs; ++i) choices[i] = i;
+  shuffle(choices.begin(), choices.end(), std::default_random_engine(crypto::rand<unsigned>()));
+
+  size_t n_iters = 0;
+  ssize_t idx = -1;
+  outs.reserve(num_outs);
+  while(outs.size() < num_outs){
+    n_iters += 1;
+    idx = (idx + 1) % n_outs;
+    size_t oi_idx = choices[(size_t)idx];
+    CHECK_AND_ASSERT_THROW_MES((n_iters / n_outs) <= outs.size(), "Fake out pick selection problem");
+
+    auto & oi = vct[oi_idx];
+    if (oi.idx == global_index)
+      continue;
+    if (oi.out.type() != typeid(cryptonote::txout_to_key))
+      continue;
+    if (oi.unlock_time > cur_height)
+      continue;
+    if (used.find(oi_idx) != used.end())
+      continue;
+
+    rct::key comm = oi.commitment();
+    auto out = boost::get<txout_to_key>(oi.out);
+    auto item = std::make_tuple(oi.idx, out.key, comm);
+    outs.push_back(item);
+    used.insert(oi_idx);
+  }
+}
+
+std::string block_tracker::dump_data()
+{
+  ostringstream ss;
+  for (auto &m_out : m_outs)
+  {
+    auto & vct = m_out.second;
+    ss << m_out.first << " => |vector| = " << vct.size() << '\n';
+
+    for (const auto & oi : vct)
+    {
+      auto out = boost::get<txout_to_key>(oi.out);
+
+      ss << "    idx: " << oi.idx
+      << ", rct: " << oi.rct
+      << ", xmr: " << oi.amount
+      << ", key: " << dump_keys(out.key.data)
+      << ", msk: " << dump_keys(oi.comm.bytes)
+      << ", txid: " << dump_keys(oi.p_tx->hash.data)
+      << '\n';
+    }
+  }
+
+  return ss.str();
+}
+
+void block_tracker::dump_data(const std::string & fname)
+{
+  ofstream myfile;
+  myfile.open (fname);
+  myfile << dump_data();
+  myfile.close();
+}
+
+std::string dump_data(const cryptonote::transaction &tx)
+{
+  ostringstream ss;
+  ss << "msg: " << dump_keys(tx.rct_signatures.message.bytes)
+     << ", vin: ";
+
+  for(auto & in : tx.vin){
+    if (typeid(txin_to_key) == in.type()){
+      auto tk = boost::get<txin_to_key>(in);
+      std::vector<uint64_t> full_off;
+      int64_t last = -1;
+
+      ss << " i: " << tk.amount << " [";
+      for(auto ix : tk.key_offsets){
+        ss << ix << ", ";
+        if (last == -1){
+          last = ix;
+          full_off.push_back(ix);
+        } else {
+          last += ix;
+          full_off.push_back((uint64_t)last);
+        }
+      }
+
+      ss << "], full: [";
+      for(auto ix : full_off){
+        ss << ix << ", ";
+      }
+      ss << "]; ";
+
+    } else if (typeid(txin_gen) == in.type()){
+      ss << " h: " << boost::get<txin_gen>(in).height << ", ";
+    } else {
+      ss << " ?, ";
+    }
+  }
+
+  ss << ", mixring: \n";
+  for (const auto & row : tx.rct_signatures.mixRing){
+    for(auto cur : row){
+      ss << "    (" << dump_keys(cur.dest.bytes) << ", " << dump_keys(cur.mask.bytes) << ")\n ";
+    }
+    ss << "; ";
+  }
+
+  return ss.str();
+}
+
+cryptonote::account_public_address get_address(const var_addr_t& inp)
+{
+  if (typeid(cryptonote::account_public_address) == inp.type()){
+    return boost::get<cryptonote::account_public_address>(inp);
+  } else if(typeid(cryptonote::account_keys) == inp.type()){
+    return boost::get<cryptonote::account_keys>(inp).m_account_address;
+  } else if (typeid(cryptonote::account_base) == inp.type()){
+    return boost::get<cryptonote::account_base>(inp).get_keys().m_account_address;
+  } else if (typeid(cryptonote::tx_destination_entry) == inp.type()){
+    return boost::get<cryptonote::tx_destination_entry>(inp).addr;
+  } else {
+    throw std::runtime_error("Unexpected type");
+  }
+}
+
+cryptonote::account_public_address get_address(const cryptonote::account_public_address& inp)
+{
+  return inp;
+}
+
+cryptonote::account_public_address get_address(const cryptonote::account_keys& inp)
+{
+  return inp.m_account_address;
+}
+
+cryptonote::account_public_address get_address(const cryptonote::account_base& inp)
+{
+  return inp.get_keys().m_account_address;
+}
+
+cryptonote::account_public_address get_address(const cryptonote::tx_destination_entry& inp)
+{
+  return inp.addr;
+}
+
+uint64_t sum_amount(const std::vector<tx_destination_entry>& destinations)
+{
+  uint64_t amount = 0;
+  for(auto & cur : destinations){
+    amount += cur.amount;
+  }
+
+  return amount;
+}
+
+uint64_t sum_amount(const std::vector<cryptonote::tx_source_entry>& sources)
+{
+  uint64_t amount = 0;
+  for(auto & cur : sources){
+    amount += cur.amount;
+  }
+
+  return amount;
+}
+
+void fill_tx_destinations(const var_addr_t& from, const std::vector<tx_destination_entry>& dests,
+                          uint64_t fee,
+                          const std::vector<tx_source_entry> &sources,
+                          std::vector<tx_destination_entry>& destinations,
+                          bool always_change)
+
+{
+  destinations.clear();
+  uint64_t amount = sum_amount(dests);
+  std::copy(dests.begin(), dests.end(), std::back_inserter(destinations));
+
+  tx_destination_entry de_change;
+  uint64_t cache_back = get_inputs_amount(sources) - (amount + fee);
+
+  if (cache_back > 0 || always_change) {
+    if (!fill_tx_destination(de_change, get_address(from), cache_back <= 0 ? 0 : cache_back))
+      throw std::runtime_error("couldn't fill transaction cache back destination");
+    destinations.push_back(de_change);
+  }
+}
+
+void fill_tx_destinations(const var_addr_t& from, const cryptonote::account_public_address& to,
+                          uint64_t amount, uint64_t fee,
+                          const std::vector<tx_source_entry> &sources,
+                          std::vector<tx_destination_entry>& destinations,
+                          std::vector<tx_destination_entry>& destinations_pure,
+                          bool always_change)
+{
+  destinations.clear();
+
+  tx_destination_entry de;
+  if (!fill_tx_destination(de, to, amount))
+    throw std::runtime_error("couldn't fill transaction destination");
+  destinations.push_back(de);
+  destinations_pure.push_back(de);
+
+  tx_destination_entry de_change;
+  uint64_t cache_back = get_inputs_amount(sources) - (amount + fee);
+
+  if (cache_back > 0 || always_change) {
+    if (!fill_tx_destination(de_change, get_address(from), cache_back <= 0 ? 0 : cache_back))
+      throw std::runtime_error("couldn't fill transaction cache back destination");
+    destinations.push_back(de_change);
+  }
+}
+
+void fill_tx_destinations(const var_addr_t& from, const cryptonote::account_public_address& to,
+                          uint64_t amount, uint64_t fee,
+                          const std::vector<tx_source_entry> &sources,
+                          std::vector<tx_destination_entry>& destinations, bool always_change)
+{
+  std::vector<tx_destination_entry> destinations_pure;
+  fill_tx_destinations(from, to, amount, fee, sources, destinations, destinations_pure, always_change);
+}
+
 void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& events, const block& blk_head,
-                                      const cryptonote::account_base& from, const cryptonote::account_base& to,
+                                      const cryptonote::account_base& from, const cryptonote::account_public_address& to,
                                       uint64_t amount, uint64_t fee, size_t nmix, std::vector<tx_source_entry>& sources,
                                       std::vector<tx_destination_entry>& destinations)
 {
@@ -494,19 +782,15 @@ void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& event
   if (!fill_tx_sources(sources, events, blk_head, from, amount + fee, nmix))
     throw std::runtime_error("couldn't fill transaction sources");
 
-  tx_destination_entry de;
+  fill_tx_destinations(from, to, amount, fee, sources, destinations, false);
-  if (!fill_tx_destination(de, to, amount))
-    throw std::runtime_error("couldn't fill transaction destination");
-  destinations.push_back(de);
-
-  tx_destination_entry de_change;
-  uint64_t cache_back = get_inputs_amount(sources) - (amount + fee);
-  if (0 < cache_back)
-  {
-    if (!fill_tx_destination(de_change, from, cache_back))
-      throw std::runtime_error("couldn't fill transaction cache back destination");
-    destinations.push_back(de_change);
 }
+
+void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& events, const block& blk_head,
+                                      const cryptonote::account_base& from, const cryptonote::account_base& to,
+                                      uint64_t amount, uint64_t fee, size_t nmix, std::vector<tx_source_entry>& sources,
+                                      std::vector<tx_destination_entry>& destinations)
+{
+  fill_tx_sources_and_destinations(events, blk_head, from, to.get_keys().m_account_address, amount, fee, nmix, sources, destinations);
 }
 
 void fill_nonce(cryptonote::block& blk, const difficulty_type& diffic, uint64_t height)
@@ -516,6 +800,32 @@ void fill_nonce(cryptonote::block& blk, const difficulty_type& diffic, uint64_t
     blk.timestamp++;
 }
 
+cryptonote::tx_destination_entry build_dst(const var_addr_t& to, bool is_subaddr, uint64_t amount)
+{
+  tx_destination_entry de;
+  de.amount = amount;
+  de.addr = get_address(to);
+  de.is_subaddress = is_subaddr;
+  return de;
+}
+
+std::vector<cryptonote::tx_destination_entry> build_dsts(const var_addr_t& to1, bool sub1, uint64_t am1)
+{
+  std::vector<cryptonote::tx_destination_entry> res;
+  res.push_back(build_dst(to1, sub1, am1));
+  return res;
+}
+
+std::vector<cryptonote::tx_destination_entry> build_dsts(std::initializer_list<dest_wrapper_t> inps)
+{
+  std::vector<cryptonote::tx_destination_entry> res;
+  res.reserve(inps.size());
+  for(auto & c : inps){
+    res.push_back(build_dst(c.addr, c.is_subaddr, c.amount));
+  }
+  return res;
+}
+
 bool construct_miner_tx_manually(size_t height, uint64_t already_generated_coins,
                                  const account_public_address& miner_address, transaction& tx, uint64_t fee,
                                  keypair* p_txkey/* = 0*/)
@@ -556,22 +866,70 @@ bool construct_miner_tx_manually(size_t height, uint64_t already_generated_coins
   return true;
 }
 
-bool construct_tx_to_key(const std::vector<test_event_entry>& events, cryptonote::transaction& tx, const block& blk_head,
+bool construct_tx_to_key(const std::vector<test_event_entry>& events, cryptonote::transaction& tx, const cryptonote::block& blk_head,
-                         const cryptonote::account_base& from, const cryptonote::account_base& to, uint64_t amount,
+                         const cryptonote::account_base& from, const var_addr_t& to, uint64_t amount,
-                         uint64_t fee, size_t nmix)
+                         uint64_t fee, size_t nmix, bool rct, rct::RangeProofType range_proof_type, int bp_version)
 {
   vector<tx_source_entry> sources;
   vector<tx_destination_entry> destinations;
-  fill_tx_sources_and_destinations(events, blk_head, from, to, amount, fee, nmix, sources, destinations);
+  fill_tx_sources_and_destinations(events, blk_head, from, get_address(to), amount, fee, nmix, sources, destinations);
 
-  return construct_tx(from.get_keys(), sources, destinations, from.get_keys().m_account_address, std::vector<uint8_t>(), tx, 0);
+  return construct_tx_rct(from.get_keys(), sources, destinations, from.get_keys().m_account_address, std::vector<uint8_t>(), tx, 0, rct, range_proof_type, bp_version);
+}
+
+bool construct_tx_to_key(const std::vector<test_event_entry>& events, cryptonote::transaction& tx, const cryptonote::block& blk_head,
+                         const cryptonote::account_base& from, std::vector<cryptonote::tx_destination_entry> destinations,
+                         uint64_t fee, size_t nmix, bool rct, rct::RangeProofType range_proof_type, int bp_version)
+{
+  vector<tx_source_entry> sources;
+  vector<tx_destination_entry> destinations_all;
+  uint64_t amount = sum_amount(destinations);
+
+  if (!fill_tx_sources(sources, events, blk_head, from, amount + fee, nmix))
+    throw std::runtime_error("couldn't fill transaction sources");
+
+  fill_tx_destinations(from, destinations, fee, sources, destinations_all, false);
+
+  return construct_tx_rct(from.get_keys(), sources, destinations_all, get_address(from), std::vector<uint8_t>(), tx, 0, rct, range_proof_type, bp_version);
+}
+
+bool construct_tx_to_key(cryptonote::transaction& tx,
+                         const cryptonote::account_base& from, const var_addr_t& to, uint64_t amount,
+                         std::vector<cryptonote::tx_source_entry> &sources,
+                         uint64_t fee, bool rct, rct::RangeProofType range_proof_type, int bp_version)
+{
+  vector<tx_destination_entry> destinations;
+  fill_tx_destinations(from, get_address(to), amount, fee, sources, destinations, rct);
+  return construct_tx_rct(from.get_keys(), sources, destinations, get_address(from), std::vector<uint8_t>(), tx, 0, rct, range_proof_type, bp_version);
+}
+
+bool construct_tx_to_key(cryptonote::transaction& tx,
+                         const cryptonote::account_base& from,
+                         const std::vector<cryptonote::tx_destination_entry>& destinations,
+                         std::vector<cryptonote::tx_source_entry> &sources,
+                         uint64_t fee, bool rct, rct::RangeProofType range_proof_type, int bp_version)
+{
+  vector<tx_destination_entry> all_destinations;
+  fill_tx_destinations(from, destinations, fee, sources, all_destinations, rct);
+  return construct_tx_rct(from.get_keys(), sources, all_destinations, get_address(from), std::vector<uint8_t>(), tx, 0, rct, range_proof_type, bp_version);
+}
+
+bool construct_tx_rct(const cryptonote::account_keys& sender_account_keys, std::vector<cryptonote::tx_source_entry>& sources, const std::vector<cryptonote::tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, cryptonote::transaction& tx, uint64_t unlock_time, bool rct, rct::RangeProofType range_proof_type, int bp_version)
+{
+  std::unordered_map<crypto::public_key, cryptonote::subaddress_index> subaddresses;
+  subaddresses[sender_account_keys.m_account_address.m_spend_public_key] = {0, 0};
+  crypto::secret_key tx_key;
+  std::vector<crypto::secret_key> additional_tx_keys;
+  std::vector<tx_destination_entry> destinations_copy = destinations;
+  rct::RCTConfig rct_config = {range_proof_type, bp_version};
+  return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, nullptr);
 }
 
 transaction construct_tx_with_fee(std::vector<test_event_entry>& events, const block& blk_head,
-                                  const account_base& acc_from, const account_base& acc_to, uint64_t amount, uint64_t fee)
+                                  const account_base& acc_from, const var_addr_t& to, uint64_t amount, uint64_t fee)
 {
   transaction tx;
-  construct_tx_to_key(events, tx, blk_head, acc_from, acc_to, amount, fee, 0);
+  construct_tx_to_key(events, tx, blk_head, acc_from, to, amount, fee, 0);
   events.push_back(tx);
   return tx;
 }
@@ -602,6 +960,24 @@ uint64_t get_balance(const cryptonote::account_base& addr, const std::vector<cry
     return res;
 }
 
+bool extract_hard_forks(const std::vector<test_event_entry>& events, v_hardforks_t& hard_forks)
+{
+  for(auto & ev : events)
+  {
+    if (typeid(event_replay_settings) == ev.type())
+    {
+      const auto & rep_settings = boost::get<event_replay_settings>(ev);
+      if (rep_settings.hard_forks)
+      {
+        const auto & hf = rep_settings.hard_forks.get();
+        std::copy(hf.begin(), hf.end(), std::back_inserter(hard_forks));
+      }
+    }
+  }
+
+  return !hard_forks.empty();
+}
+
 void get_confirmed_txs(const std::vector<cryptonote::block>& blockchain, const map_hash2tx_t& mtx, map_hash2tx_t& confirmed_txs)
 {
   std::unordered_set<crypto::hash> confirmed_hashes;
@@ -622,6 +998,74 @@ void get_confirmed_txs(const std::vector<cryptonote::block>& blockchain, const m
   }
 }
 
+bool trim_block_chain(std::vector<cryptonote::block>& blockchain, const crypto::hash& tail){
+  size_t cut = 0;
+  bool found = true;
+
+  for(size_t i = 0; i < blockchain.size(); ++i){
+    crypto::hash chash = get_block_hash(blockchain[i]);
+    if (chash == tail){
+      cut = i;
+      found = true;
+      break;
+    }
+  }
+
+  if (found && cut > 0){
+    blockchain.erase(blockchain.begin(), blockchain.begin() + cut);
+  }
+
+  return found;
+}
+
+bool trim_block_chain(std::vector<const cryptonote::block*>& blockchain, const crypto::hash& tail){
+  size_t cut = 0;
+  bool found = true;
+
+  for(size_t i = 0; i < blockchain.size(); ++i){
+    crypto::hash chash = get_block_hash(*blockchain[i]);
+    if (chash == tail){
+      cut = i;
+      found = true;
+      break;
+    }
+  }
+
+  if (found && cut > 0){
+    blockchain.erase(blockchain.begin(), blockchain.begin() + cut);
+  }
+
+  return found;
+}
+
+uint64_t num_blocks(const std::vector<test_event_entry>& events)
+{
+  uint64_t res = 0;
+  BOOST_FOREACH(const test_event_entry& ev, events)
+  {
+    if (typeid(block) == ev.type())
+    {
+      res += 1;
+    }
+  }
+
+  return res;
+}
+
+cryptonote::block get_head_block(const std::vector<test_event_entry>& events)
+{
+  for(auto it = events.rbegin(); it != events.rend(); ++it)
+  {
+    auto &ev = *it;
+    if (typeid(block) == ev.type())
+    {
+      return boost::get<block>(ev);
+    }
+  }
+
+  throw std::runtime_error("No block event");
+}
+
 bool find_block_chain(const std::vector<test_event_entry>& events, std::vector<cryptonote::block>& blockchain, map_hash2tx_t& mtx, const crypto::hash& head) {
     std::unordered_map<crypto::hash, const block*> block_index;
     BOOST_FOREACH(const test_event_entry& ev, events)
@@ -655,6 +1099,38 @@ bool find_block_chain(const std::vector<test_event_entry>& events, std::vector<c
     return b_success;
 }
 
+bool find_block_chain(const std::vector<test_event_entry>& events, std::vector<const cryptonote::block*>& blockchain, map_hash2tx_t& mtx, const crypto::hash& head) {
+    std::unordered_map<crypto::hash, const block*> block_index;
+    BOOST_FOREACH(const test_event_entry& ev, events)
+    {
+        if (typeid(block) == ev.type())
+        {
+            const block* blk = &boost::get<block>(ev);
+            block_index[get_block_hash(*blk)] = blk;
+        }
+        else if (typeid(transaction) == ev.type())
+        {
+            const transaction& tx = boost::get<transaction>(ev);
+            mtx[get_transaction_hash(tx)] = &tx;
+        }
+    }
+
+    bool b_success = false;
+    crypto::hash id = head;
+    for (auto it = block_index.find(id); block_index.end() != it; it = block_index.find(id))
+    {
+        blockchain.push_back(it->second);
+        id = it->second->prev_id;
+        if (null_hash == id)
+        {
+            b_success = true;
+            break;
+        }
+    }
+    reverse(blockchain.begin(), blockchain.end());
+    return b_success;
+}
+
 
 void test_chain_unit_base::register_callback(const std::string& cb_name, verify_callback cb)
 {

tests/core_tests/chaingen.h

@@ -37,8 +37,12 @@
 #include <boost/archive/binary_oarchive.hpp>
 #include <boost/archive/binary_iarchive.hpp>
 #include <boost/program_options.hpp>
+#include <boost/optional.hpp>
 #include <boost/serialization/vector.hpp>
 #include <boost/serialization/variant.hpp>
+#include <boost/serialization/optional.hpp>
+#include <boost/serialization/unordered_map.hpp>
+#include <boost/functional/hash.hpp>
 
 #include "include_base_utils.h"
 #include "common/boost_serialization_helper.h"
@@ -129,13 +133,32 @@ private:
   }
 };
 
+typedef std::vector<std::pair<uint8_t, uint64_t>> v_hardforks_t;
+struct event_replay_settings
+{
+  boost::optional<v_hardforks_t> hard_forks;
+
+  event_replay_settings() = default;
+
+private:
+  friend class boost::serialization::access;
+
+  template<class Archive>
+  void serialize(Archive & ar, const unsigned int /*version*/)
+  {
+    ar & hard_forks;
+  }
+};
+
+
 VARIANT_TAG(binary_archive, callback_entry, 0xcb);
 VARIANT_TAG(binary_archive, cryptonote::account_base, 0xcc);
 VARIANT_TAG(binary_archive, serialized_block, 0xcd);
 VARIANT_TAG(binary_archive, serialized_transaction, 0xce);
 VARIANT_TAG(binary_archive, event_visitor_settings, 0xcf);
+VARIANT_TAG(binary_archive, event_replay_settings, 0xda);
 
-typedef boost::variant<cryptonote::block, cryptonote::transaction, std::vector<cryptonote::transaction>, cryptonote::account_base, callback_entry, serialized_block, serialized_transaction, event_visitor_settings> test_event_entry;
+typedef boost::variant<cryptonote::block, cryptonote::transaction, std::vector<cryptonote::transaction>, cryptonote::account_base, callback_entry, serialized_block, serialized_transaction, event_visitor_settings, event_replay_settings> test_event_entry;
 typedef std::unordered_map<crypto::hash, const cryptonote::transaction*> map_hash2tx_t;
 
 class test_chain_unit_base
@@ -173,6 +196,17 @@ public:
     crypto::hash prev_id;
     uint64_t already_generated_coins;
     size_t block_weight;
+
+  private:
+    friend class boost::serialization::access;
+
+    template<class Archive>
+    void serialize(Archive & ar, const unsigned int /*version*/)
+    {
+      ar & prev_id;
+      ar & already_generated_coins;
+      ar & block_weight;
+    }
   };
 
   enum block_fields
@@ -189,6 +223,8 @@ public:
     bf_hf_version= 1 << 8
   };
 
+  test_generator() {}
+  test_generator(const test_generator &other): m_blocks_info(other.m_blocks_info) {}
   void get_block_chain(std::vector<block_info>& blockchain, const crypto::hash& head, size_t n) const;
   void get_last_n_block_weights(std::vector<size_t>& block_weights, const crypto::hash& head, size_t n) const;
   uint64_t get_already_generated_coins(const crypto::hash& blk_id) const;
@@ -198,10 +234,12 @@ public:
     uint8_t hf_version = 1);
   bool construct_block(cryptonote::block& blk, uint64_t height, const crypto::hash& prev_id,
     const cryptonote::account_base& miner_acc, uint64_t timestamp, uint64_t already_generated_coins,
-    std::vector<size_t>& block_weights, const std::list<cryptonote::transaction>& tx_list);
+    std::vector<size_t>& block_weights, const std::list<cryptonote::transaction>& tx_list,
+    const boost::optional<uint8_t>& hf_ver = boost::none);
   bool construct_block(cryptonote::block& blk, const cryptonote::account_base& miner_acc, uint64_t timestamp);
   bool construct_block(cryptonote::block& blk, const cryptonote::block& blk_prev, const cryptonote::account_base& miner_acc,
-    const std::list<cryptonote::transaction>& tx_list = std::list<cryptonote::transaction>());
+    const std::list<cryptonote::transaction>& tx_list = std::list<cryptonote::transaction>(),
+    const boost::optional<uint8_t>& hf_ver = boost::none);
 
   bool construct_block_manually(cryptonote::block& blk, const cryptonote::block& prev_block,
     const cryptonote::account_base& miner_acc, int actual_params = bf_none, uint8_t major_ver = 0,
@@ -214,30 +252,241 @@ public:
 
 private:
   std::unordered_map<crypto::hash, block_info> m_blocks_info;
+
+  friend class boost::serialization::access;
+
+  template<class Archive>
+  void serialize(Archive & ar, const unsigned int /*version*/)
+  {
+    ar & m_blocks_info;
+  }
 };
 
-inline cryptonote::difficulty_type get_test_difficulty() {return 1;}
+template<typename T>
+std::string dump_keys(T * buff32)
+{
+  std::ostringstream ss;
+  char buff[10];
+
+  ss << "[";
+  for(int i = 0; i < 32; i++)
+  {
+    snprintf(buff, 10, "0x%02x", ((uint8_t)buff32[i] & 0xff));
+    ss << buff;
+    if (i < 31)
+      ss << ",";
+  }
+  ss << "]";
+  return ss.str();
+}
+
+struct output_index {
+  const cryptonote::txout_target_v out;
+  uint64_t amount;
+  size_t blk_height; // block height
+  size_t tx_no; // index of transaction in block
+  size_t out_no; // index of out in transaction
+  size_t idx;
+  uint64_t unlock_time;
+  bool is_coin_base;
+  bool spent;
+  bool rct;
+  rct::key comm;
+  const cryptonote::block *p_blk;
+  const cryptonote::transaction *p_tx;
+
+  output_index(const cryptonote::txout_target_v &_out, uint64_t _a, size_t _h, size_t tno, size_t ono, const cryptonote::block *_pb, const cryptonote::transaction *_pt)
+      : out(_out), amount(_a), blk_height(_h), tx_no(tno), out_no(ono), idx(0), unlock_time(0),
+      is_coin_base(false), spent(false), rct(false), p_blk(_pb), p_tx(_pt)
+  {
+
+  }
+
+  output_index(const output_index &other)
+      : out(other.out), amount(other.amount), blk_height(other.blk_height), tx_no(other.tx_no), rct(other.rct),
+      out_no(other.out_no), idx(other.idx), unlock_time(other.unlock_time), is_coin_base(other.is_coin_base),
+      spent(other.spent), comm(other.comm), p_blk(other.p_blk), p_tx(other.p_tx) {  }
+
+  void set_rct(bool arct) {
+    rct = arct;
+    if (rct &&  p_tx->rct_signatures.outPk.size() > out_no)
+      comm = p_tx->rct_signatures.outPk[out_no].mask;
+    else
+      comm = rct::commit(amount, rct::identity());
+  }
+
+  rct::key commitment() const {
+    return comm;
+  }
+
+  const std::string toString() const {
+    std::stringstream ss;
+
+    ss << "output_index{blk_height=" << blk_height
+       << " tx_no=" << tx_no
+       << " out_no=" << out_no
+       << " amount=" << amount
+       << " idx=" << idx
+       << " unlock_time=" << unlock_time
+       << " spent=" << spent
+       << " is_coin_base=" << is_coin_base
+       << " rct=" << rct
+       << " comm=" << dump_keys(comm.bytes)
+       << "}";
+
+    return ss.str();
+  }
+
+  output_index& operator=(const output_index& other)
+  {
+    new(this) output_index(other);
+    return *this;
+  }
+};
+
+typedef std::tuple<uint64_t, crypto::public_key, rct::key> get_outs_entry;
+typedef std::pair<crypto::hash, size_t> output_hasher;
+typedef boost::hash<output_hasher> output_hasher_hasher;
+typedef std::map<uint64_t, std::vector<size_t> > map_output_t;
+typedef std::map<uint64_t, std::vector<output_index> > map_output_idx_t;
+typedef std::unordered_map<crypto::hash, cryptonote::block> map_block_t;
+typedef std::unordered_map<output_hasher, output_index, output_hasher_hasher> map_txid_output_t;
+typedef std::unordered_map<crypto::public_key, cryptonote::subaddress_index> subaddresses_t;
+typedef std::pair<uint64_t, size_t>  outloc_t;
+
+typedef boost::variant<cryptonote::account_public_address, cryptonote::account_keys, cryptonote::account_base, cryptonote::tx_destination_entry> var_addr_t;
+typedef struct {
+  const var_addr_t addr;
+  bool is_subaddr;
+  uint64_t amount;
+} dest_wrapper_t;
+
+// Daemon functionality
+class block_tracker
+{
+public:
+  map_output_idx_t m_outs;
+  map_txid_output_t m_map_outs;  // mapping (txid, out) -> output_index
+  map_block_t m_blocks;
+
+  block_tracker() = default;
+  block_tracker(const block_tracker &bt): m_outs(bt.m_outs), m_map_outs(bt.m_map_outs), m_blocks(bt.m_blocks) {};
+  map_txid_output_t::iterator find_out(const crypto::hash &txid, size_t out);
+  map_txid_output_t::iterator find_out(const output_hasher &id);
+  void process(const std::vector<cryptonote::block>& blockchain, const map_hash2tx_t& mtx);
+  void process(const std::vector<const cryptonote::block*>& blockchain, const map_hash2tx_t& mtx);
+  void process(const cryptonote::block* blk, const cryptonote::transaction * tx, size_t i);
+  void global_indices(const cryptonote::transaction *tx, std::vector<uint64_t> &indices);
+  void get_fake_outs(size_t num_outs, uint64_t amount, uint64_t global_index, uint64_t cur_height, std::vector<get_outs_entry> &outs);
+
+  std::string dump_data();
+  void dump_data(const std::string & fname);
+
+private:
+  friend class boost::serialization::access;
+
+  template<class Archive>
+  void serialize(Archive & ar, const unsigned int /*version*/)
+  {
+    ar & m_outs;
+    ar & m_map_outs;
+    ar & m_blocks;
+  }
+};
+
+std::string dump_data(const cryptonote::transaction &tx);
+cryptonote::account_public_address get_address(const var_addr_t& inp);
+cryptonote::account_public_address get_address(const cryptonote::account_public_address& inp);
+cryptonote::account_public_address get_address(const cryptonote::account_keys& inp);
+cryptonote::account_public_address get_address(const cryptonote::account_base& inp);
+cryptonote::account_public_address get_address(const cryptonote::tx_destination_entry& inp);
+
+inline cryptonote::difficulty_type get_test_difficulty(const boost::optional<uint8_t>& hf_ver=boost::none) {return !hf_ver || hf_ver.get() <= 1 ? 1 : 2;}
+inline uint64_t current_difficulty_window(const boost::optional<uint8_t>& hf_ver=boost::none){ return !hf_ver || hf_ver.get() <= 1 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; }
 void fill_nonce(cryptonote::block& blk, const cryptonote::difficulty_type& diffic, uint64_t height);
 
+cryptonote::tx_destination_entry build_dst(const var_addr_t& to, bool is_subaddr=false, uint64_t amount=0);
+std::vector<cryptonote::tx_destination_entry> build_dsts(const var_addr_t& to1, bool sub1=false, uint64_t am1=0);
+std::vector<cryptonote::tx_destination_entry> build_dsts(std::initializer_list<dest_wrapper_t> inps);
+uint64_t sum_amount(const std::vector<cryptonote::tx_destination_entry>& destinations);
+uint64_t sum_amount(const std::vector<cryptonote::tx_source_entry>& sources);
+
 bool construct_miner_tx_manually(size_t height, uint64_t already_generated_coins,
                                  const cryptonote::account_public_address& miner_address, cryptonote::transaction& tx,
-                                 uint64_t fee, cryptonote::keypair* p_txkey = 0);
+                                 uint64_t fee, cryptonote::keypair* p_txkey = nullptr);
+
 bool construct_tx_to_key(const std::vector<test_event_entry>& events, cryptonote::transaction& tx,
-                         const cryptonote::block& blk_head, const cryptonote::account_base& from, const cryptonote::account_base& to,
+                         const cryptonote::block& blk_head, const cryptonote::account_base& from, const var_addr_t& to, uint64_t amount,
-                         uint64_t amount, uint64_t fee, size_t nmix);
+                         uint64_t fee, size_t nmix, bool rct=false, rct::RangeProofType range_proof_type=rct::RangeProofBorromean, int bp_version = 0);
+
+bool construct_tx_to_key(const std::vector<test_event_entry>& events, cryptonote::transaction& tx, const cryptonote::block& blk_head,
+                         const cryptonote::account_base& from, std::vector<cryptonote::tx_destination_entry> destinations,
+                         uint64_t fee, size_t nmix, bool rct=false, rct::RangeProofType range_proof_type=rct::RangeProofBorromean, int bp_version = 0);
+
+bool construct_tx_to_key(cryptonote::transaction& tx, const cryptonote::account_base& from, const var_addr_t& to, uint64_t amount,
+                         std::vector<cryptonote::tx_source_entry> &sources,
+                         uint64_t fee, bool rct=false, rct::RangeProofType range_proof_type=rct::RangeProofBorromean, int bp_version = 0);
+
+bool construct_tx_to_key(cryptonote::transaction& tx, const cryptonote::account_base& from, const std::vector<cryptonote::tx_destination_entry>& destinations,
+                         std::vector<cryptonote::tx_source_entry> &sources,
+                         uint64_t fee, bool rct, rct::RangeProofType range_proof_type, int bp_version = 0);
+
 cryptonote::transaction construct_tx_with_fee(std::vector<test_event_entry>& events, const cryptonote::block& blk_head,
-                                            const cryptonote::account_base& acc_from, const cryptonote::account_base& acc_to,
+                                            const cryptonote::account_base& acc_from, const var_addr_t& to,
                                             uint64_t amount, uint64_t fee);
 
+bool construct_tx_rct(const cryptonote::account_keys& sender_account_keys,
+    std::vector<cryptonote::tx_source_entry>& sources,
+    const std::vector<cryptonote::tx_destination_entry>& destinations,
+    const boost::optional<cryptonote::account_public_address>& change_addr,
+    std::vector<uint8_t> extra, cryptonote::transaction& tx, uint64_t unlock_time,
+    bool rct=false, rct::RangeProofType range_proof_type=rct::RangeProofBorromean, int bp_version = 0);
+
+
+uint64_t num_blocks(const std::vector<test_event_entry>& events);
+cryptonote::block get_head_block(const std::vector<test_event_entry>& events);
+
 void get_confirmed_txs(const std::vector<cryptonote::block>& blockchain, const map_hash2tx_t& mtx, map_hash2tx_t& confirmed_txs);
+bool trim_block_chain(std::vector<cryptonote::block>& blockchain, const crypto::hash& tail);
+bool trim_block_chain(std::vector<const cryptonote::block*>& blockchain, const crypto::hash& tail);
 bool find_block_chain(const std::vector<test_event_entry>& events, std::vector<cryptonote::block>& blockchain, map_hash2tx_t& mtx, const crypto::hash& head);
+bool find_block_chain(const std::vector<test_event_entry>& events, std::vector<const cryptonote::block*>& blockchain, map_hash2tx_t& mtx, const crypto::hash& head);
+
+void fill_tx_destinations(const var_addr_t& from, const cryptonote::account_public_address& to,
+                          uint64_t amount, uint64_t fee,
+                          const std::vector<cryptonote::tx_source_entry> &sources,
+                          std::vector<cryptonote::tx_destination_entry>& destinations, bool always_change=false);
+
+void fill_tx_destinations(const var_addr_t& from, const std::vector<cryptonote::tx_destination_entry>& dests,
+                          uint64_t fee,
+                          const std::vector<cryptonote::tx_source_entry> &sources,
+                          std::vector<cryptonote::tx_destination_entry>& destinations,
+                          bool always_change);
+
+void fill_tx_destinations(const var_addr_t& from, const cryptonote::account_public_address& to,
+                          uint64_t amount, uint64_t fee,
+                          const std::vector<cryptonote::tx_source_entry> &sources,
+                          std::vector<cryptonote::tx_destination_entry>& destinations,
+                          std::vector<cryptonote::tx_destination_entry>& destinations_pure,
+                          bool always_change=false);
+
+
+void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& events, const cryptonote::block& blk_head,
+                                      const cryptonote::account_base& from, const cryptonote::account_public_address& to,
+                                      uint64_t amount, uint64_t fee, size_t nmix,
+                                      std::vector<cryptonote::tx_source_entry>& sources,
+                                      std::vector<cryptonote::tx_destination_entry>& destinations);
+
 void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& events, const cryptonote::block& blk_head,
                                       const cryptonote::account_base& from, const cryptonote::account_base& to,
                                       uint64_t amount, uint64_t fee, size_t nmix,
                                       std::vector<cryptonote::tx_source_entry>& sources,
                                       std::vector<cryptonote::tx_destination_entry>& destinations);
+
 uint64_t get_balance(const cryptonote::account_base& addr, const std::vector<cryptonote::block>& blockchain, const map_hash2tx_t& mtx);
 
+bool extract_hard_forks(const std::vector<test_event_entry>& events, v_hardforks_t& hard_forks);
+
 //--------------------------------------------------------------------------
 template<class t_test_class>
 auto do_check_tx_verification_context(const cryptonote::tx_verification_context& tvc, bool tx_added, size_t event_index, const cryptonote::transaction& tx, t_test_class& validator, int)
@@ -338,6 +587,12 @@ public:
     m_ev_index = ev_index;
   }
 
+  bool operator()(const event_replay_settings& settings)
+  {
+    log_event("event_replay_settings");
+    return true;
+  }
+
   bool operator()(const event_visitor_settings& settings)
   {
     log_event("event_visitor_settings");
@@ -460,13 +715,21 @@ private:
 //--------------------------------------------------------------------------
 template<class t_test_class>
 inline bool replay_events_through_core(cryptonote::core& cr, const std::vector<test_event_entry>& events, t_test_class& validator)
+{
+  return replay_events_through_core_plain(cr, events, validator, true);
+}
+//--------------------------------------------------------------------------
+template<class t_test_class>
+inline bool replay_events_through_core_plain(cryptonote::core& cr, const std::vector<test_event_entry>& events, t_test_class& validator, bool reinit=true)
 {
   TRY_ENTRY();
 
   //init core here
-
+  if (reinit) {
-  CHECK_AND_ASSERT_MES(typeid(cryptonote::block) == events[0].type(), false, "First event must be genesis block creation");
+    CHECK_AND_ASSERT_MES(typeid(cryptonote::block) == events[0].type(), false,
+                         "First event must be genesis block creation");
     cr.set_genesis_block(boost::get<cryptonote::block>(events[0]));
+  }
 
   bool r = true;
   push_core_event_visitor<t_test_class> visitor(cr, events, validator);
@@ -489,10 +752,9 @@ struct get_test_options {
   };
   get_test_options():hard_forks{std::make_pair((uint8_t)1, (uint64_t)0), std::make_pair((uint8_t)0, (uint64_t)0)}{}
 };
-
 //--------------------------------------------------------------------------
 template<class t_test_class>
-inline bool do_replay_events(std::vector<test_event_entry>& events)
+inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cryptonote::core **core)
 {
   boost::program_options::options_description desc("Allowed options");
   cryptonote::core::init_options(desc);
@@ -506,12 +768,24 @@ inline bool do_replay_events(std::vector<test_event_entry>& events)
   if (!r)
     return false;
 
-  cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects
+  *core = new cryptonote::core(nullptr);
-  cryptonote::core c(&pr);
+  auto & c = **core;
+
   // FIXME: make sure that vm has arg_testnet_on set to true or false if
   // this test needs for it to be so.
   get_test_options<t_test_class> gto;
-  if (!c.init(vm, &gto.test_options))
+
+  // Hardforks can be specified in events.
+  v_hardforks_t hardforks;
+  cryptonote::test_options test_options_tmp{};
+  const cryptonote::test_options * test_options_ = &gto.test_options;
+  if (extract_hard_forks(events, hardforks)){
+    hardforks.push_back(std::make_pair((uint8_t)0, (uint64_t)0));  // terminator
+    test_options_tmp.hard_forks = hardforks.data();
+    test_options_ = &test_options_tmp;
+  }
+
+  if (!c.init(vm, test_options_))
   {
     MERROR("Failed to init core");
     return false;
@@ -529,7 +803,31 @@ inline bool do_replay_events(std::vector<test_event_entry>& events)
 
   t_test_class validator;
   bool ret = replay_events_through_core<t_test_class>(c, events, validator);
-  c.deinit();
+//  c.deinit();
+  return ret;
+}
+//--------------------------------------------------------------------------
+template<class t_test_class>
+inline bool replay_events_through_core_validate(std::vector<test_event_entry>& events, cryptonote::core & c)
+{
+  std::vector<crypto::hash> pool_txs;
+  if (!c.get_pool_transaction_hashes(pool_txs))
+  {
+    MERROR("Failed to flush txpool");
+    return false;
+  }
+  c.get_blockchain_storage().flush_txes_from_pool(pool_txs);
+
+  t_test_class validator;
+  return replay_events_through_core_plain<t_test_class>(c, events, validator, false);
+}
+//--------------------------------------------------------------------------
+template<class t_test_class>
+inline bool do_replay_events(std::vector<test_event_entry>& events)
+{
+  cryptonote::core * core;
+  bool ret = do_replay_events_get_core<t_test_class>(events, &core);
+  core->deinit();
   return ret;
 }
 //--------------------------------------------------------------------------
@@ -546,6 +844,12 @@ inline bool do_replay_file(const std::string& filename)
 }
 
 //--------------------------------------------------------------------------
+#define DEFAULT_HARDFORKS(HARDFORKS) do { \
+  HARDFORKS.push_back(std::make_pair((uint8_t)1, (uint64_t)0)); \
+} while(0)
+
+#define ADD_HARDFORK(HARDFORKS, FORK, HEIGHT) HARDFORKS.push_back(std::make_pair((uint8_t)FORK, (uint64_t)HEIGHT))
+
 #define GENERATE_ACCOUNT(account) \
     cryptonote::account_base account; \
     account.generate();
@@ -589,6 +893,11 @@ inline bool do_replay_file(const std::string& filename)
   generator.construct_block(BLK_NAME, PREV_BLOCK, MINER_ACC);                         \
   VEC_EVENTS.push_back(BLK_NAME);
 
+#define MAKE_NEXT_BLOCK_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, HF)           \
+  cryptonote::block BLK_NAME;                                                         \
+  generator.construct_block(BLK_NAME, PREV_BLOCK, MINER_ACC, std::list<cryptonote::transaction>(), HF);                     \
+  VEC_EVENTS.push_back(BLK_NAME);
+
 #define MAKE_NEXT_BLOCK_TX1(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, TX1)         \
   cryptonote::block BLK_NAME;                                                           \
   {                                                                                   \
@@ -598,30 +907,51 @@ inline bool do_replay_file(const std::string& filename)
   }                                                                                   \
   VEC_EVENTS.push_back(BLK_NAME);
 
+#define MAKE_NEXT_BLOCK_TX1_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, TX1, HF)         \
+  cryptonote::block BLK_NAME;                                                           \
+  {                                                                                   \
+    std::list<cryptonote::transaction> tx_list;                                         \
+    tx_list.push_back(TX1);                                                           \
+    generator.construct_block(BLK_NAME, PREV_BLOCK, MINER_ACC, tx_list, HF);              \
+  }                                                                                   \
+  VEC_EVENTS.push_back(BLK_NAME);
+
 #define MAKE_NEXT_BLOCK_TX_LIST(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, TXLIST)  \
   cryptonote::block BLK_NAME;                                                           \
   generator.construct_block(BLK_NAME, PREV_BLOCK, MINER_ACC, TXLIST);                 \
   VEC_EVENTS.push_back(BLK_NAME);
 
-#define REWIND_BLOCKS_N(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, COUNT)           \
+#define MAKE_NEXT_BLOCK_TX_LIST_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, TXLIST, HF)  \
+  cryptonote::block BLK_NAME;                                                           \
+  generator.construct_block(BLK_NAME, PREV_BLOCK, MINER_ACC, TXLIST, HF);                 \
+  VEC_EVENTS.push_back(BLK_NAME);
+
+#define REWIND_BLOCKS_N_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, COUNT, HF)    \
   cryptonote::block BLK_NAME;                                                         \
   {                                                                                   \
     cryptonote::block blk_last = PREV_BLOCK;                                          \
     for (size_t i = 0; i < COUNT; ++i)                                                \
     {                                                                                 \
-      MAKE_NEXT_BLOCK(VEC_EVENTS, blk, blk_last, MINER_ACC);                          \
+      MAKE_NEXT_BLOCK_HF(VEC_EVENTS, blk, blk_last, MINER_ACC, HF);                   \
       blk_last = blk;                                                                 \
     }                                                                                 \
     BLK_NAME = blk_last;                                                              \
   }
 
+#define REWIND_BLOCKS_N(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, COUNT) REWIND_BLOCKS_N_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, COUNT, boost::none)
 #define REWIND_BLOCKS(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC) REWIND_BLOCKS_N(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW)
+#define REWIND_BLOCKS_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, HF) REWIND_BLOCKS_N_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, HF)
 
 #define MAKE_TX_MIX(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, NMIX, HEAD)                       \
   cryptonote::transaction TX_NAME;                                                             \
   construct_tx_to_key(VEC_EVENTS, TX_NAME, HEAD, FROM, TO, AMOUNT, TESTS_DEFAULT_FEE, NMIX); \
   VEC_EVENTS.push_back(TX_NAME);
 
+#define MAKE_TX_MIX_RCT(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, NMIX, HEAD)                       \
+  cryptonote::transaction TX_NAME;                                                             \
+  construct_tx_to_key(VEC_EVENTS, TX_NAME, HEAD, FROM, TO, AMOUNT, TESTS_DEFAULT_FEE, NMIX, true, rct::RangeProofPaddedBulletproof); \
+  VEC_EVENTS.push_back(TX_NAME);
+
 #define MAKE_TX(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, HEAD) MAKE_TX_MIX(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, 0, HEAD)
 
 #define MAKE_TX_MIX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD)             \
@@ -632,12 +962,36 @@ inline bool do_replay_file(const std::string& filename)
     VEC_EVENTS.push_back(t);                                                             \
   }
 
+#define MAKE_TX_MIX_LIST_RCT(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD) \
+        MAKE_TX_MIX_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD, rct::RangeProofPaddedBulletproof, 1)
+#define MAKE_TX_MIX_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD, RCT_TYPE, BP_VER)  \
+  {                                                                                      \
+    cryptonote::transaction t;                                                           \
+    construct_tx_to_key(VEC_EVENTS, t, HEAD, FROM, TO, AMOUNT, TESTS_DEFAULT_FEE, NMIX, true, RCT_TYPE, BP_VER); \
+    SET_NAME.push_back(t);                                                               \
+    VEC_EVENTS.push_back(t);                                                             \
+  }
+
+#define MAKE_TX_MIX_DEST_LIST_RCT(VEC_EVENTS, SET_NAME, FROM, TO, NMIX, HEAD)            \
+        MAKE_TX_MIX_DEST_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, NMIX, HEAD, rct::RangeProofPaddedBulletproof, 1)
+#define MAKE_TX_MIX_DEST_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, NMIX, HEAD, RCT_TYPE, BP_VER)  \
+  {                                                                                      \
+    cryptonote::transaction t;                                                           \
+    construct_tx_to_key(VEC_EVENTS, t, HEAD, FROM, TO, TESTS_DEFAULT_FEE, NMIX, true, RCT_TYPE, BP_VER); \
+    SET_NAME.push_back(t);                                                               \
+    VEC_EVENTS.push_back(t);                                                             \
+  }
+
 #define MAKE_TX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, HEAD) MAKE_TX_MIX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, 0, HEAD)
 
 #define MAKE_TX_LIST_START(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, HEAD) \
     std::list<cryptonote::transaction> SET_NAME; \
     MAKE_TX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, HEAD);
 
+#define MAKE_TX_LIST_START_RCT(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD) \
+    std::list<cryptonote::transaction> SET_NAME; \
+    MAKE_TX_MIX_LIST_RCT(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD);
+
 #define MAKE_MINER_TX_AND_KEY_MANUALLY(TX, BLK, KEY)                                                      \
   transaction TX;                                                                                         \
   if (!construct_miner_tx_manually(get_block_height(BLK) + 1, generator.get_already_generated_coins(BLK), \
@@ -668,6 +1022,46 @@ inline bool do_replay_file(const std::string& filename)
       return 1; \
     }
 
+#define CATCH_REPLAY(genclass)                                                                             \
+    catch (const std::exception& ex)                                                                       \
+    {                                                                                                      \
+      MERROR(#genclass << " generation failed: what=" << ex.what());                                       \
+    }                                                                                                      \
+    catch (...)                                                                                            \
+    {                                                                                                      \
+      MERROR(#genclass << " generation failed: generic exception");                                        \
+    }
+
+#define REPLAY_CORE(genclass)                                                                              \
+    if (generated && do_replay_events< genclass >(events))                                                 \
+    {                                                                                                      \
+      MGINFO_GREEN("#TEST# Succeeded " << #genclass);                                                      \
+    }                                                                                                      \
+    else                                                                                                   \
+    {                                                                                                      \
+      MERROR("#TEST# Failed " << #genclass);                                                               \
+      failed_tests.push_back(#genclass);                                                                   \
+    }
+
+#define REPLAY_WITH_CORE(genclass, CORE)                                                                   \
+    if (generated && replay_events_through_core_validate< genclass >(events, CORE))                        \
+    {                                                                                                      \
+      MGINFO_GREEN("#TEST# Succeeded " << #genclass);                                                      \
+    }                                                                                                      \
+    else                                                                                                   \
+    {                                                                                                      \
+      MERROR("#TEST# Failed " << #genclass);                                                               \
+      failed_tests.push_back(#genclass);                                                                   \
+    }
+
+#define CATCH_GENERATE_REPLAY(genclass)                                                                    \
+    CATCH_REPLAY(genclass);                                                                                \
+    REPLAY_CORE(genclass);
+
+#define CATCH_GENERATE_REPLAY_CORE(genclass, CORE)                                                         \
+    CATCH_REPLAY(genclass);                                                                                \
+    REPLAY_WITH_CORE(genclass, CORE);
+
 #define GENERATE_AND_PLAY(genclass) \
   if (list_tests)                                                                                          \
     std::cout << #genclass << std::endl;                                                                   \
@@ -679,25 +1073,22 @@ inline bool do_replay_file(const std::string& filename)
     try                                                                                                    \
     {                                                                                                      \
       genclass g;                                                                                          \
-      generated = g.generate(events);;                                                                     \
+      generated = g.generate(events);                                                                      \
     }                                                                                                      \
-    catch (const std::exception& ex)                                                                       \
+    CATCH_GENERATE_REPLAY(genclass);                                                                       \
+  }
+
+#define GENERATE_AND_PLAY_INSTANCE(genclass, ins, CORE)                                                    \
+  if (filter.empty() || boost::regex_match(std::string(#genclass), match, boost::regex(filter)))           \
   {                                                                                                        \
-      MERROR(#genclass << " generation failed: what=" << ex.what());                                       \
+    std::vector<test_event_entry> events;                                                                  \
-    }                                                                                                      \
+    ++tests_count;                                                                                         \
-    catch (...)                                                                                            \
+    bool generated = false;                                                                                \
+    try                                                                                                    \
     {                                                                                                      \
-      MERROR(#genclass << " generation failed: generic exception");                                        \
+      generated = ins.generate(events);                                                                    \
-    }                                                                                                      \
-    if (generated && do_replay_events< genclass >(events))                                                 \
-    {                                                                                                      \
-      MGINFO_GREEN("#TEST# Succeeded " << #genclass);                                                      \
-    }                                                                                                      \
-    else                                                                                                   \
-    {                                                                                                      \
-      MERROR("#TEST# Failed " << #genclass);                                                               \
-      failed_tests.push_back(#genclass);                                                                   \
     }                                                                                                      \
+    CATCH_GENERATE_REPLAY_CORE(genclass, CORE);                                                            \
   }
 
 #define CALL_TEST(test_name, function)                                                                     \

tests/core_tests/wallet_tools.cpp

@@ -0,0 +1,287 @@
+//
+// Created by Dusan Klinec on 2019-02-28.
+//
+
+#include "wallet_tools.h"
+#include <random>
+
+using namespace std;
+using namespace epee;
+using namespace crypto;
+using namespace cryptonote;
+
+// Shared random generator
+static std::default_random_engine RND(crypto::rand<unsigned>());
+
+void wallet_accessor_test::set_account(tools::wallet2 * wallet, cryptonote::account_base& account)
+{
+  wallet->clear();
+  wallet->m_account = account;
+  wallet->m_nettype = MAINNET;
+
+  wallet->m_key_device_type = account.get_device().get_type();
+  wallet->m_account_public_address = account.get_keys().m_account_address;
+  wallet->m_watch_only = false;
+  wallet->m_multisig = false;
+  wallet->m_multisig_threshold = 0;
+  wallet->m_multisig_signers.clear();
+  wallet->m_device_name = account.get_device().get_name();
+
+  wallet->m_subaddress_lookahead_major = 5;
+  wallet->m_subaddress_lookahead_minor = 20;
+
+  wallet->setup_new_blockchain();  // generates also subadress register
+}
+
+void wallet_accessor_test::process_parsed_blocks(tools::wallet2 * wallet, uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<tools::wallet2::parsed_block> &parsed_blocks, uint64_t& blocks_added)
+{
+  wallet->process_parsed_blocks(start_height, blocks, parsed_blocks, blocks_added);
+}
+
+void wallet_tools::process_transactions(tools::wallet2 * wallet, const std::vector<test_event_entry>& events, const cryptonote::block& blk_head, block_tracker &bt, const boost::optional<crypto::hash>& blk_tail)
+{
+  map_hash2tx_t mtx;
+  std::vector<const cryptonote::block*> blockchain;
+  find_block_chain(events, blockchain, mtx, get_block_hash(blk_head));
+
+  if (blk_tail){
+    trim_block_chain(blockchain, blk_tail.get());
+  }
+
+  process_transactions(wallet, blockchain, mtx, bt);
+}
+
+void wallet_tools::process_transactions(tools::wallet2 * wallet, const std::vector<const cryptonote::block*>& blockchain, const map_hash2tx_t & mtx, block_tracker &bt)
+{
+  uint64_t start_height=0, blocks_added=0;
+  std::vector<cryptonote::block_complete_entry> v_bche;
+  std::vector<tools::wallet2::parsed_block> v_parsed_block;
+
+  v_bche.reserve(blockchain.size());
+  v_parsed_block.reserve(blockchain.size());
+
+  size_t idx = 0;
+  for(auto bl : blockchain)
+  {
+    idx += 1;
+    uint64_t height;
+    v_bche.emplace_back();
+    v_parsed_block.emplace_back();
+
+    wallet_tools::gen_block_data(bt, bl, mtx, v_bche.back(), v_parsed_block.back(), idx == 1 ? start_height : height);
+  }
+
+  if (wallet)
+    wallet_accessor_test::process_parsed_blocks(wallet, start_height, v_bche, v_parsed_block, blocks_added);
+}
+
+bool wallet_tools::fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptonote::tx_source_entry>& sources, size_t mixin, const boost::optional<size_t>& num_utxo, const boost::optional<uint64_t>& min_amount, block_tracker &bt, std::vector<size_t> &selected, uint64_t cur_height, ssize_t offset, int step, const boost::optional<fnc_accept_tx_source_t>& fnc_accept)
+{
+  CHECK_AND_ASSERT_THROW_MES(step != 0, "Step is zero");
+  sources.clear();
+
+  auto & transfers = wallet_accessor_test::get_transfers(wallet);
+  std::unordered_set<size_t> selected_idx;
+  std::unordered_set<crypto::key_image> selected_kis;
+  const size_t ntrans = wallet->get_num_transfer_details();
+  size_t roffset = offset >= 0 ? offset : ntrans - offset - 1;
+  size_t iters = 0;
+  uint64_t sum = 0;
+  size_t cur_utxo = 0;
+  bool abort = false;
+  unsigned brk_cond = 0;
+  unsigned brk_thresh = num_utxo && min_amount ? 2 : (num_utxo || min_amount ? 1 : 0);
+
+#define EVAL_BRK_COND() do {                         \
+  brk_cond = 0;                                      \
+  if (num_utxo && num_utxo.get() <= cur_utxo)        \
+    brk_cond += 1;                                   \
+  if (min_amount && min_amount.get() <= sum)         \
+    brk_cond += 1;                                   \
+  } while(0)
+
+  for(ssize_t i = roffset; iters < ntrans && !abort; i += step, ++iters)
+  {
+    EVAL_BRK_COND();
+    if (brk_cond >= brk_thresh)
+      break;
+
+    i = i < 0 ? (i + ntrans) : i % ntrans;
+    auto & td = transfers[i];
+    if (td.m_spent)
+      continue;
+    if (td.m_block_height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW > cur_height)
+      continue;
+    if (selected_idx.find((size_t)i) != selected_idx.end()){
+      MERROR("Should not happen (selected_idx not found): " << i);
+      continue;
+    }
+    if (selected_kis.find(td.m_key_image) != selected_kis.end()){
+      MERROR("Should not happen (selected KI): " << i << "ki: " << dump_keys(td.m_key_image.data));
+      continue;
+    }
+
+    try {
+      cryptonote::tx_source_entry src;
+      wallet_tools::gen_tx_src(mixin, cur_height, td, src, bt);
+
+      // Acceptor function
+      if (fnc_accept){
+        tx_source_info_crate_t c_info{.td=&td, .src=&src, .selected_idx=&selected_idx, .selected_kis=&selected_kis,
+            .ntrans=ntrans, .iters=iters, .sum=sum, .cur_utxo=cur_utxo};
+
+        bool take_it = (fnc_accept.get())(c_info, abort);
+        if (!take_it){
+          continue;
+        }
+      }
+
+      MINFO("Selected " << i << " from tx: " << dump_keys(td.m_txid.data)
+                        << " ki: " << dump_keys(td.m_key_image.data)
+                        << " amnt: " << td.amount()
+                        << " rct: " << td.is_rct()
+                        << " glob: " << td.m_global_output_index);
+
+      sum += td.amount();
+      cur_utxo += 1;
+
+      sources.emplace_back(src);
+      selected.push_back((size_t)i);
+      selected_idx.insert((size_t)i);
+      selected_kis.insert(td.m_key_image);
+
+    } catch(const std::exception &e){
+      MTRACE("Output " << i << ", from: " <<  dump_keys(td.m_txid.data)
+                       << ", amnt: " << td.amount() << ", rct: " << td.is_rct()
+                       << ", glob: " << td.m_global_output_index << " is not applicable: " << e.what());
+    }
+  }
+
+  EVAL_BRK_COND();
+  return brk_cond >= brk_thresh;
+#undef EVAL_BRK_COND
+}
+
+void wallet_tools::gen_tx_src(size_t mixin, uint64_t cur_height, const tools::wallet2::transfer_details & td, cryptonote::tx_source_entry & src, block_tracker &bt)
+{
+  src.amount = td.amount();
+  src.rct = td.is_rct();
+
+  std::vector<tools::wallet2::get_outs_entry> outs;
+  bt.get_fake_outs(mixin, td.is_rct() ? 0 : td.amount(), td.m_global_output_index, cur_height, outs);
+
+  for (size_t n = 0; n < mixin; ++n)
+  {
+    cryptonote::tx_source_entry::output_entry oe;
+    oe.first = std::get<0>(outs[n]);
+    oe.second.dest = rct::pk2rct(std::get<1>(outs[n]));
+    oe.second.mask = std::get<2>(outs[n]);
+    src.outputs.push_back(oe);
+  }
+
+  size_t real_idx = crypto::rand<size_t>() % mixin;
+
+  cryptonote::tx_source_entry::output_entry &real_oe = src.outputs[real_idx];
+  real_oe.first = td.m_global_output_index;
+  real_oe.second.dest = rct::pk2rct(boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key);
+  real_oe.second.mask = rct::commit(td.amount(), td.m_mask);
+
+  std::sort(src.outputs.begin(), src.outputs.end(), [&](const cryptonote::tx_source_entry::output_entry i0, const cryptonote::tx_source_entry::output_entry i1) {
+    return i0.first < i1.first;
+  });
+
+  for (size_t i = 0; i < src.outputs.size(); ++i){
+    if (src.outputs[i].first == td.m_global_output_index){
+      src.real_output = i;
+      break;
+    }
+  }
+
+  src.mask = td.m_mask;
+  src.real_out_tx_key = get_tx_pub_key_from_extra(td.m_tx, td.m_pk_index);
+  src.real_out_additional_tx_keys = get_additional_tx_pub_keys_from_extra(td.m_tx);
+  src.real_output_in_tx_index = td.m_internal_output_index;
+  src.multisig_kLRki = rct::multisig_kLRki({rct::zero(), rct::zero(), rct::zero(), rct::zero()});
+}
+
+void wallet_tools::gen_block_data(block_tracker &bt, const cryptonote::block *bl, const map_hash2tx_t &mtx, cryptonote::block_complete_entry &bche, tools::wallet2::parsed_block &parsed_block, uint64_t &height)
+{
+  vector<const transaction*> vtx;
+  vtx.push_back(&(bl->miner_tx));
+  height = boost::get<txin_gen>(*bl->miner_tx.vin.begin()).height;
+
+  BOOST_FOREACH(const crypto::hash &h, bl->tx_hashes) {
+          const map_hash2tx_t::const_iterator cit = mtx.find(h);
+          CHECK_AND_ASSERT_THROW_MES(mtx.end() != cit, "block contains an unknown tx hash @ " << height << ", " << h);
+          vtx.push_back(cit->second);
+        }
+
+  bche.block = "NA";
+  bche.txs.resize(bl->tx_hashes.size());
+
+  parsed_block.error = false;
+  parsed_block.hash = get_block_hash(*bl);
+  parsed_block.block = *bl;
+  parsed_block.txes.reserve(bl->tx_hashes.size());
+
+  auto & o_indices = parsed_block.o_indices.indices;
+  o_indices.reserve(bl->tx_hashes.size() + 1);
+
+  size_t cur = 0;
+  BOOST_FOREACH(const transaction *tx, vtx){
+          cur += 1;
+          o_indices.emplace_back();
+          bt.process(bl, tx, cur - 1);
+          bt.global_indices(tx, o_indices.back().indices);
+
+          if (cur > 1)  // miner not included
+            parsed_block.txes.push_back(*tx);
+        }
+}
+
+void wallet_tools::compute_subaddresses(std::unordered_map<crypto::public_key, cryptonote::subaddress_index> &subaddresses, cryptonote::account_base & creds, size_t account, size_t minors)
+{
+  auto &hwdev = hw::get_device("default");
+  const std::vector<crypto::public_key> pkeys = hwdev.get_subaddress_spend_public_keys(creds.get_keys(), account, 0, minors);
+
+  for(uint32_t c = 0; c < pkeys.size(); ++c){
+    cryptonote::subaddress_index sidx{(uint32_t)account, c};
+    subaddresses[pkeys[c]] = sidx;
+  }
+}
+
+cryptonote::account_public_address get_address(const tools::wallet2* inp)
+{
+  return (inp)->get_account().get_keys().m_account_address;
+}
+
+bool construct_tx_to_key(cryptonote::transaction& tx,
+                         tools::wallet2 * sender_wallet, const var_addr_t& to, uint64_t amount,
+                         std::vector<cryptonote::tx_source_entry> &sources,
+                         uint64_t fee, bool rct, rct::RangeProofType range_proof_type, int bp_version)
+{
+  vector<tx_destination_entry> destinations;
+  fill_tx_destinations(sender_wallet->get_account(), get_address(to), amount, fee, sources, destinations, rct);
+  return construct_tx_rct(sender_wallet, sources, destinations, get_address(sender_wallet), std::vector<uint8_t>(), tx, 0, rct, range_proof_type, bp_version);
+}
+
+bool construct_tx_to_key(cryptonote::transaction& tx,
+                         tools::wallet2 * sender_wallet,
+                         const std::vector<cryptonote::tx_destination_entry>& destinations,
+                         std::vector<cryptonote::tx_source_entry> &sources,
+                         uint64_t fee, bool rct, rct::RangeProofType range_proof_type, int bp_version)
+{
+  vector<tx_destination_entry> all_destinations;
+  fill_tx_destinations(sender_wallet->get_account(), destinations, fee, sources, all_destinations, rct);
+  return construct_tx_rct(sender_wallet, sources, all_destinations, get_address(sender_wallet), std::vector<uint8_t>(), tx, 0, rct, range_proof_type, bp_version);
+}
+
+bool construct_tx_rct(tools::wallet2 * sender_wallet, std::vector<cryptonote::tx_source_entry>& sources, const std::vector<cryptonote::tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, cryptonote::transaction& tx, uint64_t unlock_time, bool rct, rct::RangeProofType range_proof_type, int bp_version)
+{
+  subaddresses_t & subaddresses = wallet_accessor_test::get_subaddresses(sender_wallet);
+  crypto::secret_key tx_key;
+  std::vector<crypto::secret_key> additional_tx_keys;
+  std::vector<tx_destination_entry> destinations_copy = destinations;
+  rct::RCTConfig rct_config = {range_proof_type, bp_version};
+  return construct_tx_and_get_tx_key(sender_wallet->get_account().get_keys(), subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, nullptr);
+}

tests/core_tests/wallet_tools.h

@@ -0,0 +1,86 @@
+// Copyright (c) 2014-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once
+
+#include "chaingen.h"
+#include "wallet/wallet2.h"
+
+typedef struct {
+  tools::wallet2::transfer_details * td;
+  cryptonote::tx_source_entry * src;
+
+  std::unordered_set<size_t> * selected_idx;
+  std::unordered_set<crypto::key_image> * selected_kis;
+  size_t ntrans;
+  size_t iters;
+  uint64_t sum;
+  size_t cur_utxo;
+} tx_source_info_crate_t;
+
+typedef std::function<bool(const tx_source_info_crate_t &info, bool &abort)> fnc_accept_tx_source_t;
+
+// Wallet friend, direct access to required fields and private methods
+class wallet_accessor_test
+{
+public:
+  static void set_account(tools::wallet2 * wallet, cryptonote::account_base& account);
+  static tools::wallet2::transfer_container & get_transfers(tools::wallet2 * wallet) { return wallet->m_transfers; }
+  static subaddresses_t & get_subaddresses(tools::wallet2 * wallet) { return wallet->m_subaddresses; }
+  static void process_parsed_blocks(tools::wallet2 * wallet, uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<tools::wallet2::parsed_block> &parsed_blocks, uint64_t& blocks_added);
+};
+
+class wallet_tools
+{
+public:
+  static void gen_tx_src(size_t mixin, uint64_t cur_height, const tools::wallet2::transfer_details & td, cryptonote::tx_source_entry & src, block_tracker &bt);
+  static void gen_block_data(block_tracker &bt, const cryptonote::block *bl, const map_hash2tx_t & mtx, cryptonote::block_complete_entry &bche, tools::wallet2::parsed_block &parsed_block, uint64_t &height);
+  static void compute_subaddresses(std::unordered_map<crypto::public_key, cryptonote::subaddress_index> &subaddresses, cryptonote::account_base & creds, size_t account, size_t minors);
+  static void process_transactions(tools::wallet2 * wallet, const std::vector<test_event_entry>& events, const cryptonote::block& blk_head, block_tracker &bt, const boost::optional<crypto::hash>& blk_tail=boost::none);
+  static void process_transactions(tools::wallet2 * wallet, const std::vector<const cryptonote::block*>& blockchain, const map_hash2tx_t & mtx, block_tracker &bt);
+  static bool fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptonote::tx_source_entry>& sources, size_t mixin, const boost::optional<size_t>& num_utxo, const boost::optional<uint64_t>& min_amount, block_tracker &bt, std::vector<size_t> &selected, uint64_t cur_height, ssize_t offset=0, int step=1, const boost::optional<fnc_accept_tx_source_t>& fnc_accept=boost::none);
+};
+
+cryptonote::account_public_address get_address(const tools::wallet2*);
+
+bool construct_tx_to_key(cryptonote::transaction& tx, tools::wallet2 * from_wallet, const var_addr_t& to, uint64_t amount,
+                         std::vector<cryptonote::tx_source_entry> &sources,
+                         uint64_t fee, bool rct=false, rct::RangeProofType range_proof_type=rct::RangeProofBorromean, int bp_version = 0);
+
+bool construct_tx_to_key(cryptonote::transaction& tx, tools::wallet2 * sender_wallet, const std::vector<cryptonote::tx_destination_entry>& destinations,
+                         std::vector<cryptonote::tx_source_entry> &sources,
+                         uint64_t fee, bool rct, rct::RangeProofType range_proof_type, int bp_version = 0);
+
+bool construct_tx_rct(tools::wallet2 * sender_wallet,
+                      std::vector<cryptonote::tx_source_entry>& sources,
+                      const std::vector<cryptonote::tx_destination_entry>& destinations,
+                      const boost::optional<cryptonote::account_public_address>& change_addr,
+                      std::vector<uint8_t> extra, cryptonote::transaction& tx, uint64_t unlock_time,
+                      bool rct=false, rct::RangeProofType range_proof_type=rct::RangeProofBorromean, int bp_version = 0);

tests/trezor/CMakeLists.txt

@@ -0,0 +1,63 @@
+# Copyright (c) 2014-2018, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set(trezor_tests_sources
+        trezor_tests.cpp
+        ../core_tests/chaingen.cpp
+        ../core_tests/wallet_tools.cpp)
+
+set(trezor_tests_headers
+        trezor_tests.h
+        ../core_tests/chaingen.h
+        ../core_tests/wallet_tools.h)
+
+add_executable(trezor_tests
+  ${trezor_tests_sources}
+  ${trezor_tests_headers})
+
+target_link_libraries(trezor_tests
+  PRIVATE
+    multisig
+    cryptonote_core
+    p2p
+    version
+    epee
+    device
+    device_trezor
+    wallet
+    ${CMAKE_THREAD_LIBS_INIT}
+    ${EXTRA_LIBRARIES})
+
+enable_stack_trace(trezor_tests)
+set_property(TARGET trezor_tests
+  PROPERTY
+    FOLDER "tests")
+
+add_test(
+  NAME    trezor_tests
+  COMMAND trezor_tests)

tests/trezor/trezor_tests.h

@@ -0,0 +1,248 @@
+// Copyright (c) 2014-2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once
+
+#include <device_trezor/device_trezor.hpp>
+#include "../core_tests/chaingen.h"
+#include "../core_tests/wallet_tools.h"
+
+#define TREZOR_TEST_FEE 90000000000
+#define TREZOR_TEST_MIXIN 11
+
+/************************************************************************/
+/*                                                                      */
+/************************************************************************/
+class tsx_builder;
+class gen_trezor_base : public test_chain_unit_base
+{
+public:
+  friend class tsx_builder;
+
+  gen_trezor_base();
+  gen_trezor_base(const gen_trezor_base &other);
+  virtual ~gen_trezor_base() {};
+
+  void setup_args(const std::string & trezor_path, bool heavy_tests=false);
+  virtual bool generate(std::vector<test_event_entry>& events);
+  virtual void load(std::vector<test_event_entry>& events);       // load events, init test obj
+  void fix_hf(std::vector<test_event_entry>& events);
+  void update_trackers(std::vector<test_event_entry>& events);
+
+  void fork(gen_trezor_base & other);                             // fork generated chain to another test
+  void clear();                                                   // clears m_events, bt, generator, hforks
+  void add_shared_events(std::vector<test_event_entry>& events);  // m_events -> events
+  void test_setup(std::vector<test_event_entry>& events);         // init setup env, wallets
+
+  void test_trezor_tx(std::vector<test_event_entry>& events,
+      std::vector<tools::wallet2::pending_tx>& ptxs,
+      std::vector<cryptonote::address_parse_info>& dsts_info,
+      test_generator &generator,
+      std::vector<tools::wallet2*> wallets,
+      bool is_sweep=false);
+
+  crypto::hash head_hash() { return get_block_hash(m_head); }
+  cryptonote::block head_block() { return m_head; }
+  bool heavy_tests() { return m_heavy_tests; }
+  void rct_config(rct::RCTConfig rct_config) { m_rct_config = rct_config; }
+  uint8_t cur_hf(){ return m_hard_forks.size() > 0 ? m_hard_forks.back().first : 0; }
+
+  // Static configuration
+  static const uint64_t m_ts_start;
+  static const uint64_t m_wallet_ts;
+  static const std::string  m_device_name;
+  static const std::string  m_master_seed_str;
+  static const std::string  m_device_seed;
+  static const std::string  m_alice_spend_private;
+  static const std::string  m_alice_view_private;
+
+protected:
+  void setup_trezor();
+  void init_fields();
+
+  test_generator m_generator;
+  block_tracker m_bt;
+
+  v_hardforks_t m_hard_forks;
+  cryptonote::block m_head;
+  std::vector<test_event_entry> m_events;
+
+  std::string m_trezor_path;
+  bool m_heavy_tests;
+  rct::RCTConfig m_rct_config;
+
+  cryptonote::account_base m_miner_account;
+  cryptonote::account_base m_bob_account;
+  cryptonote::account_base m_alice_account;
+  cryptonote::account_base m_eve_account;
+  hw::trezor::device_trezor * m_trezor;
+  std::unique_ptr<tools::wallet2> m_wl_alice;
+  std::unique_ptr<tools::wallet2> m_wl_bob;
+  std::unique_ptr<tools::wallet2> m_wl_eve;
+
+  friend class boost::serialization::access;
+
+  template<class Archive>
+  void serialize(Archive & ar, const unsigned int /*version*/)
+  {
+    ar & m_generator;
+  }
+};
+
+class tsx_builder {
+public:
+  tsx_builder(): m_tester(nullptr), m_from(nullptr), m_account(0), m_mixin(TREZOR_TEST_MIXIN), m_fee(TREZOR_TEST_FEE),
+  m_rct_config({rct::RangeProofPaddedBulletproof, 1 }){}
+
+  tsx_builder(gen_trezor_base * tester): m_tester(tester), m_from(nullptr), m_account(0),
+  m_mixin(TREZOR_TEST_MIXIN), m_fee(TREZOR_TEST_FEE),
+  m_rct_config({rct::RangeProofPaddedBulletproof, 1 }){}
+
+  tsx_builder * cur_height(uint64_t cur_height) { m_cur_height = cur_height; return this; }
+  tsx_builder * mixin(size_t mixin=TREZOR_TEST_MIXIN) { m_mixin = mixin; return this; }
+  tsx_builder * fee(uint64_t fee=TREZOR_TEST_FEE) { m_fee = fee; return this; }
+  tsx_builder * payment_id(const std::string & payment_id) { m_payment_id = payment_id; return this; }
+  tsx_builder * from(tools::wallet2 *from, uint32_t account=0) { m_from = from; m_account=account; return this; }
+  tsx_builder * sources(std::vector<cryptonote::tx_source_entry> & sources, std::vector<size_t> & selected_transfers);
+  tsx_builder * compute_sources(boost::optional<size_t> num_utxo=boost::none, boost::optional<uint64_t> min_amount=boost::none, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none);
+  tsx_builder * compute_sources_to_sub(boost::optional<size_t> num_utxo=boost::none, boost::optional<uint64_t> min_amount=boost::none, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none);
+  tsx_builder * compute_sources_to_sub_acc(boost::optional<size_t> num_utxo=boost::none, boost::optional<uint64_t> min_amount=boost::none, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none);
+
+  tsx_builder * destinations(std::vector<cryptonote::tx_destination_entry> &dsts);
+  tsx_builder * add_destination(const cryptonote::tx_destination_entry &dst);
+  tsx_builder * add_destination(const tools::wallet2 * wallet, bool is_subaddr=false, uint64_t amount=1000);
+  tsx_builder * add_destination(const var_addr_t addr, bool is_subaddr=false, uint64_t amount=1000);
+  tsx_builder * set_integrated(size_t idx);
+  tsx_builder * rct_config(const rct::RCTConfig & rct_config) {m_rct_config = rct_config; return this; };
+
+  tsx_builder * build_tx();
+  tsx_builder * construct_pending_tx(tools::wallet2::pending_tx &ptx, boost::optional<std::vector<uint8_t>> extra = boost::none);
+  tsx_builder * clear_current();
+  std::vector<tools::wallet2::pending_tx> build();
+  std::vector<cryptonote::address_parse_info> dest_info(){ return m_dsts_info; }
+
+protected:
+  gen_trezor_base * m_tester;
+  uint64_t m_cur_height;
+  std::vector<tools::wallet2::pending_tx> m_ptxs;         // all transactions
+
+  // current transaction
+  size_t m_mixin;
+  uint64_t m_fee;
+  tools::wallet2 * m_from;
+  uint32_t m_account;
+  cryptonote::transaction m_tx;
+  std::vector<size_t> m_selected_transfers;
+  std::vector<cryptonote::tx_source_entry> m_sources;
+  std::vector<cryptonote::tx_destination_entry> m_destinations;
+  std::vector<cryptonote::tx_destination_entry> m_destinations_orig;
+  std::vector<cryptonote::address_parse_info> m_dsts_info;
+  std::unordered_set<size_t> m_integrated;
+  std::string m_payment_id;
+  rct::RCTConfig m_rct_config;
+};
+
+class gen_trezor_ki_sync : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_1utxo : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_1utxo_paymentid_short : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_1utxo_paymentid_short_integrated : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_1utxo_paymentid_long : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_4utxo : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_4utxo_acc1 : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_4utxo_to_sub : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_4utxo_to_2sub : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_4utxo_to_1norm_2sub : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_2utxo_sub_acc_to_1norm_2sub : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_4utxo_to_7outs : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_many_utxo : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};