core, wallet: faster tx pool scanning

Includes a new RPC to get tx pool hashes fast.

src/cryptonote_core/cryptonote_core.cpp

@@ -1037,6 +1037,12 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_transaction_hashes(std::vector<crypto::hash>& txs) const
+  {
+    m_mempool.get_transaction_hashes(txs);
+    return true;
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_pool_transaction(const crypto::hash &id, transaction& tx) const
   {
     return m_mempool.get_transaction(id, tx);

src/cryptonote_core/cryptonote_core.h

@@ -396,6 +396,13 @@ namespace cryptonote
       */
      bool get_pool_transactions(std::list<transaction>& txs) const;
      
+     /**
+      * @copydoc tx_memory_pool::get_transactions
+      *
+      * @note see tx_memory_pool::get_transactions
+      */
+     bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs) const;
+
      /**
       * @copydoc tx_memory_pool::get_transaction
       *

src/cryptonote_core/tx_pool.cpp

@@ -423,6 +423,13 @@ namespace cryptonote
       txs.push_back(tx_vt.second.tx);
   }
   //------------------------------------------------------------------
+  void tx_memory_pool::get_transaction_hashes(std::vector<crypto::hash>& txs) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    for(const auto& tx_vt: m_transactions)
+      txs.push_back(get_transaction_hash(tx_vt.second.tx));
+  }
+  //------------------------------------------------------------------
   //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const
   {

src/cryptonote_core/tx_pool.h

@@ -233,6 +233,13 @@ namespace cryptonote
      */
     void get_transactions(std::list<transaction>& txs) const;
 
+    /**
+     * @brief get a list of all transaction hashes in the pool
+     *
+     * @param txs return-by-reference the list of transactions
+     */
+    void get_transaction_hashes(std::vector<crypto::hash>& txs) const;
+
     /**
      * @brief get information about all transactions and key images in the pool
      *

src/rpc/core_rpc_server.cpp

@@ -789,6 +789,14 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res)
+  {
+    CHECK_CORE_BUSY();
+    m_core.get_pool_transaction_hashes(res.tx_hashes);
+    res.status = CORE_RPC_STATUS_OK;
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res)
   {
     // FIXME: replace back to original m_p2p.send_stop_signal() after

src/rpc/core_rpc_server.h

@@ -92,6 +92,7 @@ namespace cryptonote
       MAP_URI_AUTO_JON2_IF("/set_log_level", on_set_log_level, COMMAND_RPC_SET_LOG_LEVEL, !m_restricted)
       MAP_URI_AUTO_JON2_IF("/set_log_categories", on_set_log_categories, COMMAND_RPC_SET_LOG_CATEGORIES, !m_restricted)
       MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL)
+      MAP_URI_AUTO_JON2("/get_transaction_pool_hashes.bin", on_get_transaction_pool_hashes, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES)
       MAP_URI_AUTO_JON2_IF("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON, !m_restricted)
       MAP_URI_AUTO_JON2("/getinfo", on_get_info, COMMAND_RPC_GET_INFO)
       MAP_URI_AUTO_JON2_IF("/out_peers", on_out_peers, COMMAND_RPC_OUT_PEERS, !m_restricted)
@@ -145,6 +146,7 @@ namespace cryptonote
     bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res);
     bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res);
     bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res);
+    bool on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res);
     bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res);
     bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res);
     bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res);

src/rpc/core_rpc_server_commands_defs.h

@@ -49,7 +49,7 @@ namespace cryptonote
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define CORE_RPC_VERSION_MAJOR 1
-#define CORE_RPC_VERSION_MINOR 7
+#define CORE_RPC_VERSION_MINOR 8
 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
 
@@ -1025,6 +1025,26 @@ namespace cryptonote
     };
   };
 
+  struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES
+  {
+    struct request
+    {
+      BEGIN_KV_SERIALIZE_MAP()
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      std::string status;
+      std::vector<crypto::hash> tx_hashes;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(status)
+        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(tx_hashes)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
+
   struct COMMAND_RPC_GET_CONNECTIONS
   {
     struct request

src/wallet/wallet2.cpp

@@ -1384,13 +1384,13 @@ void wallet2::update_pool_state()
   MDEBUG("update_pool_state start");
 
   // get the pool state
-  cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::request req;
+  cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request req;
-  cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::response res;
+  cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response res;
   m_daemon_rpc_mutex.lock();
-  bool r = epee::net_utils::invoke_http_json("/get_transaction_pool", req, res, m_http_client, rpc_timeout);
+  bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, m_http_client, rpc_timeout);
   m_daemon_rpc_mutex.unlock();
-  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_transaction_pool");
+  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_transaction_pool_hashes.bin");
-  THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_transaction_pool");
+  THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_transaction_pool_hashes.bin");
   THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error);
   MDEBUG("update_pool_state got pool");
 
@@ -1398,11 +1398,11 @@ void wallet2::update_pool_state()
   std::unordered_map<crypto::hash, wallet2::unconfirmed_transfer_details>::iterator it = m_unconfirmed_txs.begin();
   while (it != m_unconfirmed_txs.end())
   {
-    const std::string txid = epee::string_tools::pod_to_hex(it->first);
+    const crypto::hash &txid = it->first;
     bool found = false;
-    for (auto it2: res.transactions)
+    for (const auto &it2: res.tx_hashes)
     {
-      if (it2.id_hash == txid)
+      if (it2 == txid)
       {
         found = true;
         break;
@@ -1453,11 +1453,11 @@ void wallet2::update_pool_state()
   std::unordered_map<crypto::hash, wallet2::payment_details>::iterator uit = m_unconfirmed_payments.begin();
   while (uit != m_unconfirmed_payments.end())
   {
-    const std::string txid = string_tools::pod_to_hex(uit->first);
+    const crypto::hash &txid = uit->first;
     bool found = false;
-    for (auto it2: res.transactions)
+    for (const auto &it2: res.tx_hashes)
     {
-      if (it2.id_hash == txid)
+      if (it2 == txid)
       {
         found = true;
         break;
@@ -1466,18 +1466,16 @@ void wallet2::update_pool_state()
     auto pit = uit++;
     if (!found)
     {
+      MDEBUG("Removing " << txid << " from unconfirmed payments, not found in pool");
       m_unconfirmed_payments.erase(pit);
     }
   }
   MDEBUG("update_pool_state done second loop");
 
-  // add new pool txes to us
+  // gather txids of new pool txes to us
-  for (auto it: res.transactions)
+  std::vector<crypto::hash> txids;
+  for (const auto &txid: res.tx_hashes)
   {
-    cryptonote::blobdata txid_data;
-    if(epee::string_tools::parse_hexstr_to_binbuff(it.id_hash, txid_data) && txid_data.size() == sizeof(crypto::hash))
-    {
-      const crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data());
     if (m_scanned_pool_txs[0].find(txid) != m_scanned_pool_txs[0].end() || m_scanned_pool_txs[1].find(txid) != m_scanned_pool_txs[1].end())
     {
       LOG_PRINT_L2("Already seen " << txid << ", skipped");
@@ -1498,26 +1496,46 @@ void wallet2::update_pool_state()
       if (!found)
       {
         // not one of those we sent ourselves
+        txids.push_back(txid);
+      }
+      else
+      {
+        LOG_PRINT_L1("We sent that one");
+      }
+    }
+    else
+    {
+      LOG_PRINT_L1("Already saw that one, it's for us");
+    }
+  }
+
+  // get those txes
+  if (!txids.empty())
+  {
     cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req;
     cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res;
-          req.txs_hashes.push_back(it.id_hash);
+    for (const auto &txid: txids)
-          MDEBUG("asking for " << it.id_hash);
+      req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
+    MDEBUG("asking for " << txids.size() << " transactions");
     req.decode_as_json = false;
     m_daemon_rpc_mutex.lock();
     bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
-          MDEBUG("asked for " << it.id_hash << ", got " << r << " and " << res.status);
     m_daemon_rpc_mutex.unlock();
+    MDEBUG("Got " << r << " and " << res.status);
     if (r && res.status == CORE_RPC_STATUS_OK)
     {
-            if (res.txs.size() == 1)
+      if (res.txs.size() == txids.size())
+      {
+        size_t n = 0;
+        for (const auto &txid: txids)
         {
           // might have just been put in a block
-              if (res.txs[0].in_pool)
+          if (res.txs[n].in_pool)
           {
             cryptonote::transaction tx;
             cryptonote::blobdata bd;
             crypto::hash tx_hash, tx_prefix_hash;
-                if (epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd))
+            if (epee::string_tools::parse_hexstr_to_binbuff(res.txs[n].as_hex, bd))
             {
               if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash))
               {
@@ -1550,10 +1568,12 @@ void wallet2::update_pool_state()
           {
             LOG_PRINT_L1("Tx " << txid << " was in pool, but is no more");
           }
+          ++n;
+        }
       }
       else
       {
-              LOG_PRINT_L0("Expected 1 tx, got " << res.txs.size());
+        LOG_PRINT_L0("Expected " << txids.size() << " tx(es), got " << res.txs.size());
       }
     }
     else
@@ -1561,21 +1581,6 @@ void wallet2::update_pool_state()
       LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << res.status);
     }
   }
-        else
-        {
-          LOG_PRINT_L1("We sent that one");
-        }
-      }
-      else
-      {
-        LOG_PRINT_L1("Already saw that one, it's for us");
-      }
-    }
-    else
-    {
-      LOG_PRINT_L0("Failed to parse txid");
-    }
-  }
   MDEBUG("update_pool_state end");
 }
 //----------------------------------------------------------------------------------------------------