Merge pull request #666

27f76e2 blockchain_import: Build string for db type list (warptangent)
1aa8a9d blockchain_import: Add mode argument representing multiple DB flags (warptangent)
cffc411 blockchain_import: Support BerkeleyDB (warptangent)
19c1aaa blockchain_import: Add database type argument (warptangent)
0fedce0 blockchain_import: Make LMDB-specific names general (warptangent)

src/blockchain_utilities/blockchain_import.cpp

@@ -38,6 +38,7 @@
 #include "serialization/binary_utils.h" // dump_binary(), parse_binary()
 #include "serialization/json_utils.h" // dump_json()
 #include "include_base_utils.h"
+#include "blockchain_db/db_types.h"
 
 #include <lmdb.h> // for db flag arguments
 
@@ -77,12 +78,48 @@ using namespace cryptonote;
 using namespace epee;
 
 
-int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags)
+std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim)
+{
+  std::string result;
+  std::ostringstream s;
+  std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim));
+  result = s.str();
+  if (result.length() > 0)
+    result.erase(result.end()-strlen(delim), result.end());
+  return result;
+}
+
+// db_type: lmdb, berkeley
+// db_mode: safe, fast, fastest
+int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode)
+{
+  uint64_t BDB_FAST_MODE = 0;
+  uint64_t BDB_FASTEST_MODE = 0;
+  uint64_t BDB_SAFE_MODE = 0;
+
+#if defined(BERKELEY_DB)
+  BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC;
+  BDB_FASTEST_MODE = DB_TXN_NOSYNC;
+  BDB_SAFE_MODE = DB_TXN_SYNC;
+#endif
+
+  int db_flags = 0;
+  bool islmdb = db_type == "lmdb";
+  if (db_mode == "safe")
+    db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE;
+  else if (db_mode == "fast")
+    db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE;
+  else if (db_mode == "fastest")
+    db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE;
+  return db_flags;
+}
+
+int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& db_flags)
 {
   std::vector<std::string> db_args;
   boost::split(db_args, db_arg_str, boost::is_any_of("#"));
-  db_engine = db_args.front();
+  db_type = db_args.front();
-  boost::algorithm::trim(db_engine);
+  boost::algorithm::trim(db_type);
 
   if (db_args.size() == 1)
   {
@@ -94,28 +131,73 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, in
     return 1;
   }
 
+#if !defined(BERKELEY_DB)
+  if (db_type == "berkeley")
+  {
+    LOG_ERROR("BerkeleyDB support disabled.");
+    return false;
+  }
+#endif
+
   std::string db_arg_str2 = db_args[1];
   boost::split(db_args, db_arg_str2, boost::is_any_of(","));
-  for (auto& it : db_args)
+
+  // optionally use a composite mode instead of individual flags
+  const std::unordered_set<std::string> db_modes {"safe", "fast", "fastest"};
+  std::string db_mode;
+  if (db_args.size() == 1)
   {
-    boost::algorithm::trim(it);
+    if (db_modes.count(db_args[0]) > 0)
-    if (it.empty())
-      continue;
-    LOG_PRINT_L1("LMDB flag: " << it);
-    if (it == "nosync")
-      mdb_flags |= MDB_NOSYNC;
-    else if (it == "nometasync")
-      mdb_flags |= MDB_NOMETASYNC;
-    else if (it == "writemap")
-      mdb_flags |= MDB_WRITEMAP;
-    else if (it == "mapasync")
-      mdb_flags |= MDB_MAPASYNC;
-    else if (it == "nordahead")
-      mdb_flags |= MDB_NORDAHEAD;
-    else
     {
-      std::cerr << "unrecognized database flag: " << it << ENDL;
+      db_mode = db_args[0];
-      return 1;
+    }
+  }
+  if (! db_mode.empty())
+  {
+    db_flags = get_db_flags_from_mode(db_type, db_mode);
+  }
+  else
+  {
+    for (auto& it : db_args)
+    {
+      boost::algorithm::trim(it);
+      if (it.empty())
+	continue;
+      if (db_type == "lmdb")
+      {
+	LOG_PRINT_L1("LMDB flag: " << it);
+	if (it == "nosync")
+	  db_flags |= MDB_NOSYNC;
+	else if (it == "nometasync")
+	  db_flags |= MDB_NOMETASYNC;
+	else if (it == "writemap")
+	  db_flags |= MDB_WRITEMAP;
+	else if (it == "mapasync")
+	  db_flags |= MDB_MAPASYNC;
+	else if (it == "nordahead")
+	  db_flags |= MDB_NORDAHEAD;
+	else
+	{
+	  std::cerr << "unrecognized database flag: " << it << ENDL;
+	  return 1;
+	}
+      }
+#if defined(BERKELEY_DB)
+      else if (db_type == "berkeley")
+      {
+	if (it == "txn_write_nosync")
+	  db_flags = DB_TXN_WRITE_NOSYNC;
+	else if (it == "txn_nosync")
+	  db_flags = DB_TXN_NOSYNC;
+	else if (it == "txn_sync")
+	  db_flags = DB_TXN_SYNC;
+	else
+	{
+	  std::cerr << "unrecognized database flag: " << it << ENDL;
+	  return 1;
+	}
+      }
+#endif
     }
   }
   return 0;
@@ -131,7 +213,7 @@ int pop_blocks(FakeCore& simple_core, int num_blocks)
     if (simple_core.support_batch)
       use_batch = true;
     else
-      LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring");
+      LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
   }
 
   if (use_batch)
@@ -176,11 +258,11 @@ template <typename FakeCore>
 int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0)
 {
 #if !defined(BLOCKCHAIN_DB)
-  static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value,
+  static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_db, FakeCore>::value,
       "FakeCore constraint error");
 #endif
 #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
-  if (std::is_same<fake_core_lmdb, FakeCore>::value)
+  if (std::is_same<fake_core_db, FakeCore>::value)
   {
     // Reset stats, in case we're using newly created db, accumulating stats
     // from addition of genesis block.
@@ -250,7 +332,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
     if (simple_core.support_batch)
       use_batch = true;
     else
-      LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring");
+      LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
   }
 
   if (use_batch)
@@ -530,11 +612,19 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
 int main(int argc, char* argv[])
 {
 #if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY)
-  std::string default_db_engine = "memory";
+  std::string default_db_type = "memory";
+  std::string default_db_engine_compiled = "memory";
 #else
-  std::string default_db_engine = "lmdb";
+  std::string default_db_type = "lmdb";
+  std::string default_db_engine_compiled = "blockchain_db";
 #endif
 
+  std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types;
+  db_types_all.insert("memory");
+
+  std::string available_dbs = join_set_strings(db_types_all, ", ");
+  available_dbs = "available: " + available_dbs;
+
   uint32_t log_level = LOG_LEVEL_0;
   uint64_t num_blocks = 0;
   uint64_t block_stop = 0;
@@ -566,8 +656,7 @@ int main(int argc, char* argv[])
       , false
   };
   const command_line::arg_descriptor<std::string> arg_database = {
-    "database", "available: memory, lmdb"
+    "database", available_dbs.c_str(), default_db_type
-      , default_db_engine
   };
   const command_line::arg_descriptor<bool> arg_verify =  {"verify",
     "Verify blocks and transactions during import", true};
@@ -648,8 +737,6 @@ int main(int argc, char* argv[])
     }
   }
 
-  std::vector<std::string> db_engines {"memory", "lmdb"};
-
   opt_testnet = command_line::get_arg(vm, arg_testnet_on);
   auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
   m_config_folder = command_line::get_arg(vm, data_dir_arg);
@@ -677,23 +764,34 @@ int main(int argc, char* argv[])
   }
 
 
-  std::string db_engine;
+  std::string db_type;
-  int mdb_flags = 0;
+  std::string db_engine_compiled;
+  int db_flags = 0;
   int res = 0;
-  res = parse_db_arguments(db_arg_str, db_engine, mdb_flags);
+  res = parse_db_arguments(db_arg_str, db_type, db_flags);
   if (res)
   {
     std::cerr << "Error parsing database argument(s)" << ENDL;
     return 1;
   }
 
-  if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end())
+  if (db_types_all.count(db_type) == 0)
   {
-    std::cerr << "Invalid database engine: " << db_engine << std::endl;
+    std::cerr << "Invalid database type: " << db_type << std::endl;
     return 1;
   }
 
-  LOG_PRINT_L0("database: " << db_engine);
+  if ((db_type == "lmdb") || (db_type == "berkeley"))
+  {
+    db_engine_compiled = "blockchain_db";
+  }
+  else
+  {
+    db_engine_compiled = "memory";
+  }
+
+  LOG_PRINT_L0("database: " << db_type);
+  LOG_PRINT_L0("database flags: " << db_flags);
   LOG_PRINT_L0("verify:  " << std::boolalpha << opt_verify << std::noboolalpha);
   if (opt_batch)
   {
@@ -722,31 +820,31 @@ int main(int argc, char* argv[])
 
   // for multi_db_runtime:
 #if !defined(BLOCKCHAIN_DB)
-  if (db_engine == "lmdb")
+  if (db_type == "lmdb" || db_type == "berkeley")
   {
-    fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags);
+    fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags);
     import_from_file(simple_core, import_file_path, block_stop);
   }
-  else if (db_engine == "memory")
+  else if (db_type == "memory")
   {
     fake_core_memory simple_core(m_config_folder, opt_testnet);
     import_from_file(simple_core, import_file_path, block_stop);
   }
   else
   {
-    std::cerr << "database engine unrecognized" << ENDL;
+    std::cerr << "database type unrecognized" << ENDL;
     return 1;
   }
 
   // for multi_db_compile:
 #else
-  if (db_engine != default_db_engine)
+  if (db_engine_compiled != default_db_engine_compiled)
   {
-    std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl;
+    std::cerr << "Invalid database type for compiled version: " << db_type << std::endl;
     return 1;
   }
 #if BLOCKCHAIN_DB == DB_LMDB
-  fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags);
+  fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags);
 #else
   fake_core_memory simple_core(m_config_folder, opt_testnet);
 #endif

src/blockchain_utilities/fake_core.h

@@ -33,6 +33,7 @@
 #include "cryptonote_core/blockchain_storage.h" // in-memory DB
 #include "blockchain_db/blockchain_db.h"
 #include "blockchain_db/lmdb/db_lmdb.h"
+#include "blockchain_db/berkeleydb/db_bdb.h"
 
 using namespace cryptonote;
 
@@ -47,7 +48,7 @@ namespace
 
 #if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB
 
-struct fake_core_lmdb
+struct fake_core_db
 {
   Blockchain m_storage;
   HardFork* m_hardfork = nullptr;
@@ -57,15 +58,26 @@ struct fake_core_lmdb
 
   // for multi_db_runtime:
 #if !defined(BLOCKCHAIN_DB)
-  fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool)
+  fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(&m_storage), m_storage(m_pool)
   // for multi_db_compile:
 #else
-  fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool)
+  fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(m_storage), m_storage(m_pool)
 #endif
   {
     m_pool.init(path.string());
 
-    BlockchainDB* db = new BlockchainLMDB();
+    BlockchainDB* db = nullptr;
+    if (db_type == "lmdb")
+      db = new BlockchainLMDB();
+#if defined(BERKELEY_DB)
+    else if (db_type == "berkeley")
+      db = new BlockchainBDB();
+#endif
+    else
+    {
+      LOG_ERROR("Attempted to use non-existent database type: " << db_type);
+      throw std::runtime_error("Attempting to use non-existent database type");
+    }
 
     boost::filesystem::path folder(path);
 
@@ -76,7 +88,7 @@ struct fake_core_lmdb
     const std::string filename = folder.string();
     try
     {
-      db->open(filename, mdb_flags);
+      db->open(filename, db_flags);
     }
     catch (const std::exception& e)
     {
@@ -96,7 +108,7 @@ struct fake_core_lmdb
     support_batch = true;
     support_add_block = true;
   }
-  ~fake_core_lmdb()
+  ~fake_core_db()
   {
     m_storage.get_db().check_hard_fork_info();
     m_storage.deinit();