Merge pull request #666

27f76e2 blockchain_import: Build string for db type list (warptangent)
1aa8a9d blockchain_import: Add mode argument representing multiple DB flags (warptangent)
cffc411 blockchain_import: Support BerkeleyDB (warptangent)
19c1aaa blockchain_import: Add database type argument (warptangent)
0fedce0 blockchain_import: Make LMDB-specific names general (warptangent)
This commit is contained in:
Riccardo Spagni 2016-02-17 17:09:21 +02:00
commit 3225a7d28d
No known key found for this signature in database
GPG Key ID: 55432DF31CCD4FCD
2 changed files with 160 additions and 50 deletions

View File

@ -38,6 +38,7 @@
#include "serialization/binary_utils.h" // dump_binary(), parse_binary() #include "serialization/binary_utils.h" // dump_binary(), parse_binary()
#include "serialization/json_utils.h" // dump_json() #include "serialization/json_utils.h" // dump_json()
#include "include_base_utils.h" #include "include_base_utils.h"
#include "blockchain_db/db_types.h"
#include <lmdb.h> // for db flag arguments #include <lmdb.h> // for db flag arguments
@ -77,12 +78,48 @@ using namespace cryptonote;
using namespace epee; using namespace epee;
int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags) std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim)
{
std::string result;
std::ostringstream s;
std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim));
result = s.str();
if (result.length() > 0)
result.erase(result.end()-strlen(delim), result.end());
return result;
}
// db_type: lmdb, berkeley
// db_mode: safe, fast, fastest
int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode)
{
uint64_t BDB_FAST_MODE = 0;
uint64_t BDB_FASTEST_MODE = 0;
uint64_t BDB_SAFE_MODE = 0;
#if defined(BERKELEY_DB)
BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC;
BDB_FASTEST_MODE = DB_TXN_NOSYNC;
BDB_SAFE_MODE = DB_TXN_SYNC;
#endif
int db_flags = 0;
bool islmdb = db_type == "lmdb";
if (db_mode == "safe")
db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE;
else if (db_mode == "fast")
db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE;
else if (db_mode == "fastest")
db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE;
return db_flags;
}
int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& db_flags)
{ {
std::vector<std::string> db_args; std::vector<std::string> db_args;
boost::split(db_args, db_arg_str, boost::is_any_of("#")); boost::split(db_args, db_arg_str, boost::is_any_of("#"));
db_engine = db_args.front(); db_type = db_args.front();
boost::algorithm::trim(db_engine); boost::algorithm::trim(db_type);
if (db_args.size() == 1) if (db_args.size() == 1)
{ {
@ -94,28 +131,73 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, in
return 1; return 1;
} }
#if !defined(BERKELEY_DB)
if (db_type == "berkeley")
{
LOG_ERROR("BerkeleyDB support disabled.");
return false;
}
#endif
std::string db_arg_str2 = db_args[1]; std::string db_arg_str2 = db_args[1];
boost::split(db_args, db_arg_str2, boost::is_any_of(",")); boost::split(db_args, db_arg_str2, boost::is_any_of(","));
for (auto& it : db_args)
// optionally use a composite mode instead of individual flags
const std::unordered_set<std::string> db_modes {"safe", "fast", "fastest"};
std::string db_mode;
if (db_args.size() == 1)
{ {
boost::algorithm::trim(it); if (db_modes.count(db_args[0]) > 0)
if (it.empty())
continue;
LOG_PRINT_L1("LMDB flag: " << it);
if (it == "nosync")
mdb_flags |= MDB_NOSYNC;
else if (it == "nometasync")
mdb_flags |= MDB_NOMETASYNC;
else if (it == "writemap")
mdb_flags |= MDB_WRITEMAP;
else if (it == "mapasync")
mdb_flags |= MDB_MAPASYNC;
else if (it == "nordahead")
mdb_flags |= MDB_NORDAHEAD;
else
{ {
std::cerr << "unrecognized database flag: " << it << ENDL; db_mode = db_args[0];
return 1; }
}
if (! db_mode.empty())
{
db_flags = get_db_flags_from_mode(db_type, db_mode);
}
else
{
for (auto& it : db_args)
{
boost::algorithm::trim(it);
if (it.empty())
continue;
if (db_type == "lmdb")
{
LOG_PRINT_L1("LMDB flag: " << it);
if (it == "nosync")
db_flags |= MDB_NOSYNC;
else if (it == "nometasync")
db_flags |= MDB_NOMETASYNC;
else if (it == "writemap")
db_flags |= MDB_WRITEMAP;
else if (it == "mapasync")
db_flags |= MDB_MAPASYNC;
else if (it == "nordahead")
db_flags |= MDB_NORDAHEAD;
else
{
std::cerr << "unrecognized database flag: " << it << ENDL;
return 1;
}
}
#if defined(BERKELEY_DB)
else if (db_type == "berkeley")
{
if (it == "txn_write_nosync")
db_flags = DB_TXN_WRITE_NOSYNC;
else if (it == "txn_nosync")
db_flags = DB_TXN_NOSYNC;
else if (it == "txn_sync")
db_flags = DB_TXN_SYNC;
else
{
std::cerr << "unrecognized database flag: " << it << ENDL;
return 1;
}
}
#endif
} }
} }
return 0; return 0;
@ -131,7 +213,7 @@ int pop_blocks(FakeCore& simple_core, int num_blocks)
if (simple_core.support_batch) if (simple_core.support_batch)
use_batch = true; use_batch = true;
else else
LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
} }
if (use_batch) if (use_batch)
@ -176,11 +258,11 @@ template <typename FakeCore>
int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0) int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0)
{ {
#if !defined(BLOCKCHAIN_DB) #if !defined(BLOCKCHAIN_DB)
static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value, static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_db, FakeCore>::value,
"FakeCore constraint error"); "FakeCore constraint error");
#endif #endif
#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
if (std::is_same<fake_core_lmdb, FakeCore>::value) if (std::is_same<fake_core_db, FakeCore>::value)
{ {
// Reset stats, in case we're using newly created db, accumulating stats // Reset stats, in case we're using newly created db, accumulating stats
// from addition of genesis block. // from addition of genesis block.
@ -250,7 +332,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
if (simple_core.support_batch) if (simple_core.support_batch)
use_batch = true; use_batch = true;
else else
LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
} }
if (use_batch) if (use_batch)
@ -530,11 +612,19 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
int main(int argc, char* argv[]) int main(int argc, char* argv[])
{ {
#if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY) #if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY)
std::string default_db_engine = "memory"; std::string default_db_type = "memory";
std::string default_db_engine_compiled = "memory";
#else #else
std::string default_db_engine = "lmdb"; std::string default_db_type = "lmdb";
std::string default_db_engine_compiled = "blockchain_db";
#endif #endif
std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types;
db_types_all.insert("memory");
std::string available_dbs = join_set_strings(db_types_all, ", ");
available_dbs = "available: " + available_dbs;
uint32_t log_level = LOG_LEVEL_0; uint32_t log_level = LOG_LEVEL_0;
uint64_t num_blocks = 0; uint64_t num_blocks = 0;
uint64_t block_stop = 0; uint64_t block_stop = 0;
@ -566,8 +656,7 @@ int main(int argc, char* argv[])
, false , false
}; };
const command_line::arg_descriptor<std::string> arg_database = { const command_line::arg_descriptor<std::string> arg_database = {
"database", "available: memory, lmdb" "database", available_dbs.c_str(), default_db_type
, default_db_engine
}; };
const command_line::arg_descriptor<bool> arg_verify = {"verify", const command_line::arg_descriptor<bool> arg_verify = {"verify",
"Verify blocks and transactions during import", true}; "Verify blocks and transactions during import", true};
@ -648,8 +737,6 @@ int main(int argc, char* argv[])
} }
} }
std::vector<std::string> db_engines {"memory", "lmdb"};
opt_testnet = command_line::get_arg(vm, arg_testnet_on); opt_testnet = command_line::get_arg(vm, arg_testnet_on);
auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
m_config_folder = command_line::get_arg(vm, data_dir_arg); m_config_folder = command_line::get_arg(vm, data_dir_arg);
@ -677,23 +764,34 @@ int main(int argc, char* argv[])
} }
std::string db_engine; std::string db_type;
int mdb_flags = 0; std::string db_engine_compiled;
int db_flags = 0;
int res = 0; int res = 0;
res = parse_db_arguments(db_arg_str, db_engine, mdb_flags); res = parse_db_arguments(db_arg_str, db_type, db_flags);
if (res) if (res)
{ {
std::cerr << "Error parsing database argument(s)" << ENDL; std::cerr << "Error parsing database argument(s)" << ENDL;
return 1; return 1;
} }
if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end()) if (db_types_all.count(db_type) == 0)
{ {
std::cerr << "Invalid database engine: " << db_engine << std::endl; std::cerr << "Invalid database type: " << db_type << std::endl;
return 1; return 1;
} }
LOG_PRINT_L0("database: " << db_engine); if ((db_type == "lmdb") || (db_type == "berkeley"))
{
db_engine_compiled = "blockchain_db";
}
else
{
db_engine_compiled = "memory";
}
LOG_PRINT_L0("database: " << db_type);
LOG_PRINT_L0("database flags: " << db_flags);
LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha); LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha);
if (opt_batch) if (opt_batch)
{ {
@ -722,31 +820,31 @@ int main(int argc, char* argv[])
// for multi_db_runtime: // for multi_db_runtime:
#if !defined(BLOCKCHAIN_DB) #if !defined(BLOCKCHAIN_DB)
if (db_engine == "lmdb") if (db_type == "lmdb" || db_type == "berkeley")
{ {
fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags); fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags);
import_from_file(simple_core, import_file_path, block_stop); import_from_file(simple_core, import_file_path, block_stop);
} }
else if (db_engine == "memory") else if (db_type == "memory")
{ {
fake_core_memory simple_core(m_config_folder, opt_testnet); fake_core_memory simple_core(m_config_folder, opt_testnet);
import_from_file(simple_core, import_file_path, block_stop); import_from_file(simple_core, import_file_path, block_stop);
} }
else else
{ {
std::cerr << "database engine unrecognized" << ENDL; std::cerr << "database type unrecognized" << ENDL;
return 1; return 1;
} }
// for multi_db_compile: // for multi_db_compile:
#else #else
if (db_engine != default_db_engine) if (db_engine_compiled != default_db_engine_compiled)
{ {
std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl; std::cerr << "Invalid database type for compiled version: " << db_type << std::endl;
return 1; return 1;
} }
#if BLOCKCHAIN_DB == DB_LMDB #if BLOCKCHAIN_DB == DB_LMDB
fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags); fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags);
#else #else
fake_core_memory simple_core(m_config_folder, opt_testnet); fake_core_memory simple_core(m_config_folder, opt_testnet);
#endif #endif

View File

@ -33,6 +33,7 @@
#include "cryptonote_core/blockchain_storage.h" // in-memory DB #include "cryptonote_core/blockchain_storage.h" // in-memory DB
#include "blockchain_db/blockchain_db.h" #include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h" #include "blockchain_db/lmdb/db_lmdb.h"
#include "blockchain_db/berkeleydb/db_bdb.h"
using namespace cryptonote; using namespace cryptonote;
@ -47,7 +48,7 @@ namespace
#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB #if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB
struct fake_core_lmdb struct fake_core_db
{ {
Blockchain m_storage; Blockchain m_storage;
HardFork* m_hardfork = nullptr; HardFork* m_hardfork = nullptr;
@ -57,15 +58,26 @@ struct fake_core_lmdb
// for multi_db_runtime: // for multi_db_runtime:
#if !defined(BLOCKCHAIN_DB) #if !defined(BLOCKCHAIN_DB)
fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool) fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(&m_storage), m_storage(m_pool)
// for multi_db_compile: // for multi_db_compile:
#else #else
fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool) fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(m_storage), m_storage(m_pool)
#endif #endif
{ {
m_pool.init(path.string()); m_pool.init(path.string());
BlockchainDB* db = new BlockchainLMDB(); BlockchainDB* db = nullptr;
if (db_type == "lmdb")
db = new BlockchainLMDB();
#if defined(BERKELEY_DB)
else if (db_type == "berkeley")
db = new BlockchainBDB();
#endif
else
{
LOG_ERROR("Attempted to use non-existent database type: " << db_type);
throw std::runtime_error("Attempting to use non-existent database type");
}
boost::filesystem::path folder(path); boost::filesystem::path folder(path);
@ -76,7 +88,7 @@ struct fake_core_lmdb
const std::string filename = folder.string(); const std::string filename = folder.string();
try try
{ {
db->open(filename, mdb_flags); db->open(filename, db_flags);
} }
catch (const std::exception& e) catch (const std::exception& e)
{ {
@ -96,7 +108,7 @@ struct fake_core_lmdb
support_batch = true; support_batch = true;
support_add_block = true; support_add_block = true;
} }
~fake_core_lmdb() ~fake_core_db()
{ {
m_storage.get_db().check_hard_fork_info(); m_storage.get_db().check_hard_fork_info();
m_storage.deinit(); m_storage.deinit();