Merge pull request #451
ecf0b83
db_bdb: add versioning, to detect incompatible format changes (moneromooo-monero)f7e9904
db_lmdb: add versioning, to detect incompatible format changes (moneromooo-monero)
This commit is contained in:
commit
3f6ce0c161
|
@ -38,6 +38,10 @@
|
|||
using epee::string_tools::pod_to_hex;
|
||||
#define DB_DEFAULT_TX (m_write_txn != nullptr ? *m_write_txn : (DbTxn*) nullptr)
|
||||
|
||||
// Increase when the DB changes in a non backward compatible way, and there
|
||||
// is no automatic conversion, so that a full resync is needed.
|
||||
#define VERSION 0
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
|
@ -121,6 +125,8 @@ const char* const BDB_SPENT_KEYS = "spent_keys";
|
|||
const char* const BDB_HF_STARTING_HEIGHTS = "hf_starting_heights";
|
||||
const char* const BDB_HF_VERSIONS = "hf_versions";
|
||||
|
||||
const char* const BDB_PROPERTIES = "properties";
|
||||
|
||||
const unsigned int MB = 1024 * 1024;
|
||||
// ND: FIXME: db keeps running out of locks when doing full syncs. Possible bug??? Set it to 5K for now.
|
||||
const unsigned int DB_MAX_LOCKS = 5000;
|
||||
|
@ -180,6 +186,22 @@ private:
|
|||
std::unique_ptr<char[]> m_data;
|
||||
};
|
||||
|
||||
template<>
|
||||
struct Dbt_copy<const char*>: public Dbt
|
||||
{
|
||||
Dbt_copy(const char *s) :
|
||||
m_data(strdup(s))
|
||||
{
|
||||
size_t len = strlen(s) + 1; // include the NUL, makes it easier for compare
|
||||
set_data(m_data.get());
|
||||
set_size(len);
|
||||
set_ulen(len);
|
||||
set_flags(DB_DBT_USERMEM);
|
||||
}
|
||||
private:
|
||||
std::unique_ptr<char[]> m_data;
|
||||
};
|
||||
|
||||
struct Dbt_safe : public Dbt
|
||||
{
|
||||
Dbt_safe()
|
||||
|
@ -793,6 +815,8 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags)
|
|||
m_hf_starting_heights = new Db(m_env, 0);
|
||||
m_hf_versions = new Db(m_env, 0);
|
||||
|
||||
m_properties = new Db(m_env, 0);
|
||||
|
||||
// Tell DB about Dbs that need duplicate support
|
||||
// Note: no need to tell about sorting,
|
||||
// as the default is insertion order, which we want
|
||||
|
@ -845,6 +869,8 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags)
|
|||
m_hf_starting_heights->open(txn, BDB_HF_STARTING_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0);
|
||||
m_hf_versions->open(txn, BDB_HF_VERSIONS, NULL, DB_RECNO, DB_CREATE, 0);
|
||||
|
||||
m_properties->open(txn, BDB_PROPERTIES, NULL, DB_HASH, DB_CREATE, 0);
|
||||
|
||||
txn.commit();
|
||||
|
||||
DB_BTREE_STAT* stats;
|
||||
|
@ -865,6 +891,56 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags)
|
|||
m_num_outputs = stats->bt_nkeys;
|
||||
delete stats;
|
||||
|
||||
// checks for compatibility
|
||||
bool compatible = true;
|
||||
|
||||
Dbt_copy<const char*> key("version");
|
||||
Dbt_copy<uint32_t> result;
|
||||
auto get_result = m_properties->get(DB_DEFAULT_TX, &key, &result, 0);
|
||||
if (get_result == 0)
|
||||
{
|
||||
if (result > VERSION)
|
||||
{
|
||||
LOG_PRINT_RED_L0("Existing BerkeleyDB database was made by a later version. We don't know how it will change yet.");
|
||||
compatible = false;
|
||||
}
|
||||
else if (VERSION > 0 && result < VERSION)
|
||||
{
|
||||
compatible = false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// if not found, but we're on version 0, it's fine. If the DB's empty, it's fine too.
|
||||
if (VERSION > 0 && m_height > 0)
|
||||
compatible = false;
|
||||
}
|
||||
|
||||
if (!compatible)
|
||||
{
|
||||
m_open = false;
|
||||
LOG_PRINT_RED_L0("Existing BerkeleyDB database is incompatible with this version.");
|
||||
LOG_PRINT_RED_L0("Please delete the existing database and resync.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (1 /* this can't be set readonly atm */)
|
||||
{
|
||||
// only write version on an empty DB
|
||||
if (m_height == 0)
|
||||
{
|
||||
Dbt_copy<const char*> k("version");
|
||||
Dbt_copy<uint32_t> v(VERSION);
|
||||
auto put_result = m_properties->put(DB_DEFAULT_TX, &k, &v, 0);
|
||||
if (put_result != 0)
|
||||
{
|
||||
m_open = false;
|
||||
LOG_PRINT_RED_L0("Failed to write version to database.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// run checkpoint thread
|
||||
m_run_checkpoint = true;
|
||||
m_checkpoint_thread.reset(new boost::thread(&BlockchainBDB::checkpoint_worker, this));
|
||||
|
@ -920,6 +996,8 @@ void BlockchainBDB::sync()
|
|||
|
||||
m_hf_starting_heights->sync(0);
|
||||
m_hf_versions->sync(0);
|
||||
|
||||
m_properties->sync(0);
|
||||
}
|
||||
catch (const std::exception& e)
|
||||
{
|
||||
|
@ -1000,6 +1078,9 @@ std::vector<std::string> BlockchainBDB::get_filenames() const
|
|||
m_hf_versions->get_dbname(pfname, pdbname);
|
||||
filenames.push_back(fname);
|
||||
|
||||
m_properties->get_dbname(pfname, pdbname);
|
||||
filenames.push_back(fname);
|
||||
|
||||
std::vector<std::string> full_paths;
|
||||
|
||||
for (auto& filename : filenames)
|
||||
|
|
|
@ -445,6 +445,8 @@ private:
|
|||
Db* m_hf_starting_heights;
|
||||
Db* m_hf_versions;
|
||||
|
||||
Db* m_properties;
|
||||
|
||||
uint64_t m_height;
|
||||
uint64_t m_num_outputs;
|
||||
std::string m_folder;
|
||||
|
|
|
@ -39,6 +39,10 @@
|
|||
|
||||
using epee::string_tools::pod_to_hex;
|
||||
|
||||
// Increase when the DB changes in a non backward compatible way, and there
|
||||
// is no automatic conversion, so that a full resync is needed.
|
||||
#define VERSION 0
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
|
@ -122,6 +126,19 @@ private:
|
|||
std::unique_ptr<char[]> data;
|
||||
};
|
||||
|
||||
template<>
|
||||
struct MDB_val_copy<const char*>: public MDB_val
|
||||
{
|
||||
MDB_val_copy(const char *s) :
|
||||
data(strdup(s))
|
||||
{
|
||||
mv_size = strlen(s) + 1; // include the NUL, makes it easier for compares
|
||||
mv_data = data.get();
|
||||
}
|
||||
private:
|
||||
std::unique_ptr<char[]> data;
|
||||
};
|
||||
|
||||
auto compare_uint64 = [](const MDB_val *a, const MDB_val *b)
|
||||
{
|
||||
const uint64_t va = *(const uint64_t*)a->mv_data;
|
||||
|
@ -154,6 +171,13 @@ int compare_hash32(const MDB_val *a, const MDB_val *b)
|
|||
return 0;
|
||||
}
|
||||
|
||||
int compare_string(const MDB_val *a, const MDB_val *b)
|
||||
{
|
||||
const char *va = (const char*) a->mv_data;
|
||||
const char *vb = (const char*) b->mv_data;
|
||||
return strcmp(va, vb);
|
||||
}
|
||||
|
||||
const char* const LMDB_BLOCKS = "blocks";
|
||||
const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps";
|
||||
const char* const LMDB_BLOCK_HEIGHTS = "block_heights";
|
||||
|
@ -178,6 +202,8 @@ const char* const LMDB_SPENT_KEYS = "spent_keys";
|
|||
const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights";
|
||||
const char* const LMDB_HF_VERSIONS = "hf_versions";
|
||||
|
||||
const char* const LMDB_PROPERTIES = "properties";
|
||||
|
||||
inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string)
|
||||
{
|
||||
if (mdb_dbi_open(txn, name, flags, &dbi))
|
||||
|
@ -1037,6 +1063,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
|
|||
lmdb_db_open(txn, LMDB_HF_STARTING_HEIGHTS, MDB_CREATE, m_hf_starting_heights, "Failed to open db handle for m_hf_starting_heights");
|
||||
lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions");
|
||||
|
||||
lmdb_db_open(txn, LMDB_PROPERTIES, MDB_CREATE, m_properties, "Failed to open db handle for m_properties");
|
||||
|
||||
mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
|
||||
mdb_set_dupsort(txn, m_tx_outputs, compare_uint64);
|
||||
mdb_set_compare(txn, m_spent_keys, compare_hash32);
|
||||
|
@ -1046,6 +1074,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
|
|||
mdb_set_compare(txn, m_tx_heights, compare_hash32);
|
||||
mdb_set_compare(txn, m_hf_starting_heights, compare_uint8);
|
||||
mdb_set_compare(txn, m_hf_versions, compare_uint64);
|
||||
mdb_set_compare(txn, m_properties, compare_string);
|
||||
|
||||
// get and keep current height
|
||||
MDB_stat db_stats;
|
||||
|
@ -1059,6 +1088,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
|
|||
throw0(DB_ERROR("Failed to query m_output_indices"));
|
||||
m_num_outputs = db_stats.ms_entries;
|
||||
|
||||
bool compatible = true;
|
||||
|
||||
// ND: This "new" version of the lmdb database is incompatible with
|
||||
// the previous version. Ensure that the output_keys database is
|
||||
// sizeof(output_data_t) in length. Otherwise, inform user and
|
||||
|
@ -1077,16 +1108,60 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
|
|||
|
||||
// LOG_PRINT_L0("Output keys size: " << v.mv_size);
|
||||
if(v.mv_size != sizeof(output_data_t))
|
||||
{
|
||||
txn.abort();
|
||||
mdb_env_close(m_env);
|
||||
m_open = false;
|
||||
LOG_PRINT_RED_L0("Existing lmdb database is incompatible with this version.");
|
||||
LOG_PRINT_RED_L0("Please delete the existing database and resync.");
|
||||
return;
|
||||
}
|
||||
compatible = false;
|
||||
}
|
||||
|
||||
MDB_val_copy<const char*> k("version");
|
||||
MDB_val v;
|
||||
auto get_result = mdb_get(txn, m_properties, &k, &v);
|
||||
if(get_result == MDB_SUCCESS)
|
||||
{
|
||||
if (*(const uint32_t*)v.mv_data > VERSION)
|
||||
{
|
||||
LOG_PRINT_RED_L0("Existing lmdb database was made by a later version. We don't know how it will change yet.");
|
||||
compatible = false;
|
||||
}
|
||||
else if (VERSION > 0 && *(const uint32_t*)v.mv_data < VERSION)
|
||||
{
|
||||
compatible = false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// if not found, but we're on version 0, it's fine. If the DB's empty, it's fine too.
|
||||
if (VERSION > 0 && m_height > 0)
|
||||
compatible = false;
|
||||
}
|
||||
|
||||
if (!compatible)
|
||||
{
|
||||
txn.abort();
|
||||
mdb_env_close(m_env);
|
||||
m_open = false;
|
||||
LOG_PRINT_RED_L0("Existing lmdb database is incompatible with this version.");
|
||||
LOG_PRINT_RED_L0("Please delete the existing database and resync.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!(mdb_flags & MDB_RDONLY))
|
||||
{
|
||||
// only write version on an empty DB
|
||||
if (m_height == 0)
|
||||
{
|
||||
MDB_val_copy<const char*> k("version");
|
||||
MDB_val_copy<uint32_t> v(VERSION);
|
||||
auto put_result = mdb_put(txn, m_properties, &k, &v, 0);
|
||||
if (put_result != MDB_SUCCESS)
|
||||
{
|
||||
txn.abort();
|
||||
mdb_env_close(m_env);
|
||||
m_open = false;
|
||||
LOG_PRINT_RED_L0("Failed to write version to database.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// commit the transaction
|
||||
txn.commit();
|
||||
|
||||
|
|
|
@ -306,6 +306,8 @@ private:
|
|||
MDB_dbi m_hf_starting_heights;
|
||||
MDB_dbi m_hf_versions;
|
||||
|
||||
MDB_dbi m_properties;
|
||||
|
||||
uint64_t m_height;
|
||||
uint64_t m_num_outputs;
|
||||
std::string m_folder;
|
||||
|
|
Loading…
Reference in New Issue