Mirrors
/
wownero
mirror of https://git.wownero.com/wownero/wownero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

blockchain_blackball: fix chain reaction phase in incremental mode 

It makes it a lot slower, unfortunately
This commit is contained in:
moneromooo-monero 2018-08-12 21:46:02 +00:00
parent 2b2a681b01
commit d6d276c604
No known key found for this signature in database
GPG Key ID: 686F07454D6CEFC3
1 changed files with 35 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
src/blockchain_utilities/blockchain_blackball.cpp
View File

@ -60,7 +60,6 @@ static MDB_dbi dbi_outputs;
static MDB_dbi dbi_processed_txidx; static MDB_dbi dbi_processed_txidx;
static MDB_dbi dbi_spent; static MDB_dbi dbi_spent;
static MDB_dbi dbi_ring_instances; static MDB_dbi dbi_ring_instances;
static MDB_dbi dbi_newly_spent;
static MDB_dbi dbi_stats; static MDB_dbi dbi_stats;
static MDB_env *env = NULL; static MDB_env *env = NULL;
@ -79,7 +78,6 @@ struct output_data
// processed_txidx: string -> uint64_t // processed_txidx: string -> uint64_t
// spent: 128 bits, zerokval // spent: 128 bits, zerokval
// ring_instances: vector<uint64_t> -> uint64_t // ring_instances: vector<uint64_t> -> uint64_t
// newly_spent: 128 bits, zerokval
// stats: string -> arbitrary // stats: string -> arbitrary
// //
@ -240,7 +238,7 @@ static void init(std::string cache_filename)
dbr = mdb_env_create(&env); dbr = mdb_env_create(&env);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
dbr = mdb_env_set_maxdbs(env, 7); dbr = mdb_env_set_maxdbs(env, 6);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
const std::string actual_filename = get_cache_filename(cache_filename); const std::string actual_filename = get_cache_filename(cache_filename);
dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664);
@ -270,10 +268,6 @@ static void init(std::string cache_filename)
dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn, "newly_spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_newly_spent);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_dupsort(txn, dbi_newly_spent, compare_double64);
dbr = mdb_dbi_open(txn, "stats", MDB_CREATE, &dbi_stats); dbr = mdb_dbi_open(txn, "stats", MDB_CREATE, &dbi_stats);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
@ -291,7 +285,6 @@ static void close()
mdb_dbi_close(env, dbi_processed_txidx); mdb_dbi_close(env, dbi_processed_txidx);
mdb_dbi_close(env, dbi_spent); mdb_dbi_close(env, dbi_spent);
mdb_dbi_close(env, dbi_ring_instances); mdb_dbi_close(env, dbi_ring_instances);
mdb_dbi_close(env, dbi_newly_spent);
mdb_dbi_close(env, dbi_stats); mdb_dbi_close(env, dbi_stats);
mdb_env_close(env); mdb_env_close(env);
env = NULL; env = NULL;
@ -423,7 +416,7 @@ static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v)
return c; return c;
} }
static uint64_t get_num_spent_outputs(bool newly) static uint64_t get_num_spent_outputs()
{ {
MDB_txn *txn; MDB_txn *txn;
bool tx_active = false; bool tx_active = false;
@ -434,7 +427,7 @@ static uint64_t get_num_spent_outputs(bool newly)
tx_active = true; tx_active = true;
MDB_cursor *cur; MDB_cursor *cur;
dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); dbr = mdb_cursor_open(txn, dbi_spent, &cur);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr)));
MDB_val k, v; MDB_val k, v;
mdb_size_t count = 0; mdb_size_t count = 0;
@ -454,10 +447,10 @@ static uint64_t get_num_spent_outputs(bool newly)
return count; return count;
} }
static void add_spent_output(MDB_txn *txn, const output_data &od, bool newly) static void add_spent_output(MDB_txn *txn, const output_data &od)
{ {
MDB_cursor *cur; MDB_cursor *cur;
int dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); int dbr = mdb_cursor_open(txn, dbi_spent, &cur);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr)));
MDB_val v = {sizeof(od), (void*)&od}; MDB_val v = {sizeof(od), (void*)&od};
dbr = mdb_cursor_put(cur, (MDB_val *)&zerokval, &v, MDB_NODUPDATA); dbr = mdb_cursor_put(cur, (MDB_val *)&zerokval, &v, MDB_NODUPDATA);
@ -465,10 +458,10 @@ static void add_spent_output(MDB_txn *txn, const output_data &od, bool newly)
mdb_cursor_close(cur); mdb_cursor_close(cur);
} }
static bool is_output_spent(MDB_txn *txn, const output_data &od, bool newly) static bool is_output_spent(MDB_txn *txn, const output_data &od)
{ {
MDB_cursor *cur; MDB_cursor *cur;
int dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); int dbr = mdb_cursor_open(txn, dbi_spent, &cur);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr)));
MDB_val v = {sizeof(od), (void*)&od}; MDB_val v = {sizeof(od), (void*)&od};
dbr = mdb_cursor_get(cur, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); dbr = mdb_cursor_get(cur, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
@ -478,10 +471,10 @@ static bool is_output_spent(MDB_txn *txn, const output_data &od, bool newly)
return spent; return spent;
} }
static std::vector<output_data> get_spent_outputs(MDB_txn *txn, bool newly) static std::vector<output_data> get_spent_outputs(MDB_txn *txn)
{ {
MDB_cursor *cur; MDB_cursor *cur;
int dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); int dbr = mdb_cursor_open(txn, dbi_spent, &cur);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr)));
MDB_val k, v; MDB_val k, v;
uint64_t count = 0; uint64_t count = 0;
@ -507,12 +500,6 @@ static std::vector<output_data> get_spent_outputs(MDB_txn *txn, bool newly)
return outs; return outs;
} }
static void clear_spent_outputs(MDB_txn *txn, bool newly)
{
int dbr = mdb_drop(txn, newly ? dbi_newly_spent : dbi_spent, 0);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to clear spent outputs: " + std::string(mdb_strerror(dbr)));
}
static uint64_t get_processed_txidx(const std::string &name) static uint64_t get_processed_txidx(const std::string &name)
{ {
MDB_txn *txn; MDB_txn *txn;
@ -899,7 +886,7 @@ int main(int argc, char* argv[])
size_t done = 0; size_t done = 0;
const uint64_t start_blackballed_outputs = get_num_spent_outputs(false); const uint64_t start_blackballed_outputs = get_num_spent_outputs();
tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0]))); tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0])));
@ -958,7 +945,7 @@ int main(int argc, char* argv[])
MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring");
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
blackballs.push_back(pkey); blackballs.push_back(pkey);
add_spent_output(txn, output_data(txin.amount, absolute[0]), true); add_spent_output(txn, output_data(txin.amount, absolute[0]));
inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1"); inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1");
} }
else if (n == 0 && instances == new_ring.size()) else if (n == 0 && instances == new_ring.size())
@ -969,7 +956,7 @@ int main(int argc, char* argv[])
MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings");
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
blackballs.push_back(pkey); blackballs.push_back(pkey);
add_spent_output(txn, output_data(txin.amount, absolute[o]), true); add_spent_output(txn, output_data(txin.amount, absolute[o]));
inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings");
} }
} }
@ -1002,7 +989,7 @@ int main(int argc, char* argv[])
MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element");
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
blackballs.push_back(pkey); blackballs.push_back(pkey);
add_spent_output(txn, output_data(txin.amount, common[0]), true); add_spent_output(txn, output_data(txin.amount, common[0]));
inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack"); inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack");
} }
else else
@ -1024,7 +1011,7 @@ int main(int argc, char* argv[])
ringdb.blackball(blackballs); ringdb.blackball(blackballs);
blackballs.clear(); blackballs.clear();
} }
set_processed_txidx(txn, canonical, start_idx); set_processed_txidx(txn, canonical, start_idx+1);
++records; ++records;
if (records >= records_per_sync) if (records >= records_per_sync)
@ -1052,9 +1039,19 @@ int main(int argc, char* argv[])
break; break;
} }
while (get_num_spent_outputs(true) != 0) std::vector<output_data> work_spent;
if (get_num_spent_outputs() > start_blackballed_outputs)
{ {
LOG_PRINT_L0("Secondary pass due to " << get_num_spent_outputs(true) << " newly found spent outputs"); MDB_txn *txn;
dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
work_spent = get_spent_outputs(txn);
mdb_txn_abort(txn);
}
while (!work_spent.empty())
{
LOG_PRINT_L0("Secondary pass on " << work_spent.size() << " spent outputs");
int dbr = resize_env(cache_dir.c_str()); int dbr = resize_env(cache_dir.c_str());
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr)));
@ -1063,14 +1060,10 @@ int main(int argc, char* argv[])
dbr = mdb_txn_begin(env, NULL, 0, &txn); dbr = mdb_txn_begin(env, NULL, 0, &txn);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
std::vector<output_data> work_spent = get_spent_outputs(txn, true);
clear_spent_outputs(txn, true);
for (const auto &od: work_spent)
add_spent_output(txn, od, false);
std::vector<crypto::public_key> blackballs; std::vector<crypto::public_key> blackballs;
for (const output_data &od: work_spent) std::vector<output_data> scan_spent = std::move(work_spent);
work_spent.clear();
for (const output_data &od: scan_spent)
{ {
std::vector<crypto::key_image> key_images = get_key_images(txn, od); std::vector<crypto::key_image> key_images = get_key_images(txn, od);
for (const crypto::key_image &ki: key_images) for (const crypto::key_image &ki: key_images)
@ -1083,18 +1076,19 @@ int main(int argc, char* argv[])
for (uint64_t out: absolute) for (uint64_t out: absolute)
{ {
output_data new_od(od.amount, out); output_data new_od(od.amount, out);
if (is_output_spent(txn, new_od, false)) if (is_output_spent(txn, new_od))
++known; ++known;
else else
last_unknown = out; last_unknown = out;
} }
if (known == absolute.size() - 1 && !is_output_spent(txn, output_data(od.amount, last_unknown), false) && !is_output_spent(txn, output_data(od.amount, last_unknown), true)) if (known == absolute.size() - 1 && !is_output_spent(txn, output_data(od.amount, last_unknown)))
{ {
const crypto::public_key pkey = get_output_key(cur0, od.amount, last_unknown); const crypto::public_key pkey = get_output_key(cur0, od.amount, last_unknown);
MINFO("Blackballing output " << pkey << ", due to being used in a " << MINFO("Blackballing output " << pkey << ", due to being used in a " <<
absolute.size() << "-ring where all other outputs are known to be spent"); absolute.size() << "-ring where all other outputs are known to be spent");
blackballs.push_back(pkey); blackballs.push_back(pkey);
add_spent_output(txn, output_data(od.amount, last_unknown), true); add_spent_output(txn, output_data(od.amount, last_unknown));
work_spent.push_back(output_data(od.amount, last_unknown));
inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction");
} }
} }
@ -1108,8 +1102,8 @@ int main(int argc, char* argv[])
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr)));
} }
uint64_t diff = get_num_spent_outputs(false) - start_blackballed_outputs; uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs;
LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs(false) << " total outputs blackballed"); LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs() << " total outputs blackballed");
MDB_txn *txn; MDB_txn *txn;
dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);