ringdb: allow blackballing many outputs at once

It cuts down on txn commits, and speeds up blackballing substantially
This commit is contained in:
moneromooo-monero 2018-08-12 21:41:44 +00:00
parent fad88e18a9
commit 50cb370d5b
No known key found for this signature in database
GPG Key ID: 686F07454D6CEFC3
4 changed files with 77 additions and 45 deletions

View File

@ -417,6 +417,7 @@ int main(int argc, char* argv[])
if (it != state.processed_heights.end()) if (it != state.processed_heights.end())
start_idx = it->second; start_idx = it->second;
LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx); LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx);
std::vector<crypto::public_key> blackballs;
for_all_transactions(inputs[n], start_idx, [&](const cryptonote::transaction_prefix &tx)->bool for_all_transactions(inputs[n], start_idx, [&](const cryptonote::transaction_prefix &tx)->bool
{ {
for (const auto &in: tx.vin) for (const auto &in: tx.vin)
@ -439,7 +440,7 @@ int main(int argc, char* argv[])
{ {
const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]); const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]);
MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring");
ringdb.blackball(pkey); blackballs.push_back(pkey);
newly_spent.insert(output_data(txin.amount, absolute[0])); newly_spent.insert(output_data(txin.amount, absolute[0]));
} }
else if (state.ring_instances[new_ring] == new_ring.size()) else if (state.ring_instances[new_ring] == new_ring.size())
@ -448,7 +449,7 @@ int main(int argc, char* argv[])
{ {
const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]); const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]);
MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings");
ringdb.blackball(pkey); blackballs.push_back(pkey);
newly_spent.insert(output_data(txin.amount, absolute[o])); newly_spent.insert(output_data(txin.amount, absolute[o]));
} }
} }
@ -476,7 +477,7 @@ int main(int argc, char* argv[])
{ {
const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]);
MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element");
ringdb.blackball(pkey); blackballs.push_back(pkey);
newly_spent.insert(output_data(txin.amount, common[0])); newly_spent.insert(output_data(txin.amount, common[0]));
} }
else else
@ -490,6 +491,11 @@ int main(int argc, char* argv[])
} }
} }
state.relative_rings[txin.k_image] = new_ring; state.relative_rings[txin.k_image] = new_ring;
if (!blackballs.empty())
{
ringdb.blackball(blackballs);
blackballs.clear();
}
} }
if (stop_requested) if (stop_requested)
{ {
@ -513,6 +519,7 @@ int main(int argc, char* argv[])
for (const auto &e: work_spent) for (const auto &e: work_spent)
state.spent.insert(e); state.spent.insert(e);
std::vector<crypto::public_key> blackballs;
for (const output_data &od: work_spent) for (const output_data &od: work_spent)
{ {
for (const crypto::key_image &ki: state.outputs[od]) for (const crypto::key_image &ki: state.outputs[od])
@ -533,13 +540,19 @@ int main(int argc, char* argv[])
const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown);
MINFO("Blackballing output " << pkey << ", due to being used in a " << MINFO("Blackballing output " << pkey << ", due to being used in a " <<
absolute.size() << "-ring where all other outputs are known to be spent"); absolute.size() << "-ring where all other outputs are known to be spent");
ringdb.blackball(pkey); blackballs.push_back(pkey);
newly_spent.insert(output_data(od.amount, last_unknown)); newly_spent.insert(output_data(od.amount, last_unknown));
} }
} }
} }
} }
if (!blackballs.empty())
{
ringdb.blackball(blackballs);
blackballs.clear();
}
LOG_PRINT_L0("Saving state data to " << state_file_path); LOG_PRINT_L0("Saving state data to " << state_file_path);
std::ofstream state_data_out; std::ofstream state_data_out;
state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc);

View File

@ -146,7 +146,7 @@ static int resize_env(MDB_env *env, const char *db_path, size_t needed)
MDB_stat mst; MDB_stat mst;
int ret; int ret;
needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB needed = std::max(needed, (size_t)(100ul * 1024 * 1024)); // at least 100 MB
ret = mdb_env_info(env, &mei); ret = mdb_env_info(env, &mei);
if (ret) if (ret)
@ -374,7 +374,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im
return true; return true;
} }
bool ringdb::blackball_worker(const crypto::public_key &output, int op) bool ringdb::blackball_worker(const std::vector<crypto::public_key> &outputs, int op)
{ {
MDB_txn *txn; MDB_txn *txn;
MDB_cursor *cursor; MDB_cursor *cursor;
@ -382,7 +382,9 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op)
bool tx_active = false; bool tx_active = false;
bool ret = true; bool ret = true;
dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack THROW_WALLET_EXCEPTION_IF(outputs.size() > 1 && op == BLACKBALL_QUERY, tools::error::wallet_internal_error, "Blackball query only makes sense for a single output");
dbr = resize_env(env, filename.c_str(), 32 * 2 * outputs.size()); // a pubkey, and some slack
THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr)));
dbr = mdb_txn_begin(env, NULL, 0, &txn); dbr = mdb_txn_begin(env, NULL, 0, &txn);
THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
@ -391,40 +393,49 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op)
MDB_val key = zerokeyval; MDB_val key = zerokeyval;
MDB_val data; MDB_val data;
data.mv_data = (void*)&output;
data.mv_size = sizeof(output);
switch (op) for (const crypto::public_key &output: outputs)
{ {
case BLACKBALL_BLACKBALL: data.mv_data = (void*)&output;
MDEBUG("Blackballing output " << output); data.mv_size = sizeof(output);
dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA);
if (dbr == MDB_KEYEXIST) switch (op)
dbr = 0; {
break; case BLACKBALL_BLACKBALL:
case BLACKBALL_UNBLACKBALL: MDEBUG("Blackballing output " << output);
MDEBUG("Unblackballing output " << output); dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA);
dbr = mdb_del(txn, dbi_blackballs, &key, &data); if (dbr == MDB_KEYEXIST)
if (dbr == MDB_NOTFOUND) dbr = 0;
dbr = 0; break;
break; case BLACKBALL_UNBLACKBALL:
case BLACKBALL_QUERY: MDEBUG("Unblackballing output " << output);
dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); dbr = mdb_del(txn, dbi_blackballs, &key, &data);
THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); if (dbr == MDB_NOTFOUND)
dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); dbr = 0;
THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); break;
ret = dbr != MDB_NOTFOUND; case BLACKBALL_QUERY:
if (dbr == MDB_NOTFOUND) dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor);
dbr = 0; THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr)));
mdb_cursor_close(cursor); dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH);
break; THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr)));
case BLACKBALL_CLEAR: ret = dbr != MDB_NOTFOUND;
dbr = mdb_drop(txn, dbi_blackballs, 0); if (dbr == MDB_NOTFOUND)
break; dbr = 0;
default: mdb_cursor_close(cursor);
THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); break;
case BLACKBALL_CLEAR:
break;
default:
THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op");
}
THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr)));
}
if (op == BLACKBALL_CLEAR)
{
dbr = mdb_drop(txn, dbi_blackballs, 0);
THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to clear blackballs table: " + std::string(mdb_strerror(dbr)));
} }
THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr)));
dbr = mdb_txn_commit(txn); dbr = mdb_txn_commit(txn);
THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr))); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr)));
@ -432,24 +443,32 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op)
return ret; return ret;
} }
bool ringdb::blackball(const std::vector<crypto::public_key> &outputs)
{
return blackball_worker(outputs, BLACKBALL_BLACKBALL);
}
bool ringdb::blackball(const crypto::public_key &output) bool ringdb::blackball(const crypto::public_key &output)
{ {
return blackball_worker(output, BLACKBALL_BLACKBALL); std::vector<crypto::public_key> outputs(1, output);
return blackball_worker(outputs, BLACKBALL_BLACKBALL);
} }
bool ringdb::unblackball(const crypto::public_key &output) bool ringdb::unblackball(const crypto::public_key &output)
{ {
return blackball_worker(output, BLACKBALL_UNBLACKBALL); std::vector<crypto::public_key> outputs(1, output);
return blackball_worker(outputs, BLACKBALL_UNBLACKBALL);
} }
bool ringdb::blackballed(const crypto::public_key &output) bool ringdb::blackballed(const crypto::public_key &output)
{ {
return blackball_worker(output, BLACKBALL_QUERY); std::vector<crypto::public_key> outputs(1, output);
return blackball_worker(outputs, BLACKBALL_QUERY);
} }
bool ringdb::clear_blackballs() bool ringdb::clear_blackballs()
{ {
return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR); return blackball_worker(std::vector<crypto::public_key>(), BLACKBALL_CLEAR);
} }
} }

View File

@ -50,12 +50,13 @@ namespace tools
bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative);
bool blackball(const crypto::public_key &output); bool blackball(const crypto::public_key &output);
bool blackball(const std::vector<crypto::public_key> &outputs);
bool unblackball(const crypto::public_key &output); bool unblackball(const crypto::public_key &output);
bool blackballed(const crypto::public_key &output); bool blackballed(const crypto::public_key &output);
bool clear_blackballs(); bool clear_blackballs();
private: private:
bool blackball_worker(const crypto::public_key &output, int op); bool blackball_worker(const std::vector<crypto::public_key> &outputs, int op);
private: private:
std::string filename; std::string filename;

View File

@ -6238,8 +6238,7 @@ bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &out
bool ret = true; bool ret = true;
if (!add) if (!add)
ret &= m_ringdb->clear_blackballs(); ret &= m_ringdb->clear_blackballs();
for (const auto &output: outputs) ret &= m_ringdb->blackball(outputs);
ret &= m_ringdb->blackball(output);
return ret; return ret;
} }
catch (const std::exception &e) { return false; } catch (const std::exception &e) { return false; }