From d06202dcc07b924103b93c7068a3fdb1499c3784 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 15 Aug 2022 11:42:19 +0000 Subject: [PATCH] blockchain_prune: faster on my anecdotal SSD, goes from about 9 hours to 1h20. --- src/blockchain_utilities/blockchain_prune.cpp | 128 ++++++++++++------ 1 file changed, 87 insertions(+), 41 deletions(-) diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp index 4a91cf7cc..a78d7ada9 100644 --- a/src/blockchain_utilities/blockchain_prune.cpp +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -52,9 +52,11 @@ using namespace cryptonote; static std::string db_path; // default to fast:1 -static uint64_t records_per_sync = 128; +static uint64_t records_per_sync = 16 * 65536; static const size_t slack = 512 * 1024 * 1024; +static std::vector is_v1; + static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name) { std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string()); @@ -89,6 +91,14 @@ static void close(MDB_env *env) mdb_env_close(env); } +static void mark_v1_tx(const MDB_val &k, const MDB_val &v) +{ + const uint64_t tx_id = *(const uint64_t*)k.mv_data; + if (tx_id >= is_v1.size()) + is_v1.resize(tx_id + 1, false); + is_v1[tx_id] = cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + static void add_size(MDB_env *env, uint64_t bytes) { try @@ -136,7 +146,7 @@ static void check_resize(MDB_env *env, size_t bytes) add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize); } -static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes) +static bool resize_point(size_t &nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes) { if (nrecords % records_per_sync && bytes <= slack / 2) return false; @@ -146,10 +156,11 @@ static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &b dbr = mdb_txn_begin(env, NULL, 0, txn); if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); bytes = 0; + nrecords = 0; return true; } -static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0) +static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0, void (*f)(const MDB_val&, const MDB_val&) = 0) { MDB_dbi dbi0, dbi1; MDB_txn *txn0, *txn1; @@ -200,6 +211,11 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned dbr = mdb_cursor_open(txn1, dbi1, &cur1); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + if (flags & MDB_DUPSORT) + putflags |= MDB_APPENDDUP; + else + putflags |= MDB_APPEND; + MDB_val k; MDB_val v; MDB_cursor_op op = MDB_FIRST; @@ -214,7 +230,8 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret))); bytes += k.mv_size + v.mv_size; - if (resize_point(++nrecords, env1, &txn1, bytes)) + ++nrecords; + if (resize_point(nrecords, env1, &txn1, bytes)) { dbr = mdb_cursor_open(txn1, dbi1, &cur1); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); @@ -223,6 +240,9 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned ret = mdb_cursor_put(cur1, &k, &v, putflags); if (ret) throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret))); + + if (f) + (*f)(k, v); } mdb_cursor_close(cur1); @@ -235,17 +255,6 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned mdb_dbi_close(env0, dbi0); } -static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) -{ - MDB_val v; - int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); - if (ret) - throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret))); - if (v.mv_size == 0) - throw std::runtime_error("Invalid transaction pruned data"); - return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); -} - static void prune(MDB_env *env0, MDB_env *env1) { MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties; @@ -324,7 +333,10 @@ static void prune(MDB_env *env0, MDB_env *env1) mdb_dbi_close(env0, dbi0_blocks); const uint64_t blockchain_height = stats.ms_entries; size_t nrecords = 0, bytes = 0; + std::vector prunable_needed; + // go through all txes tx indices, recording which ones should have their prunable part retained + MINFO("Marking prunable txes"); MDB_cursor_op op = MDB_FIRST; while (1) { @@ -336,7 +348,8 @@ static void prune(MDB_env *env0, MDB_env *env1) const txindex *ti = (const txindex*)v.mv_data; const uint64_t block_height = ti->data.block_id; - MDB_val_set(kk, ti->data.tx_id); + const uint64_t tx_id = ti->data.tx_id; + MDB_val_set(kk, tx_id); if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) { MDEBUG(block_height << "/" << blockchain_height << " is in tip"); @@ -344,22 +357,23 @@ static void prune(MDB_env *env0, MDB_env *env1) dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0); if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr))); bytes += kk.mv_size + vv.mv_size; - } - if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk)) - { - MDB_val vv; - dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET); - if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr))); - bytes += kk.mv_size + vv.mv_size; - if (resize_point(++nrecords, env1, &txn1, bytes)) + + ++nrecords; + if (resize_point(nrecords, env1, &txn1, bytes)) { dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); } - dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0); - if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr))); + } + if (tx_id >= is_v1.size()) + throw std::runtime_error("tx_id out of range of is_v1 vector"); + if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1[tx_id]) + { + if (tx_id >= prunable_needed.size()) + prunable_needed.resize(tx_id + 1, false); + prunable_needed[tx_id] = true; } else { @@ -367,6 +381,37 @@ static void prune(MDB_env *env0, MDB_env *env1) } } + // go through prunable parts, carrying over those we need + MINFO("Copying retained prunable data"); + op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur0_txs_prunable, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret))); + + const uint64_t tx_id = *(const uint64_t*)k.mv_data; + if (tx_id >= prunable_needed.size()) + throw std::runtime_error("tx_id out of range of prunable_needed vector"); + if (prunable_needed[tx_id]) + { + dbr = mdb_cursor_put(cur1_txs_prunable, &k, &v, MDB_APPEND); + if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr))); + + bytes += k.mv_size + v.mv_size; + ++nrecords; + if (resize_point(nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + } + } + mdb_cursor_close(cur1_txs_prunable_tip); mdb_cursor_close(cur1_txs_prunable); mdb_cursor_close(cur0_txs_prunable); @@ -419,7 +464,7 @@ static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags) else if(options[0] == "fastest") { db_flags = DBF_FASTEST; - records_per_sync = 1000; // default to fastest:async:1000 + // default to fastest:async:N } else return false; @@ -455,7 +500,7 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor arg_db_sync_mode = { "db-sync-mode" , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." - , "fast:1000" + , "fast:" + std::to_string(records_per_sync) }; const command_line::arg_descriptor arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"}; @@ -601,26 +646,27 @@ int main(int argc, char* argv[]) MDB_env *env0 = NULL, *env1 = NULL; open(env0, paths[0], db_flags, true); open(env1, paths[1], db_flags, false); - copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND); - copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "blocks", MDB_INTEGERKEY, 0); + copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); //copy_table(env0, env1, "txs", MDB_INTEGERKEY); - copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND); - copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND); + copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, 0, NULL, &mark_v1_tx); + copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0); // not copied: prunable, prunable_tip copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND); - copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); - copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); - copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, 0); + copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_meta", 0, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_blob", 0, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "alt_blocks", 0, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, 0); copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string); if (already_pruned) { - copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64); - copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, 0, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); } else {