1066 available_dbs =
"available: " + available_dbs;
1072 boost::filesystem::path output_file_path;
1074 po::options_description desc_cmd_only(
"Command line options");
1075 po::options_description desc_cmd_sett(
"Command line options and settings options");
1077 "spent-output-db-dir",
"Specify spent output database directory",
1078 get_default_db_path(),
1082 "database", available_dbs.c_str(), default_db_type
1090 ,
"Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." 1095 const command_line::arg_descriptor<bool> arg_force_chain_reaction_pass = {
"force-chain-reaction-pass",
"Run the chain reaction pass even if no new blockchain data was processed"};
1110 po::options_description desc_options(
"Allowed options");
1111 desc_options.add(desc_cmd_only).add(desc_cmd_sett);
1113 po::positional_options_description positional_options;
1114 positional_options.add(arg_inputs.name, -1);
1116 po::variables_map vm;
1119 auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options);
1120 po::store(parser.run(), vm);
1130 std::cout << desc_options << std::endl;
1149 std::vector<std::pair<uint64_t, uint64_t>> extra_spent_outputs = extra_spent_list.empty() ? std::vector<std::pair<uint64_t, uint64_t>>() : load_outputs(extra_spent_list);
1154 std::cerr <<
"Invalid database type: " << db_type << std::endl;
1159 if (!parse_db_sync_mode(db_sync_mode))
1161 MERROR(
"Invalid db sync mode: " << db_sync_mode);
1172 const std::string cache_dir = (output_file_path /
"spent-outputs-cache").
string();
1179 const uint64_t start_blackballed_outputs = get_num_spent_outputs();
1183 bool stop_requested =
false;
1185 stop_requested =
true;
1188 int dbr = resize_env(cache_dir.c_str());
1196 open_db(inputs[0], &env0, &txn0, &cur0, &dbi0);
1198 if (!extra_spent_outputs.empty())
1200 MINFO(
"Adding " << extra_spent_outputs.size() <<
" extra spent outputs");
1208 std::vector<std::pair<uint64_t, uint64_t>> blackballs;
1209 for (
const std::pair<uint64_t, uint64_t> &output: extra_spent_outputs)
1211 if (!is_output_spent(cur,
output_data(output.first, output.second)))
1213 blackballs.push_back(output);
1214 if (add_spent_output(cur,
output_data(output.first, output.second)))
1215 inc_stat(txn, output.first ?
"pre-rct-extra" :
"rct-ring-extra");
1218 if (!blackballs.empty())
1220 ringdb.blackball(blackballs);
1228 for (
size_t n = 0; n < inputs.size(); ++n)
1230 const std::string canonical = boost::filesystem::canonical(inputs[n]).string();
1231 uint64_t start_idx = get_processed_txidx(canonical);
1232 if (n > 0 && start_idx == 0)
1234 start_idx = find_first_diverging_transaction(inputs[0], inputs[n]);
1235 LOG_PRINT_L0(
"First diverging transaction at " << start_idx);
1237 LOG_PRINT_L0(
"Reading blockchain from " << inputs[n] <<
" from " << start_idx);
1246 std::vector<std::pair<uint64_t, uint64_t>> blackballs;
1250 std::cout <<
"\r" << start_idx <<
"/" << n_txes <<
" \r" << std::flush;
1251 const bool miner_tx = tx.
vin.size() == 1 && tx.
vin[0].type() ==
typeid(
txin_gen);
1252 for (
const auto &in: tx.
vin)
1256 const auto &txin = boost::get<txin_to_key>(
in);
1257 if (opt_rct_only && txin.amount != 0)
1263 add_key_image(txn,
output_data(txin.amount, out), txin.k_image);
1265 std::vector<uint64_t> relative_ring;
1266 std::vector<uint64_t> new_ring = canonicalize(txin.key_offsets);
1267 const uint32_t ring_size = txin.key_offsets.size();
1268 const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring);
1269 uint64_t pa_total = 0, pa_spent = 0;
1271 get_per_amount_outputs(txn, txin.amount, pa_total, pa_spent);
1272 if (n == 0 && ring_size == 1)
1274 const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[0]);
1277 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to being used in a 1-ring");
1278 std::cout <<
"\r" << start_idx <<
"/" << n_txes <<
" \r" << std::flush;
1280 blackballs.push_back(output);
1281 if (add_spent_output(cur,
output_data(txin.amount, absolute[0])))
1282 inc_stat(txn, txin.amount ?
"pre-rct-ring-size-1" :
"rct-ring-size-1");
1284 else if (n == 0 && instances == new_ring.size())
1286 for (
size_t o = 0; o < new_ring.size(); ++o)
1288 const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]);
1291 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to being used in " << new_ring.size() <<
" identical " << new_ring.size() <<
"-rings");
1292 std::cout <<
"\r" << start_idx <<
"/" << n_txes <<
" \r" << std::flush;
1294 blackballs.push_back(output);
1295 if (add_spent_output(cur,
output_data(txin.amount, absolute[o])))
1296 inc_stat(txn, txin.amount ?
"pre-rct-duplicate-rings" :
"rct-duplicate-rings");
1299 else if (n == 0 && !opt_rct_only && pa_spent + 1 == pa_total)
1301 for (
size_t o = 0; o < pa_total; ++o)
1303 const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, o);
1306 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to as many outputs of that amount being spent as exist so far");
1307 std::cout <<
"\r" << start_idx <<
"/" << n_txes <<
" \r" << std::flush;
1309 blackballs.push_back(output);
1310 if (add_spent_output(cur,
output_data(txin.amount, o)))
1311 inc_stat(txn, txin.amount ?
"pre-rct-full-count" :
"rct-full-count");
1314 else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size())
1316 for (
size_t o = 0; o < new_ring.size(); ++o)
1318 const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]);
1321 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to being used in " << new_ring.size() <<
" subsets of " << new_ring.size() <<
"-rings");
1322 std::cout <<
"\r" << start_idx <<
"/" << n_txes <<
" \r" << std::flush;
1324 blackballs.push_back(output);
1325 if (add_spent_output(cur,
output_data(txin.amount, absolute[o])))
1326 inc_stat(txn, txin.amount ?
"pre-rct-subset-rings" :
"rct-subset-rings");
1329 else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring))
1331 MDEBUG(
"Key image " << txin.k_image <<
" already seen: rings " <<
1332 boost::join(relative_ring | boost::adaptors::transformed([](
uint64_t out){return std::to_string(out);}),
" ") <<
1333 ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](
uint64_t out){return std::to_string(out);}),
" "));
1334 std::cout <<
"\r" << start_idx <<
"/" << n_txes <<
" \r" << std::flush;
1335 if (relative_ring != txin.key_offsets)
1337 MDEBUG(
"Rings are different");
1338 std::cout <<
"\r" << start_idx <<
"/" << n_txes <<
" \r" << std::flush;
1341 std::vector<uint64_t> common;
1344 if (std::find(r1.begin(), r1.end(),
out) != r1.end())
1345 common.push_back(out);
1349 MERROR(
"Rings for the same key image are disjoint");
1350 std::cout <<
"\r" << start_idx <<
"/" << n_txes <<
" \r" << std::flush;
1352 else if (common.size() == 1)
1354 const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, common[0]);
1357 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to being used in rings with a single common element");
1358 std::cout <<
"\r" << start_idx <<
"/" << n_txes <<
" \r" << std::flush;
1360 blackballs.push_back(output);
1361 if (add_spent_output(cur,
output_data(txin.amount, common[0])))
1362 inc_stat(txn, txin.amount ?
"pre-rct-key-image-attack" :
"rct-key-image-attack");
1366 MDEBUG(
"The intersection has more than one element, it's still ok");
1367 std::cout <<
"\r" << start_idx <<
"/" << n_txes <<
" \r" << std::flush;
1368 for (
const auto &out: r0)
1369 if (std::find(common.begin(), common.end(),
out) != common.end())
1370 new_ring.push_back(out);
1377 set_relative_ring(txn, txin.k_image, new_ring);
1379 inc_per_amount_outputs(txn, txin.amount, 0, 1);
1382 set_processed_txidx(txn, canonical, start_idx+1);
1385 for (
const auto &out: tx.
vout)
1389 if (opt_rct_only && amount != 0)
1393 inc_per_amount_outputs(txn, amount, 1, 0);
1398 if (records >= records_per_sync)
1400 if (!blackballs.empty())
1402 ringdb.blackball(blackballs);
1408 int dbr = resize_env(cache_dir.c_str());
1419 MINFO(
"Stopping scan...");
1427 LOG_PRINT_L0(
"blockchain from " << inputs[n] <<
" processed till tx idx " << start_idx);
1432 std::vector<output_data> work_spent;
1435 goto skip_secondary_passes;
1437 if (opt_force_chain_reaction_pass || get_num_spent_outputs() > start_blackballed_outputs)
1442 work_spent = get_spent_outputs(txn);
1446 while (!work_spent.empty())
1448 LOG_PRINT_L0(
"Secondary pass on " << work_spent.size() <<
" spent outputs");
1450 int dbr = resize_env(cache_dir.c_str());
1460 std::vector<std::pair<uint64_t, uint64_t>> blackballs;
1461 std::vector<output_data> scan_spent =
std::move(work_spent);
1465 std::vector<crypto::key_image> key_images = get_key_images(txn, od);
1468 std::vector<uint64_t> relative_ring;
1476 if (is_output_spent(cur, new_od))
1481 if (known == absolute.size() - 1)
1483 const std::pair<uint64_t, uint64_t> output = std::make_pair(od.amount, last_unknown);
1486 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to being used in a " <<
1487 absolute.size() <<
"-ring where all other outputs are known to be spent");
1489 blackballs.push_back(output);
1490 if (add_spent_output(cur,
output_data(od.amount, last_unknown)))
1491 inc_stat(txn, od.amount ?
"pre-rct-chain-reaction" :
"rct-chain-reaction");
1492 work_spent.push_back(
output_data(od.amount, last_unknown));
1498 MINFO(
"Stopping secondary passes. Secondary passes are not incremental, they will re-run fully.");
1502 if (!blackballs.empty())
1504 ringdb.blackball(blackballs);
1512 skip_secondary_passes:
1513 uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs;
1514 LOG_PRINT_L0(
std::to_string(diff) <<
" new outputs marked as spent, " << get_num_spent_outputs() <<
" total outputs marked as spent");
1520 get_num_outputs(txn0, cur0, dbi0, pre_rct,
rct);
1521 MINFO(
"Total pre-rct outputs: " << pre_rct);
1522 MINFO(
"Total rct outputs: " <<
rct);
1523 static const struct {
const char *
key;
uint64_t base; } stat_keys[] = {
1524 {
"pre-rct-ring-size-1", pre_rct }, {
"rct-ring-size-1",
rct },
1525 {
"pre-rct-duplicate-rings", pre_rct }, {
"rct-duplicate-rings",
rct },
1526 {
"pre-rct-subset-rings", pre_rct }, {
"rct-subset-rings",
rct },
1527 {
"pre-rct-full-count", pre_rct }, {
"rct-full-count",
rct },
1528 {
"pre-rct-key-image-attack", pre_rct }, {
"rct-key-image-attack",
rct },
1529 {
"pre-rct-extra", pre_rct }, {
"rct-ring-extra",
rct },
1530 {
"pre-rct-chain-reaction", pre_rct }, {
"rct-chain-reaction",
rct },
1532 for (
const auto &
key: stat_keys)
1535 if (!get_stat(txn,
key.key, data))
1537 float percent =
key.base ? 100.0f * data /
key.base : 0.0f;
1538 MINFO(
key.key <<
": " << data <<
" (" << percent <<
"%)");
1542 if (!opt_export.empty())
1550 export_spent_outputs(cur, opt_export);
1555 LOG_PRINT_L0(
"Blockchain spent output data exported OK");
1556 close_db(env0, txn0, cur0, dbi0);
const char *const ELECTRONEUM_RELEASE_NAME
#define CHECK_AND_ASSERT_THROW_MES(expr, message)
void mdb_cursor_close(MDB_cursor *cursor)
Close a cursor handle.
int mdb_txn_commit(MDB_txn *txn)
Commit all the operations of a transaction into the database.
void mlog_set_log(const char *log)
#define CATCH_ENTRY(location, return_val)
std::string mlog_get_default_log_path(const char *default_filename)
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size=MAX_LOG_FILE_SIZE, const std::size_t max_log_files=MAX_LOG_FILES)
const command_line::arg_descriptor< std::string > arg_db_sync_mode
const arg_descriptor< bool > arg_help
std::string blockchain_db_types(const std::string &sep)
std::vector< uint64_t > relative_output_offsets_to_absolute(const std::vector< uint64_t > &off)
struct MDB_env MDB_env
Opaque structure for a database environment.
struct MDB_txn MDB_txn
Opaque structure for a transaction handle.
std::vector< tx_out > vout
const char *const ELECTRONEUM_VERSION_FULL
bool blockchain_valid_db_type(const std::string &db_type)
std::vector< txin_v > vin
bool handle_error_helper(const boost::program_options::options_description &desc, F parser)
const command_line::arg_descriptor< std::string > arg_log_level
std::vector< uint64_t > absolute_output_offsets_to_relative(const std::vector< uint64_t > &off)
unsigned __int64 uint64_t
unsigned int MDB_dbi
A handle for an individual database in the DB environment.
int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
Create a transaction for use with the environment.
void add_arg(boost::program_options::options_description &description, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg, bool unique=true)
void mdb_txn_abort(MDB_txn *txn)
Abandon all the operations of the transaction instead of saving them.
struct MDB_cursor MDB_cursor
Opaque structure for navigating through a database.
const T & move(const T &t)
T get_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, false, true > &arg)
char * mdb_strerror(int err)
Return a string describing a given error code.
std::string to_string(t_connection_type type)
bool is_arg_defaulted(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor)
Create a cursor handle.