29 #include <unordered_map> 30 #include <unordered_set> 31 #include <boost/range/adaptor/transformed.hpp> 32 #include <boost/algorithm/string.hpp> 46 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY 47 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "bcutil" 49 namespace po = boost::program_options;
53 static bool stop_requested =
false;
54 static uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0;
55 static bool opt_cache_outputs =
false, opt_cache_txes =
false, opt_cache_blocks =
false;
64 template <
typename t_archive>
void serialize(t_archive &
a,
const unsigned int ver)
78 return a.amount ^
a.offset;
85 std::vector<std::pair<uint64_t, std::vector<uint64_t>>>
vin;
86 std::vector<crypto::public_key>
vout;
95 vin.reserve(tx.
vin.size());
96 for (
size_t ring = 0; ring < tx.
vin.size(); ++ring)
106 throw std::runtime_error(
"Bad vin type");
110 vout.reserve(tx.
vout.size());
111 for (
size_t out = 0; out < tx.
vout.size(); ++out)
115 const auto &txout = boost::get<cryptonote::txout_to_key>(tx.
vout[out].target);
116 vout.push_back(txout.key);
121 throw std::runtime_error(
"Bad vout type");
126 template <
typename t_archive>
void serialize(t_archive &
a,
const unsigned int ver)
137 std::unordered_map<crypto::hash, std::unordered_set<ancestor>>
ancestry;
139 std::unordered_map<crypto::hash, ::tx_data_t>
tx_cache;
144 template <
typename t_archive>
void serialize(t_archive &
a,
const unsigned int ver)
151 std::unordered_map<crypto::hash, cryptonote::transaction> old_tx_cache;
153 for (
const auto i: old_tx_cache)
154 tx_cache.insert(std::make_pair(i.first, ::
tx_data_t(i.second)));
162 std::unordered_map<uint64_t, cryptonote::block> old_block_cache;
164 block_cache.resize(old_block_cache.size());
165 for (
const auto i: old_block_cache)
166 block_cache[i.first] = i.second;
178 std::pair<std::unordered_map<ancestor, unsigned int>::iterator,
bool> p = ancestry.insert(std::make_pair(
ancestor{amount, offset}, 1));
185 static size_t get_full_ancestry(
const std::unordered_map<ancestor, unsigned int> &ancestry)
188 for (
const auto &i: ancestry)
193 static size_t get_deduplicated_ancestry(
const std::unordered_map<ancestor, unsigned int> &ancestry)
195 return ancestry.size();
198 static void add_ancestry(std::unordered_map<
crypto::hash, std::unordered_set<ancestor>> &ancestry,
const crypto::hash &txid,
const std::unordered_set<ancestor> &ancestors)
200 std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator,
bool> p = ancestry.insert(std::make_pair(txid, ancestors));
203 for (
const auto &e: ancestors)
204 p.first->second.insert(e);
210 std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator,
bool> p = ancestry.insert(std::make_pair(txid, std::unordered_set<ancestor>()));
211 p.first->second.insert(new_ancestor);
214 static std::unordered_set<ancestor> get_ancestry(
const std::unordered_map<
crypto::hash, std::unordered_set<ancestor>> &ancestry,
const crypto::hash &txid)
216 std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::const_iterator i = ancestry.find(txid);
217 if (i == ancestry.end())
221 return std::unordered_set<ancestor>();
241 if (opt_cache_blocks)
251 std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i =
state.tx_cache.find(txid);
253 if (i !=
state.tx_cache.end())
263 LOG_PRINT_L0(
"Failed to get txid " << txid <<
" from db");
274 state.tx_cache.insert(std::make_pair(txid, tx_data));
281 std::unordered_map<ancestor, crypto::hash>::const_iterator i =
state.output_cache.find({amount, offset});
282 if (i !=
state.output_cache.end())
291 if (!get_block_from_height(
state, db, od.
height, b))
298 const auto &txout = boost::get<cryptonote::txout_to_key>(b.
miner_tx.
vout[
out].target);
299 if (txout.key == od.
pubkey)
302 if (opt_cache_outputs)
303 state.output_cache.insert(std::make_pair(
ancestor{amount, offset}, txid));
316 if (!get_transaction(
state, db, block_txid, tx_data3))
319 for (
size_t out = 0;
out < tx_data3.vout.size(); ++
out)
321 if (tx_data3.vout[out] == od.
pubkey)
324 if (opt_cache_outputs)
325 state.output_cache.insert(std::make_pair(
ancestor{amount, offset}, txid));
333 int main(
int argc,
char* argv[])
342 available_dbs =
"available: " + available_dbs;
348 boost::filesystem::path output_file_path;
350 po::options_description desc_cmd_only(
"Command line options");
351 po::options_description desc_cmd_sett(
"Command line options and settings options");
354 "database", available_dbs.c_str(), default_db_type
382 po::options_description desc_options(
"Allowed options");
383 desc_options.add(desc_cmd_only).add(desc_cmd_sett);
385 po::variables_map vm;
388 auto parser = po::command_line_parser(argc, argv).options(desc_options);
389 po::store(parser.run(), vm);
399 std::cout << desc_options << std::endl;
425 if ((!opt_txid_string.empty()) + !!opt_height + !opt_output_string.empty() > 1)
427 std::cerr <<
"Only one of --txid, --height, --output can be given" << std::endl;
431 uint64_t output_amount = 0, output_offset = 0;
432 if (!opt_txid_string.empty())
436 std::cerr <<
"Invalid txid" << std::endl;
440 else if (!opt_output_string.empty())
442 if (sscanf(opt_output_string.c_str(),
"%" SCNu64 "/%" SCNu64, &output_amount, &output_offset) != 2)
444 std::cerr <<
"Invalid output" << std::endl;
452 std::cerr <<
"Invalid database type: " << db_type << std::endl;
467 LOG_PRINT_L0(
"Initializing source blockchain (BlockchainDB)");
468 std::unique_ptr<Blockchain> core_storage;
470 core_storage.reset(
new Blockchain(m_mempool));
474 LOG_ERROR(
"Attempted to use non-existent database type: " << db_type);
475 throw std::runtime_error(
"Attempting to use non-existent database type");
480 LOG_PRINT_L0(
"Loading blockchain from folder " << filename <<
" ...");
486 catch (
const std::exception& e)
491 r = core_storage->
init(db, net_type);
494 LOG_PRINT_L0(
"Source blockchain storage initialized OK");
496 std::vector<crypto::hash> start_txids;
500 const std::string state_file_path = (boost::filesystem::path(opt_data_dir) /
"ancestry-state.bin").
string();
501 LOG_PRINT_L0(
"Loading state data from " << state_file_path);
502 std::ifstream state_data_in;
503 state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in);
504 if (!state_data_in.fail())
511 catch (
const std::exception &e)
513 MERROR(
"Failed to load state data from " << state_file_path <<
", restarting from scratch");
516 state_data_in.close();
520 stop_requested =
true;
527 MINFO(
"Starting from height " <<
state.height);
528 state.block_cache.reserve(db_height);
531 size_t block_ancestry_size = 0;
540 if (opt_cache_blocks)
542 state.block_cache.resize(h + 1);
543 state.block_cache[h] = b;
545 std::vector<crypto::hash> txids;
547 if (opt_include_coinbase)
553 printf(
"%lu/%lu \r", (
unsigned long)h, (
unsigned long)db_height);
556 std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i =
state.tx_cache.find(txid);
558 if (i !=
state.tx_cache.end())
568 LOG_PRINT_L0(
"Failed to get txid " << txid <<
" from db");
579 state.tx_cache.insert(std::make_pair(txid, tx_data));
581 if (tx_data.coinbase)
583 add_ancestry(
state.ancestry, txid, std::unordered_set<ancestor>());
587 for (
size_t ring = 0; ring < tx_data.vin.size(); ++ring)
589 const uint64_t amount = tx_data.vin[ring].first;
590 const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second;
591 for (
uint64_t offset: absolute_offsets)
593 add_ancestry(
state.ancestry, txid,
ancestor{amount, offset});
597 if (!get_output_txid(
state, db, amount, offset, output_txid))
599 LOG_PRINT_L0(
"Output originating transaction not found");
602 add_ancestry(
state.ancestry, txid, get_ancestry(
state.ancestry, output_txid));
606 const size_t ancestry_size = get_ancestry(
state.ancestry, txid).size();
607 block_ancestry_size += ancestry_size;
608 MINFO(txid <<
": " << ancestry_size);
613 MINFO(
"Height " << h <<
": " << (block_ancestry_size / txids.size()) <<
" average over " << txids.size() << stats_msg);
620 LOG_PRINT_L0(
"Saving state data to " << state_file_path);
621 std::ofstream state_data_out;
622 state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc);
623 if (!state_data_out.fail())
630 catch (
const std::exception &e)
632 MERROR(
"Failed to save state data to " << state_file_path);
634 state_data_out.close();
639 if (
state.height < db_height)
641 MWARNING(
"The state file is only built up to height " <<
state.height <<
", but the blockchain reached height " << db_height);
642 MWARNING(
"You may want to run with --refresh if you want to get ancestry for newer data");
646 if (!opt_txid_string.empty())
648 start_txids.push_back(opt_txid);
650 else if (!opt_output_string.empty())
653 if (!get_output_txid(
state, db, output_amount, output_offset, txid))
658 start_txids.push_back(txid);
670 start_txids.push_back(txid);
673 if (start_txids.empty())
681 LOG_PRINT_L0(
"Checking ancestry for txid " << start_txid);
683 std::unordered_map<ancestor, unsigned int> ancestry;
685 std::list<crypto::hash> txids;
686 txids.push_back(start_txid);
687 while (!txids.empty())
696 if (!get_transaction(
state, db, txid, tx_data2))
699 const bool coinbase = tx_data2.coinbase;
703 for (
size_t ring = 0; ring < tx_data2.vin.size(); ++ring)
706 const uint64_t amount = tx_data2.vin[ring].first;
707 auto absolute_offsets = tx_data2.vin[ring].second;
708 for (
uint64_t offset: absolute_offsets)
710 add_ancestor(ancestry, amount, offset);
715 if (!get_output_txid(
state, db, amount, offset, output_txid))
717 LOG_PRINT_L0(
"Output originating transaction not found");
721 add_ancestry(
state.ancestry, txid, get_ancestry(
state.ancestry, output_txid));
722 txids.push_back(output_txid);
723 MDEBUG(
"adding txid: " << output_txid);
729 MINFO(
"Ancestry for " << start_txid <<
": " << get_deduplicated_ancestry(ancestry) <<
" / " << get_full_ancestry(ancestry));
730 for (
const auto &i: ancestry)
739 if (opt_show_cache_stats)
741 <<
"%, blocks " <<
std::to_string(cached_blocks*100./total_blocks)
742 <<
"%, outputs " <<
std::to_string(cached_outputs*100./total_outputs)
const char *const ELECTRONEUM_RELEASE_NAME
std::vector< crypto::hash > tx_hashes
virtual std::string get_db_name() const =0
gets the name of the folder the BlockchainDB's file(s) should be in
virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t &height) const =0
fetch a block blob by height
int main(int argc, char *argv[])
void serialize(t_archive &a, const unsigned int ver)
std::vector< std::pair< uint64_t, std::vector< uint64_t > > > vin
std::string print_etn(uint64_t amount, unsigned int decimal_point)
#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message)
void mlog_set_log(const char *log)
#define CATCH_ENTRY(location, return_val)
std::string mlog_get_default_log_path(const char *default_filename)
void serialize(t_archive &a, const unsigned int ver)
std::unordered_map< ancestor, crypto::hash > output_cache
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size=MAX_LOG_FILE_SIZE, const std::size_t max_log_files=MAX_LOG_FILES)
provides the implementation of varint's
const command_line::arg_descriptor< bool, false > arg_stagenet_on
const arg_descriptor< bool > arg_help
std::vector< uint64_t > key_offsets
std::string blockchain_db_types(const std::string &sep)
std::vector< uint64_t > relative_output_offsets_to_absolute(const std::vector< uint64_t > &off)
virtual void open(const std::string &filename, const int db_flags=0)=0
open a db, or create it if necessary.
virtual bool get_pruned_tx_blob(const crypto::hash &h, cryptonote::blobdata &tx) const =0
fetches the pruned transaction blob with the given hash
std::vector< tx_out > vout
Holds cryptonote related classes and helpers.
const char *const ELECTRONEUM_VERSION_FULL
bool blockchain_valid_db_type(const std::string &db_type)
std::vector< txin_v > vin
mdb_size_t count(MDB_cursor *cur)
const command_line::arg_descriptor< bool, false > arg_testnet_on
virtual output_data_t get_output_key(const uint64_t &amount, const uint64_t &index, bool include_commitmemt=true) const =0
get some of an output's data
BlockchainDB * new_db(const std::string &db_type)
bool handle_error_helper(const boost::program_options::options_description &desc, F parser)
std::unordered_map< crypto::hash, std::unordered_set< ancestor > > ancestry
const command_line::arg_descriptor< std::string > arg_log_level
crypto::public_key pubkey
the output's public key (for spend verification)
unsigned __int64 uint64_t
const command_line::arg_descriptor< std::string, false, true, 2 > arg_data_dir
BOOST_CLASS_VERSION(nodetool::node_server< cryptonote::t_cryptonote_protocol_handler< tests::proxy_core > >, 1)
uint64_t height
the height of the block which created the output
virtual uint64_t height() const =0
fetch the current blockchain height
bool parse_and_validate_tx_base_from_blob(const blobdata &tx_blob, transaction &tx)
void add_arg(boost::program_options::options_description &description, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg, bool unique=true)
const GenericPointer< typename T::ValueType > T2 T::AllocatorType & a
Transaction pool, handles transactions which are not part of a block.
The BlockchainDB backing store interface declaration/contract.
std::vector< crypto::public_key > vout
void serialize(t_archive &a, const unsigned int ver)
crypto::hash get_transaction_hash(const transaction &t)
T get_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, false, true > &arg)
std::vector< cryptonote::block > block_cache
size_t operator()(const ancestor &a) const
bool init(BlockchainDB *db, const network_type nettype=MAINNET, bool offline=false, const cryptonote::test_options *test_options=NULL, difficulty_type fixed_difficulty=0, const GetCheckpointsCallback &get_checkpoints=nullptr, bool ignore_bsig=false, bool fallback_to_pow=false)
Initialize the Blockchain state.
std::string to_string(t_connection_type type)
std::unordered_map< crypto::hash, ::tx_data_t > tx_cache
bool operator==(const ancestor &other) const
tx_data_t(const cryptonote::transaction &tx)
a struct containing output metadata
bool is_arg_defaulted(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
bool deinit()
Uninitializes the blockchain state.
bool parse_and_validate_block_from_blob(const blobdata &b_blob, block &b, crypto::hash *block_hash)