35 #include <boost/filesystem.hpp> 36 #include <boost/algorithm/string.hpp> 37 #include <boost/chrono.hpp> 39 #include <openssl/sha.h> 52 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY 53 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "bcutil" 58 bool opt_batch =
true;
59 bool opt_verify =
true;
60 bool opt_resume =
true;
61 bool opt_testnet =
true;
62 bool opt_stagenet =
true;
75 uint64_t db_batch_size_verify = 5000;
82 namespace po = boost::program_options;
91 if (db_mode ==
"safe")
93 else if (db_mode ==
"fast")
95 else if (db_mode ==
"fastest")
102 std::vector<std::string> db_args;
103 boost::split(db_args, db_arg_str, boost::is_any_of(
"#"));
104 db_type = db_args.front();
107 if (db_args.size() == 1)
111 else if (db_args.size() > 2)
113 std::cerr <<
"unrecognized database argument format: " << db_arg_str <<
ENDL;
118 boost::split(db_args, db_arg_str2, boost::is_any_of(
","));
121 const std::unordered_set<std::string> db_modes {
"safe",
"fast",
"fastest"};
123 if (db_args.size() == 1)
125 if (db_modes.count(db_args[0]) > 0)
127 db_mode = db_args[0];
130 if (! db_mode.empty())
139 bool use_batch = opt_batch;
146 std::vector<transaction> popped_txs;
149 boost::scoped_ptr<boost::thread> t(
new boost::thread([&i, &
num_blocks]() {
150 time_t pop_start_time =
time(NULL);
151 time_t partial_pop_time;
152 double estimate_time;
155 partial_pop_time =
time(NULL);
158 estimate_time = (
num_blocks - i) * (((partial_pop_time - pop_start_time)*1000) / i);
159 estimate_time = ceil(estimate_time/60000);
163 std::cout <<
"\rPoping blocks from database (aprox. " << estimate_time <<
" minute(s) remaining): " << i + 1 <<
"/" <<
num_blocks <<
" ";
165 boost::this_thread::sleep_for(boost::chrono::milliseconds{200});
168 std::cout <<
"\r \r";
201 if (!force &&
blocks.size() < db_batch_size)
204 std::vector<block> pblocks;
207 MERROR(
"Failed to prepare to add blocks");
210 if (!pblocks.empty() && pblocks.size() !=
blocks.size())
212 MERROR(
"Unexpected parsed blocks size");
221 for(
auto& tx_blob: block_entry.txs)
227 MERROR(
"transaction verification failed, tx_id = " 240 if(bvc.m_verification_failed)
242 MERROR(
"Block verification failed, id = " 247 if(bvc.m_marked_as_orphaned)
249 MERROR(
"Block received at sync phase was marked as orphaned");
269 boost::filesystem::path fs_import_file_path(import_file_path);
270 boost::system::error_code ec;
271 if (!boost::filesystem::exists(fs_import_file_path, ec))
273 MFATAL(
"bootstrap file not found: " << fs_import_file_path);
277 uint64_t start_height = 1, seek_height;
281 seek_height = start_height;
286 MINFO(
"bootstrap file last block number: " << total_source_blocks-1 <<
" (zero-based height) total blocks: " << total_source_blocks);
288 if (total_source_blocks-1 <= start_height)
294 std::cout <<
"Preparing to read blocks..." <<
ENDL;
297 std::ifstream import_file;
298 import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in);
302 if (import_file.fail())
304 MFATAL(
"import_file.open() fail");
309 uint8_t major_version, minor_version;
325 block_stop = total_source_blocks - 1;
330 MINFO(
"start block: " << start_height <<
" stop block: " <<
333 bool use_batch = opt_batch && !opt_verify;
335 MINFO(
"Reading blockchain from bootstrap file...");
338 std::vector<block_complete_entry>
blocks;
343 import_file.seekg(pos);
344 bytes_read = bootstrap.
count_bytes(import_file, start_height-seek_height, h, q2);
357 pos = import_file.tellg();
358 bytes = bootstrap.
count_bytes(import_file, db_batch_size, h2, q2);
359 if (import_file.eof())
361 import_file.seekg(pos);
367 import_file.read(buffer1,
sizeof(chunk_size));
370 std::cout << refresh_string;
371 MINFO(
"End of file reached");
375 bytes_read +=
sizeof(chunk_size);
377 str1.assign(buffer1,
sizeof(chunk_size));
380 throw std::runtime_error(
"Error in deserialization of chunk size");
382 MDEBUG(
"chunk_size: " << chunk_size);
387 throw std::runtime_error(
"Aborting: chunk size exceeds buffer size");
393 else if (chunk_size == 0) {
394 MFATAL(
"ERROR: chunk_size == 0");
397 import_file.read(buffer_block, chunk_size);
399 if (import_file.eof())
401 std::cout << refresh_string;
402 MINFO(
"End of file reached - file was truncated");
408 MFATAL(
"ERROR: unexpected end of file: bytes read before error: " 409 << import_file.gcount() <<
" of chunk_size " << chunk_size);
413 bytes_read += chunk_size;
414 MDEBUG(
"Total bytes read: " << bytes_read);
418 std::cout << refresh_string <<
"block " << h-1
419 <<
" / " << block_stop
420 <<
"\r" << std::flush;
422 MINFO(
"Specified block number reached - stopping. block: " << h-1 <<
" total blocks: " << h);
429 str1.assign(buffer_block, chunk_size);
432 if (major_version == 0)
448 throw std::runtime_error(
"Error in deserialization of chunk");
450 int display_interval = 1000;
451 int progress_interval = 10;
456 if ((h-1) % display_interval == 0)
458 std::cout << refresh_string;
459 MDEBUG(
"loading block number " << h-1);
463 MDEBUG(
"loading block number " << h-1);
468 if ((h-1) % progress_interval == 0)
470 std::cout << refresh_string <<
"block " << h-1
471 <<
" / " << block_stop
472 <<
"\r" << std::flush;
479 std::vector<cryptonote::blobdata> txs;
480 for (
const auto &tx: bp.
txs)
495 std::vector<std::pair<transaction, blobdata>> txs;
496 std::vector<transaction> archived_txs;
498 archived_txs = bp.
txs;
512 txs.push_back(std::make_pair(tx,
tx_to_blob(tx)));
528 catch (
const std::exception& e)
530 std::cout << refresh_string;
531 MFATAL(
"Error adding block to blockchain: " << e.what());
538 if ((h-1) % db_batch_size == 0)
542 std::cout << refresh_string;
544 std::cout <<
ENDL <<
"[- batch commit at height " << h-1 <<
" -]" <<
ENDL;
546 pos = import_file.tellg();
547 bytes = bootstrap.
count_bytes(import_file, db_batch_size, h2, q2);
548 import_file.seekg(pos);
558 catch (
const std::exception& e)
560 std::cout << refresh_string;
561 MFATAL(
"exception while reading from file, height=" << h <<
": " << e.what());
590 MINFO(
"Number of blocks imported: " << num_imported);
593 MINFO(
"Finished at block: " << h-1 <<
" total blocks: " << h);
603 for(i = 0; i < SHA256_DIGEST_LENGTH; i++)
605 sprintf(outputBuffer + (i * 2),
"%02x",
hash[i]);
608 outputBuffer[64] = 0;
613 FILE* file = fopen(path,
"rb");
616 unsigned char hash[SHA256_DIGEST_LENGTH];
618 SHA256_Init(&sha256);
619 const int bufSize = 32768;
620 char* buffer = (
char*)std::malloc(bufSize);
623 while((bytesRead = fread(buffer, 1, bufSize, file)))
625 SHA256_Update(&sha256, buffer, bytesRead);
627 SHA256_Final(
hash, &sha256);
642 std::vector<std::string> records;
645 static const std::vector<std::string> dns_urls = {
646 "raw.electroneumpulse.com",
647 "raw.electroneumpulse.info",
648 "raw.electroneumpulse.net",
649 "raw.electroneumpulse.org" 664 if (strcmp(calc_hash, checksum.c_str()) != 0) {
665 MINFO(
"Invalid input-file checksum (" << calc_hash <<
"), expected: " << checksum);
669 }
catch(
const std::exception &ex) {
672 MINFO(
"Unable to verify input-file checksum due to: " << ex.what());
679 int main(
int argc,
char* argv[])
688 available_dbs =
"available: " + available_dbs;
700 po::options_description desc_cmd_only(
"Command line options");
701 po::options_description desc_cmd_sett(
"Command line options and settings options");
710 ,
"Count blocks in bootstrap file and exit" 714 "database", available_dbs.c_str(), default_db_type
717 "Blindly trust the import file and use potentially malicious blocks and transactions during import (only enable if you exported the file yourself)",
false};
719 "Batch transactions for faster import",
true};
721 "Resume from current height if output database already exists",
true};
736 desc_cmd_sett.add_options()
737 (arg_noverify.name,
make_semantic(arg_noverify), arg_noverify.description)
738 (arg_batch.name,
make_semantic(arg_batch), arg_batch.description)
739 (arg_resume.name,
make_semantic(arg_resume), arg_resume.description)
742 po::options_description desc_options(
"Allowed options");
743 desc_options.add(desc_cmd_only).add(desc_cmd_sett);
746 po::variables_map vm;
765 std::cout << desc_options << std::endl;
771 std::cerr <<
"Error: batch-size set, but batch option not enabled" <<
ENDL;
776 std::cerr <<
"Error: batch-size must be > 0" <<
ENDL;
786 if (db_batch_size > db_batch_size_verify)
788 db_batch_size = db_batch_size_verify;
794 if (opt_testnet && opt_stagenet)
796 std::cerr <<
"Error: Can't specify more than one of --testnet and --stagenet" <<
ENDL;
808 MINFO(
"Starting...");
810 boost::filesystem::path fs_import_file_path;
815 fs_import_file_path = boost::filesystem::path(m_config_folder) /
"export" /
BLOCKCHAIN_RAW;
817 import_file_path = fs_import_file_path.string();
833 std::cerr <<
"Error parsing database argument(s)" <<
ENDL;
839 std::cerr <<
"Invalid database type: " << db_type << std::endl;
843 MINFO(
"database: " << db_type);
844 MINFO(
"database flags: " << db_flags);
845 MINFO(
"verify: " << std::boolalpha << opt_verify << std::noboolalpha);
848 MINFO(
"batch: " << std::boolalpha << opt_batch << std::noboolalpha
849 <<
" batch size: " << db_batch_size);
853 MINFO(
"batch: " << std::boolalpha << opt_batch << std::noboolalpha);
855 MINFO(
"resume: " << std::boolalpha << opt_resume << std::noboolalpha);
856 MINFO(
"nettype: " << (opt_testnet ?
"testnet" : opt_stagenet ?
"stagenet" :
"mainnet"));
858 MINFO(
"bootstrap file path: " << import_file_path);
859 MINFO(
"database path: " << m_config_folder);
864 "Import is set to proceed WITHOUT VERIFICATION.\n" 865 "This is a DANGEROUS operation: if the file was tampered with in transit, or obtained from a malicious source,\n" 866 "you could end up with a compromised database. It is recommended to NOT use " << arg_noverify.name <<
".\n" 867 "*****************************************************************************************\n" 868 "You have 90 seconds to press ^C or terminate this program before unverified import starts\n" 869 "*****************************************************************************************");
880 #if defined(PER_BLOCK_CHECKPOINT) 885 if (!
core.
init(vm,
nullptr, get_checkpoints))
887 std::cerr <<
"Failed to initialize core" <<
ENDL;
903 MINFO(
"Dropping hard fork tables...");
919 std::cout <<
std::string(
"Error loading blockchain db: ") + e.
what() +
" -- shutting down now" <<
ENDL;
const char *const ELECTRONEUM_RELEASE_NAME
bool validate_file_checksum_against_dns(std::string import_file_path)
bool deinit()
performs safe shutdown steps for core and core components
bool init(const boost::program_options::variables_map &vm, const test_options *test_options=NULL, const GetCheckpointsCallback &get_checkpoints=nullptr)
initializes the core as needed
uint64_t count_bytes(std::ifstream &import_file, uint64_t blocks, uint64_t &h, bool &quit)
void get_blob_hash(const epee::span< const char > &blob, crypto::hash &res)
bool m_verification_failed
void mlog_set_log(const char *log)
#define CATCH_ENTRY(location, return_val)
std::string mlog_get_default_log_path(const char *default_filename)
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size=MAX_LOG_FILE_SIZE, const std::size_t max_log_files=MAX_LOG_FILES)
int pop_blocks(cryptonote::core &core, int num_blocks)
uint64_t num_blocks(const std::vector< test_event_entry > &events)
#define MCLOG_RED(level, cat, x)
std::function< const epee::span< const unsigned char >cryptonote::network_type network)> GetCheckpointsCallback
Callback routine that returns checkpoints data for specific network type.
const command_line::arg_descriptor< bool, false > arg_stagenet_on
void disable_dns_checkpoints(bool disable=true)
set whether or not to enable or disable DNS checkpoints
const arg_descriptor< bool > arg_help
std::string blockchain_db_types(const std::string &sep)
int get_db_flags_from_mode(const std::string &db_mode)
uint64_t get_current_blockchain_height() const
get the current height of the blockchain
bool parse_binary(const std::string &blob, T &v)
void calc_sha256(const char *path, char output[65])
uint64_t cumulative_difficulty
virtual void set_batch_transactions(bool)=0
sets whether or not to batch transactions
bool prepare_handle_incoming_blocks(const std::vector< block_complete_entry > &blocks_entry, std::vector< block > &blocks)
performs some preprocessing on a group of incoming blocks to speed up verification ...
uint64_t count_blocks(const std::string &dir_path, std::streampos &start_pos, uint64_t &seek_height)
Holds cryptonote related classes and helpers.
void show_stats()
show profiling stats
const char *const ELECTRONEUM_VERSION_FULL
bool blockchain_valid_db_type(const std::string &db_type)
blobdata tx_to_blob(const transaction &tx)
bool handle_incoming_tx(const blobdata &tx_blob, tx_verification_context &tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
handles an incoming transaction
bool cleanup_handle_incoming_blocks(bool force_sync=false)
incoming blocks post-processing, cleanup, and disk sync
const command_line::arg_descriptor< bool, false > arg_testnet_on
bool handle_error_helper(const boost::program_options::options_description &desc, F parser)
std::enable_if<!std::is_same< T, bool >::value, bool >::type has_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
boost::program_options::typed_value< T, char > * make_semantic(const arg_descriptor< T, true > &)
#define NUM_BLOCKS_PER_CHUNK
const command_line::arg_descriptor< std::string > arg_log_level
difficulty_type cumulative_difficulty
#define CHUNK_SIZE_WARNING_THRESHOLD
std::vector< transaction > txs
int main(int argc, char *argv[])
handles core cryptonote functionality
unsigned __int64 uint64_t
const command_line::arg_descriptor< std::string, false, true, 2 > arg_data_dir
virtual void batch_stop()=0
ends a batch transaction
static void init_options(boost::program_options::options_description &desc)
adds command line options to the given options set
Useful when application has potentially harmful situtaions.
void add_arg(boost::program_options::options_description &description, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg, bool unique=true)
const BlockchainDB & get_db() const
get a reference to the BlockchainDB in use by Blockchain
bool handle_incoming_block(const blobdata &block_blob, const block *b, block_verification_context &bvc, bool update_miner_blocktemplate=true)
handles an incoming block
A generic BlockchainDB exception.
Blockchain & get_blockchain_storage()
gets the Blockchain instance
virtual void drop_hard_fork_info()=0
delete hard fork info from database
const T & move(const T &t)
boost::multiprecision::uint128_t difficulty_type
uint64_t get_next_long_term_block_weight(uint64_t block_weight) const
gets the long term block weight for a new block
T get_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, false, true > &arg)
boost::program_options::basic_parsed_options< charT > parse_command_line(int argc, const charT *const argv[], const boost::program_options::options_description &desc, bool allow_unregistered=false)
uint64_t seek_to_first_chunk(std::ifstream &import_file, uint8_t &major_version, uint8_t &minor_version)
int import_from_file(cryptonote::core &core, const std::string &import_file_path, uint64_t block_stop=0)
std::string to_string(t_connection_type type)
int parse_db_arguments(const std::string &db_arg_str, std::string &db_type, int &db_flags)
int check_flush(cryptonote::core &core, std::vector< block_complete_entry > &blocks, bool force)
blobdata block_to_blob(const block &b)
const epee::span< const unsigned char > GetCheckpointsData(cryptonote::network_type network)
const char * what() const
void reset_stats()
reset profiling stats
bool is_arg_defaulted(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
std::vector< transaction > txs
void sha256_hash_string(unsigned char hash[SHA256_DIGEST_LENGTH], char outputBuffer[65])
virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0)=0
tells the BlockchainDB to start a new "batch" of blocks