Electroneum
blockchain_import.cpp File Reference
#include <atomic>
#include <cstdio>
#include <algorithm>
#include <fstream>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/chrono.hpp>
#include <unistd.h>
#include <openssl/sha.h>
#include "misc_log_ex.h"
#include "bootstrap_file.h"
#include "bootstrap_serialization.h"
#include "blocks/blocks.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "serialization/binary_utils.h"
#include "serialization/json_utils.h"
#include "include_base_utils.h"
#include "blockchain_db/db_types.h"
#include "cryptonote_core/cryptonote_core.h"
#include "common/dns_utils.h"
Include dependency graph for blockchain_import.cpp:

Go to the source code of this file.

Macros

#define ELECTRONEUM_DEFAULT_LOG_CATEGORY   "bcutil"
 

Functions

int get_db_flags_from_mode (const std::string &db_mode)
 
int parse_db_arguments (const std::string &db_arg_str, std::string &db_type, int &db_flags)
 
int pop_blocks (cryptonote::core &core, int num_blocks)
 
int check_flush (cryptonote::core &core, std::vector< block_complete_entry > &blocks, bool force)
 
int import_from_file (cryptonote::core &core, const std::string &import_file_path, uint64_t block_stop=0)
 
void sha256_hash_string (unsigned char hash[SHA256_DIGEST_LENGTH], char outputBuffer[65])
 
void calc_sha256 (const char *path, char output[65])
 
bool validate_file_checksum_against_dns (std::string import_file_path)
 
int main (int argc, char *argv[])
 

Macro Definition Documentation

◆ ELECTRONEUM_DEFAULT_LOG_CATEGORY

#define ELECTRONEUM_DEFAULT_LOG_CATEGORY   "bcutil"

Definition at line 53 of file blockchain_import.cpp.

Function Documentation

◆ calc_sha256()

void calc_sha256 ( const char *  path,
char  output[65] 
)

Definition at line 611 of file blockchain_import.cpp.

612 {
613  FILE* file = fopen(path, "rb");
614  if(!file) return;
615 
616  unsigned char hash[SHA256_DIGEST_LENGTH];
617  SHA256_CTX sha256;
618  SHA256_Init(&sha256);
619  const int bufSize = 32768;
620  char* buffer = (char*)std::malloc(bufSize);
621  int bytesRead = 0;
622  if(!buffer) return;
623  while((bytesRead = fread(buffer, 1, bufSize, file)))
624  {
625  SHA256_Update(&sha256, buffer, bytesRead);
626  }
627  SHA256_Final(hash, &sha256);
628 
629  sha256_hash_string(hash, output);
630  fclose(file);
631  free(buffer);
632 }
POD_CLASS hash
Definition: hash.h:50
void sha256_hash_string(unsigned char hash[SHA256_DIGEST_LENGTH], char outputBuffer[65])
Here is the call graph for this function:
Here is the caller graph for this function:

◆ check_flush()

int check_flush ( cryptonote::core core,
std::vector< block_complete_entry > &  blocks,
bool  force 
)

Definition at line 197 of file blockchain_import.cpp.

198 {
199  if (blocks.empty())
200  return 0;
201  if (!force && blocks.size() < db_batch_size)
202  return 0;
203 
204  std::vector<block> pblocks;
206  {
207  MERROR("Failed to prepare to add blocks");
208  return 1;
209  }
210  if (!pblocks.empty() && pblocks.size() != blocks.size())
211  {
212  MERROR("Unexpected parsed blocks size");
214  return 1;
215  }
216 
217  size_t blockidx = 0;
218  for(const block_complete_entry& block_entry: blocks)
219  {
220  // process transactions
221  for(auto& tx_blob: block_entry.txs)
222  {
224  core.handle_incoming_tx(tx_blob, tvc, true, true, false);
225  if(tvc.m_verification_failed)
226  {
227  MERROR("transaction verification failed, tx_id = "
230  return 1;
231  }
232  }
233 
234  // process block
235 
236  block_verification_context bvc = boost::value_initialized<block_verification_context>();
237 
238  core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx++], bvc, false); // <--- process block
239 
240  if(bvc.m_verification_failed)
241  {
242  MERROR("Block verification failed, id = "
243  << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block)));
245  return 1;
246  }
247  if(bvc.m_marked_as_orphaned)
248  {
249  MERROR("Block received at sync phase was marked as orphaned");
251  return 1;
252  }
253 
254  } // each download block
256  return 1;
257 
258  blocks.clear();
259  return 0;
260 }
#define MERROR(x)
Definition: misc_log_ex.h:73
void get_blob_hash(const epee::span< const char > &blob, crypto::hash &res)
bool prepare_handle_incoming_blocks(const std::vector< block_complete_entry > &blocks_entry, std::vector< block > &blocks)
performs some preprocessing on a group of incoming blocks to speed up verification ...
bool handle_incoming_tx(const blobdata &tx_blob, tx_verification_context &tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
handles an incoming transaction
bool cleanup_handle_incoming_blocks(bool force_sync=false)
incoming blocks post-processing, cleanup, and disk sync
std::string pod_to_hex(const t_pod_type &s)
Definition: string_tools.h:317
handles core cryptonote functionality
bool handle_incoming_block(const blobdata &block_blob, const block *b, block_verification_context &bvc, bool update_miner_blocktemplate=true)
handles an incoming block
#define AUTO_VAL_INIT(v)
Definition: misc_language.h:53
Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_db_flags_from_mode()

int get_db_flags_from_mode ( const std::string &  db_mode)

Definition at line 88 of file blockchain_import.cpp.

89 {
90  int db_flags = 0;
91  if (db_mode == "safe")
92  db_flags = DBF_SAFE;
93  else if (db_mode == "fast")
94  db_flags = DBF_FAST;
95  else if (db_mode == "fastest")
96  db_flags = DBF_FASTEST;
97  return db_flags;
98 }
#define DBF_FASTEST
#define DBF_SAFE
#define DBF_FAST

◆ import_from_file()

int import_from_file ( cryptonote::core core,
const std::string &  import_file_path,
uint64_t  block_stop = 0 
)

Definition at line 262 of file blockchain_import.cpp.

263 {
264  // Reset stats, in case we're using newly created db, accumulating stats
265  // from addition of genesis block.
266  // This aligns internal db counts with importer counts.
268 
269  boost::filesystem::path fs_import_file_path(import_file_path);
270  boost::system::error_code ec;
271  if (!boost::filesystem::exists(fs_import_file_path, ec))
272  {
273  MFATAL("bootstrap file not found: " << fs_import_file_path);
274  return false;
275  }
276 
277  uint64_t start_height = 1, seek_height;
278  if (opt_resume)
280 
281  seek_height = start_height;
282  BootstrapFile bootstrap;
283  std::streampos pos;
284  // BootstrapFile bootstrap(import_file_path);
285  uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height);
286  MINFO("bootstrap file last block number: " << total_source_blocks-1 << " (zero-based height) total blocks: " << total_source_blocks);
287 
288  if (total_source_blocks-1 <= start_height)
289  {
290  return false;
291  }
292 
293  std::cout << ENDL;
294  std::cout << "Preparing to read blocks..." << ENDL;
295  std::cout << ENDL;
296 
297  std::ifstream import_file;
298  import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in);
299 
300  uint64_t h = 0;
301  uint64_t num_imported = 0;
302  if (import_file.fail())
303  {
304  MFATAL("import_file.open() fail");
305  return false;
306  }
307 
308  // 4 byte magic + (currently) 1024 byte header structures
309  uint8_t major_version, minor_version;
310  bootstrap.seek_to_first_chunk(import_file, major_version, minor_version);
311 
312  std::string str1;
313  char buffer1[1024];
314  char buffer_block[BUFFER_SIZE];
315  block b;
316  transaction tx;
317  int quit = 0;
318  uint64_t bytes_read;
319 
320  // Note that a new blockchain will start with block number 0 (total blocks: 1)
321  // due to genesis block being added at initialization.
322 
323  if (! block_stop)
324  {
325  block_stop = total_source_blocks - 1;
326  }
327 
328  // These are what we'll try to use, and they don't have to be a determination
329  // from source and destination blockchains, but those are the defaults.
330  MINFO("start block: " << start_height << " stop block: " <<
331  block_stop);
332 
333  bool use_batch = opt_batch && !opt_verify;
334 
335  MINFO("Reading blockchain from bootstrap file...");
336  std::cout << ENDL;
337 
338  std::vector<block_complete_entry> blocks;
339 
340  // Skip to start_height before we start adding.
341  {
342  bool q2 = false;
343  import_file.seekg(pos);
344  bytes_read = bootstrap.count_bytes(import_file, start_height-seek_height, h, q2);
345  if (q2)
346  {
347  quit = 2;
348  goto quitting;
349  }
350  h = start_height;
351  }
352 
353  if (use_batch)
354  {
355  uint64_t bytes, h2;
356  bool q2;
357  pos = import_file.tellg();
358  bytes = bootstrap.count_bytes(import_file, db_batch_size, h2, q2);
359  if (import_file.eof())
360  import_file.clear();
361  import_file.seekg(pos);
362  core.get_blockchain_storage().get_db().batch_start(db_batch_size, bytes);
363  }
364  while (! quit)
365  {
366  uint32_t chunk_size;
367  import_file.read(buffer1, sizeof(chunk_size));
368  // TODO: bootstrap.read_chunk();
369  if (! import_file) {
370  std::cout << refresh_string;
371  MINFO("End of file reached");
372  quit = 1;
373  break;
374  }
375  bytes_read += sizeof(chunk_size);
376 
377  str1.assign(buffer1, sizeof(chunk_size));
378  if (! ::serialization::parse_binary(str1, chunk_size))
379  {
380  throw std::runtime_error("Error in deserialization of chunk size");
381  }
382  MDEBUG("chunk_size: " << chunk_size);
383 
384  if (chunk_size > BUFFER_SIZE)
385  {
386  MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
387  throw std::runtime_error("Aborting: chunk size exceeds buffer size");
388  }
389  if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD)
390  {
391  MINFO("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD);
392  }
393  else if (chunk_size == 0) {
394  MFATAL("ERROR: chunk_size == 0");
395  return 2;
396  }
397  import_file.read(buffer_block, chunk_size);
398  if (! import_file) {
399  if (import_file.eof())
400  {
401  std::cout << refresh_string;
402  MINFO("End of file reached - file was truncated");
403  quit = 1;
404  break;
405  }
406  else
407  {
408  MFATAL("ERROR: unexpected end of file: bytes read before error: "
409  << import_file.gcount() << " of chunk_size " << chunk_size);
410  return 2;
411  }
412  }
413  bytes_read += chunk_size;
414  MDEBUG("Total bytes read: " << bytes_read);
415 
416  if (h > block_stop)
417  {
418  std::cout << refresh_string << "block " << h-1
419  << " / " << block_stop
420  << "\r" << std::flush;
421  std::cout << ENDL << ENDL;
422  MINFO("Specified block number reached - stopping. block: " << h-1 << " total blocks: " << h);
423  quit = 1;
424  break;
425  }
426 
427  try
428  {
429  str1.assign(buffer_block, chunk_size);
431  bool res;
432  if (major_version == 0)
433  {
435  res = ::serialization::parse_binary(str1, bp1);
436  if (res)
437  {
438  bp.block = std::move(bp1.block);
439  bp.txs = std::move(bp1.txs);
440  bp.block_weight = bp1.block_weight;
443  }
444  }
445  else
447  if (!res)
448  throw std::runtime_error("Error in deserialization of chunk");
449 
450  int display_interval = 1000;
451  int progress_interval = 10;
452  // NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later supported.
453  for (int chunk_ind = 0; chunk_ind < NUM_BLOCKS_PER_CHUNK; ++chunk_ind)
454  {
455  ++h;
456  if ((h-1) % display_interval == 0)
457  {
458  std::cout << refresh_string;
459  MDEBUG("loading block number " << h-1);
460  }
461  else
462  {
463  MDEBUG("loading block number " << h-1);
464  }
465  b = bp.block;
466  MDEBUG("block prev_id: " << b.prev_id << ENDL);
467 
468  if ((h-1) % progress_interval == 0)
469  {
470  std::cout << refresh_string << "block " << h-1
471  << " / " << block_stop
472  << "\r" << std::flush;
473  }
474 
475  if (opt_verify)
476  {
479  std::vector<cryptonote::blobdata> txs;
480  for (const auto &tx: bp.txs)
481  {
482  txs.push_back(cryptonote::blobdata());
483  cryptonote::tx_to_blob(tx, txs.back());
484  }
485  blocks.push_back({block, txs});
486  int ret = check_flush(core, blocks, false);
487  if (ret)
488  {
489  quit = 2; // make sure we don't commit partial block data
490  break;
491  }
492  }
493  else
494  {
495  std::vector<std::pair<transaction, blobdata>> txs;
496  std::vector<transaction> archived_txs;
497 
498  archived_txs = bp.txs;
499 
500  // tx number 1: coinbase tx
501  // tx number 2 onwards: archived_txs
502  for (const transaction &tx : archived_txs)
503  {
504  // add blocks with verification.
505  // for Blockchain and blockchain_storage add_new_block().
506  // for add_block() method, without (much) processing.
507  // don't add coinbase transaction to txs.
508  //
509  // because add_block() calls
510  // add_transaction(blk_hash, blk.miner_tx) first, and
511  // then a for loop for the transactions in txs.
512  txs.push_back(std::make_pair(tx, tx_to_blob(tx)));
513  }
514 
515  size_t block_weight;
516  difficulty_type cumulative_difficulty;
517  uint64_t coins_generated;
518 
519  block_weight = bp.block_weight;
520  cumulative_difficulty = bp.cumulative_difficulty;
521  coins_generated = bp.coins_generated;
522 
523  try
524  {
525  uint64_t long_term_block_weight = core.get_blockchain_storage().get_next_long_term_block_weight(block_weight);
526  core.get_blockchain_storage().get_db().add_block(std::make_pair(b, block_to_blob(b)), block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs);
527  }
528  catch (const std::exception& e)
529  {
530  std::cout << refresh_string;
531  MFATAL("Error adding block to blockchain: " << e.what());
532  quit = 2; // make sure we don't commit partial block data
533  break;
534  }
535 
536  if (use_batch)
537  {
538  if ((h-1) % db_batch_size == 0)
539  {
540  uint64_t bytes, h2;
541  bool q2;
542  std::cout << refresh_string;
543  // zero-based height
544  std::cout << ENDL << "[- batch commit at height " << h-1 << " -]" << ENDL;
546  pos = import_file.tellg();
547  bytes = bootstrap.count_bytes(import_file, db_batch_size, h2, q2);
548  import_file.seekg(pos);
549  core.get_blockchain_storage().get_db().batch_start(db_batch_size, bytes);
550  std::cout << ENDL;
552  }
553  }
554  }
555  ++num_imported;
556  }
557  }
558  catch (const std::exception& e)
559  {
560  std::cout << refresh_string;
561  MFATAL("exception while reading from file, height=" << h << ": " << e.what());
562  return 2;
563  }
564  } // while
565 
566 quitting:
567  import_file.close();
568 
569  if (opt_verify)
570  {
571  int ret = check_flush(core, blocks, true);
572  if (ret)
573  return ret;
574  }
575 
576  if (use_batch)
577  {
578  if (quit > 1)
579  {
580  // There was an error, so don't commit pending data.
581  // Destructor will abort write txn.
582  }
583  else
584  {
586  }
587  }
588 
590  MINFO("Number of blocks imported: " << num_imported);
591  if (h > 0)
592  // TODO: if there was an error, the last added block is probably at zero-based height h-2
593  MINFO("Finished at block: " << h-1 << " total blocks: " << h);
594 
595  std::cout << ENDL;
596  return 0;
597 }
const char * res
Definition: hmac_keccak.cpp:41
#define MINFO(x)
Definition: misc_log_ex.h:75
uint64_t count_bytes(std::ifstream &import_file, uint64_t blocks, uint64_t &h, bool &quit)
#define MFATAL(x)
Definition: misc_log_ex.h:72
::std::string string
Definition: gtest-port.h:1097
#define BUFFER_SIZE
uint64_t get_current_blockchain_height() const
get the current height of the blockchain
Definition: blockchain.cpp:322
bool parse_binary(const std::string &blob, T &v)
Definition: binary_utils.h:41
unsigned char uint8_t
Definition: stdint.h:124
#define MDEBUG(x)
Definition: misc_log_ex.h:76
uint64_t count_blocks(const std::string &dir_path, std::streampos &start_pos, uint64_t &seek_height)
void show_stats()
show profiling stats
blobdata tx_to_blob(const transaction &tx)
#define NUM_BLOCKS_PER_CHUNK
unsigned int uint32_t
Definition: stdint.h:126
#define CHUNK_SIZE_WARNING_THRESHOLD
handles core cryptonote functionality
unsigned __int64 uint64_t
Definition: stdint.h:136
virtual void batch_stop()=0
ends a batch transaction
#define MWARNING(x)
Definition: misc_log_ex.h:74
const BlockchainDB & get_db() const
get a reference to the BlockchainDB in use by Blockchain
Definition: blockchain.h:963
std::string blobdata
Definition: blobdatatype.h:39
Blockchain & get_blockchain_storage()
gets the Blockchain instance
const T & move(const T &t)
Definition: gtest-port.h:1317
boost::multiprecision::uint128_t difficulty_type
Definition: difficulty.h:43
#define ENDL
Definition: misc_log_ex.h:149
uint64_t get_next_long_term_block_weight(uint64_t block_weight) const
gets the long term block weight for a new block
uint64_t seek_to_first_chunk(std::ifstream &import_file, uint8_t &major_version, uint8_t &minor_version)
int check_flush(cryptonote::core &core, std::vector< block_complete_entry > &blocks, bool force)
blobdata block_to_blob(const block &b)
void reset_stats()
reset profiling stats
virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0)=0
tells the BlockchainDB to start a new "batch" of blocks
Here is the call graph for this function:

◆ main()

int main ( int  argc,
char *  argv[] 
)

Definition at line 679 of file blockchain_import.cpp.

680 {
681  TRY_ENTRY();
682 
684 
685  std::string default_db_type = "lmdb";
686 
687  std::string available_dbs = cryptonote::blockchain_db_types(", ");
688  available_dbs = "available: " + available_dbs;
689 
690  uint32_t log_level = 0;
691  uint64_t num_blocks = 0;
692  uint64_t block_stop = 0;
693  std::string m_config_folder;
694  std::string db_arg_str;
695 
697 
698  std::string import_file_path;
699 
700  po::options_description desc_cmd_only("Command line options");
701  po::options_description desc_cmd_sett("Command line options and settings options");
702  const command_line::arg_descriptor<std::string> arg_input_file = {"input-file", "Specify input file", "", true};
703  const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
704  const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
705  const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size};
706  const command_line::arg_descriptor<uint64_t> arg_pop_blocks = {"pop-blocks", "Remove blocks from end of blockchain", num_blocks};
707  const command_line::arg_descriptor<bool> arg_drop_hf = {"drop-hard-fork", "Drop hard fork subdbs", false};
708  const command_line::arg_descriptor<bool> arg_count_blocks = {
709  "count-blocks"
710  , "Count blocks in bootstrap file and exit"
711  , false
712  };
713  const command_line::arg_descriptor<std::string> arg_database = {
714  "database", available_dbs.c_str(), default_db_type
715  };
716  const command_line::arg_descriptor<bool> arg_noverify = {"dangerous-unverified-import",
717  "Blindly trust the import file and use potentially malicious blocks and transactions during import (only enable if you exported the file yourself)", false};
718  const command_line::arg_descriptor<bool> arg_batch = {"batch",
719  "Batch transactions for faster import", true};
720  const command_line::arg_descriptor<bool> arg_resume = {"resume",
721  "Resume from current height if output database already exists", true};
722 
723  command_line::add_arg(desc_cmd_sett, arg_input_file);
724  command_line::add_arg(desc_cmd_sett, arg_log_level);
725  command_line::add_arg(desc_cmd_sett, arg_database);
726  command_line::add_arg(desc_cmd_sett, arg_batch_size);
727  command_line::add_arg(desc_cmd_sett, arg_block_stop);
728 
729  command_line::add_arg(desc_cmd_only, arg_count_blocks);
730  command_line::add_arg(desc_cmd_only, arg_pop_blocks);
731  command_line::add_arg(desc_cmd_only, arg_drop_hf);
733 
734  // call add_options() directly for these arguments since
735  // command_line helpers support only boolean switch, not boolean argument
736  desc_cmd_sett.add_options()
737  (arg_noverify.name, make_semantic(arg_noverify), arg_noverify.description)
738  (arg_batch.name, make_semantic(arg_batch), arg_batch.description)
739  (arg_resume.name, make_semantic(arg_resume), arg_resume.description)
740  ;
741 
742  po::options_description desc_options("Allowed options");
743  desc_options.add(desc_cmd_only).add(desc_cmd_sett);
744  cryptonote::core::init_options(desc_options);
745 
746  po::variables_map vm;
747  bool r = command_line::handle_error_helper(desc_options, [&]()
748  {
749  po::store(po::parse_command_line(argc, argv, desc_options), vm);
750  po::notify(vm);
751  return true;
752  });
753  if (! r)
754  return 1;
755 
756  opt_verify = !command_line::get_arg(vm, arg_noverify);
757  opt_batch = command_line::get_arg(vm, arg_batch);
758  opt_resume = command_line::get_arg(vm, arg_resume);
759  block_stop = command_line::get_arg(vm, arg_block_stop);
760  db_batch_size = command_line::get_arg(vm, arg_batch_size);
761 
763  {
764  std::cout << "Electroneum '" << ELECTRONEUM_RELEASE_NAME << "' (v" << ELECTRONEUM_VERSION_FULL << ")" << ENDL << ENDL;
765  std::cout << desc_options << std::endl;
766  return 1;
767  }
768 
769  if (! opt_batch && !command_line::is_arg_defaulted(vm, arg_batch_size))
770  {
771  std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL;
772  return 1;
773  }
774  if (! db_batch_size)
775  {
776  std::cerr << "Error: batch-size must be > 0" << ENDL;
777  return 1;
778  }
779  if (opt_verify && command_line::is_arg_defaulted(vm, arg_batch_size))
780  {
781  // usually want batch size default lower if verify on, so progress can be
782  // frequently saved.
783  //
784  // currently, with Windows, default batch size is low, so ignore
785  // default db_batch_size_verify unless it's even lower
786  if (db_batch_size > db_batch_size_verify)
787  {
788  db_batch_size = db_batch_size_verify;
789  }
790  }
791 
794  if (opt_testnet && opt_stagenet)
795  {
796  std::cerr << "Error: Can't specify more than one of --testnet and --stagenet" << ENDL;
797  return 1;
798  }
799  m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir);
800  db_arg_str = command_line::get_arg(vm, arg_database);
801 
802  mlog_configure(mlog_get_default_log_path("electroneum-blockchain-import.log"), true);
805  else
806  mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
807 
808  MINFO("Starting...");
809 
810  boost::filesystem::path fs_import_file_path;
811 
812  if (command_line::has_arg(vm, arg_input_file))
813  fs_import_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_input_file));
814  else
815  fs_import_file_path = boost::filesystem::path(m_config_folder) / "export" / BLOCKCHAIN_RAW;
816 
817  import_file_path = fs_import_file_path.string();
818 
819  if (command_line::has_arg(vm, arg_count_blocks))
820  {
821  BootstrapFile bootstrap;
822  bootstrap.count_blocks(import_file_path);
823  return 0;
824  }
825 
826 
827  std::string db_type;
828  int db_flags = 0;
829  int res = 0;
830  res = parse_db_arguments(db_arg_str, db_type, db_flags);
831  if (res)
832  {
833  std::cerr << "Error parsing database argument(s)" << ENDL;
834  return 1;
835  }
836 
838  {
839  std::cerr << "Invalid database type: " << db_type << std::endl;
840  return 1;
841  }
842 
843  MINFO("database: " << db_type);
844  MINFO("database flags: " << db_flags);
845  MINFO("verify: " << std::boolalpha << opt_verify << std::noboolalpha);
846  if (opt_batch)
847  {
848  MINFO("batch: " << std::boolalpha << opt_batch << std::noboolalpha
849  << " batch size: " << db_batch_size);
850  }
851  else
852  {
853  MINFO("batch: " << std::boolalpha << opt_batch << std::noboolalpha);
854  }
855  MINFO("resume: " << std::boolalpha << opt_resume << std::noboolalpha);
856  MINFO("nettype: " << (opt_testnet ? "testnet" : opt_stagenet ? "stagenet" : "mainnet"));
857 
858  MINFO("bootstrap file path: " << import_file_path);
859  MINFO("database path: " << m_config_folder);
860 
861  if (!opt_verify && !validate_file_checksum_against_dns(import_file_path))
862  {
863  MCLOG_RED(el::Level::Warning, "global", "\n"
864  "Import is set to proceed WITHOUT VERIFICATION.\n"
865  "This is a DANGEROUS operation: if the file was tampered with in transit, or obtained from a malicious source,\n"
866  "you could end up with a compromised database. It is recommended to NOT use " << arg_noverify.name << ".\n"
867  "*****************************************************************************************\n"
868  "You have 90 seconds to press ^C or terminate this program before unverified import starts\n"
869  "*****************************************************************************************");
870  sleep(90);
871  }
872 
873  cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects
874  cryptonote::core core(&pr);
875 
876  try
877  {
878 
880 #if defined(PER_BLOCK_CHECKPOINT)
881  const GetCheckpointsCallback& get_checkpoints = blocks::GetCheckpointsData;
882 #else
883  const GetCheckpointsCallback& get_checkpoints = nullptr;
884 #endif
885  if (!core.init(vm, nullptr, get_checkpoints))
886  {
887  std::cerr << "Failed to initialize core" << ENDL;
888  return 1;
889  }
891 
892  if (!command_line::is_arg_defaulted(vm, arg_pop_blocks))
893  {
894  num_blocks = command_line::get_arg(vm, arg_pop_blocks);
898  return 0;
899  }
900 
901  if (!command_line::is_arg_defaulted(vm, arg_drop_hf))
902  {
903  MINFO("Dropping hard fork tables...");
905  core.deinit();
906  return 0;
907  }
908 
909  import_from_file(core, import_file_path, block_stop);
910 
911  // ensure db closed
912  // - transactions properly checked and handled
913  // - disk sync if needed
914  //
915  core.deinit();
916  }
917  catch (const DB_ERROR& e)
918  {
919  std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now" << ENDL;
920  core.deinit();
921  return 1;
922  }
923 
924  return 0;
925 
926  CATCH_ENTRY("Import error", 1);
927 }
const char * res
Definition: hmac_keccak.cpp:41
const char *const ELECTRONEUM_RELEASE_NAME
bool validate_file_checksum_against_dns(std::string import_file_path)
bool deinit()
performs safe shutdown steps for core and core components
bool init(const boost::program_options::variables_map &vm, const test_options *test_options=NULL, const GetCheckpointsCallback &get_checkpoints=nullptr)
initializes the core as needed
#define MINFO(x)
Definition: misc_log_ex.h:75
bool set_module_name_and_folder(const std::string &path_to_process_)
Definition: string_tools.h:249
::std::string string
Definition: gtest-port.h:1097
void mlog_set_log(const char *log)
Definition: mlog.cpp:288
#define CATCH_ENTRY(location, return_val)
Definition: misc_log_ex.h:152
std::string mlog_get_default_log_path(const char *default_filename)
Definition: mlog.cpp:72
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size=MAX_LOG_FILE_SIZE, const std::size_t max_log_files=MAX_LOG_FILES)
Definition: mlog.cpp:148
int pop_blocks(cryptonote::core &core, int num_blocks)
uint64_t num_blocks(const std::vector< test_event_entry > &events)
Definition: chaingen.cpp:1044
#define MCLOG_RED(level, cat, x)
Definition: misc_log_ex.h:58
std::function< const epee::span< const unsigned char >cryptonote::network_type network)> GetCheckpointsCallback
Callback routine that returns checkpoints data for specific network type.
Definition: blockchain.h:92
const command_line::arg_descriptor< bool, false > arg_stagenet_on
void disable_dns_checkpoints(bool disable=true)
set whether or not to enable or disable DNS checkpoints
const arg_descriptor< bool > arg_help
std::string blockchain_db_types(const std::string &sep)
uint64_t get_current_blockchain_height() const
get the current height of the blockchain
Definition: blockchain.cpp:322
virtual void set_batch_transactions(bool)=0
sets whether or not to batch transactions
uint64_t count_blocks(const std::string &dir_path, std::streampos &start_pos, uint64_t &seek_height)
bool on_startup()
Definition: util.cpp:778
const char *const ELECTRONEUM_VERSION_FULL
bool blockchain_valid_db_type(const std::string &db_type)
const command_line::arg_descriptor< bool, false > arg_testnet_on
bool handle_error_helper(const boost::program_options::options_description &desc, F parser)
Definition: command_line.h:237
std::enable_if<!std::is_same< T, bool >::value, bool >::type has_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
Definition: command_line.h:258
boost::program_options::typed_value< T, char > * make_semantic(const arg_descriptor< T, true > &)
Definition: command_line.h:120
#define TRY_ENTRY()
Definition: misc_log_ex.h:151
unsigned int uint32_t
Definition: stdint.h:126
const command_line::arg_descriptor< std::string > arg_log_level
#define BLOCKCHAIN_RAW
handles core cryptonote functionality
unsigned __int64 uint64_t
Definition: stdint.h:136
const command_line::arg_descriptor< std::string, false, true, 2 > arg_data_dir
static void init_options(boost::program_options::options_description &desc)
adds command line options to the given options set
Useful when application has potentially harmful situtaions.
void add_arg(boost::program_options::options_description &description, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg, bool unique=true)
Definition: command_line.h:188
const BlockchainDB & get_db() const
get a reference to the BlockchainDB in use by Blockchain
Definition: blockchain.h:963
A generic BlockchainDB exception.
Blockchain & get_blockchain_storage()
gets the Blockchain instance
virtual void drop_hard_fork_info()=0
delete hard fork info from database
#define ENDL
Definition: misc_log_ex.h:149
T get_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, false, true > &arg)
Definition: command_line.h:271
boost::program_options::basic_parsed_options< charT > parse_command_line(int argc, const charT *const argv[], const boost::program_options::options_description &desc, bool allow_unregistered=false)
Definition: command_line.h:224
int import_from_file(cryptonote::core &core, const std::string &import_file_path, uint64_t block_stop=0)
std::string to_string(t_connection_type type)
int parse_db_arguments(const std::string &db_arg_str, std::string &db_type, int &db_flags)
const epee::span< const unsigned char > GetCheckpointsData(cryptonote::network_type network)
Definition: blocks.cpp:21
const char * what() const
bool is_arg_defaulted(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
Definition: command_line.h:265
Here is the call graph for this function:

◆ parse_db_arguments()

int parse_db_arguments ( const std::string &  db_arg_str,
std::string &  db_type,
int &  db_flags 
)

Definition at line 100 of file blockchain_import.cpp.

101 {
102  std::vector<std::string> db_args;
103  boost::split(db_args, db_arg_str, boost::is_any_of("#"));
104  db_type = db_args.front();
105  boost::algorithm::trim(db_type);
106 
107  if (db_args.size() == 1)
108  {
109  return 0;
110  }
111  else if (db_args.size() > 2)
112  {
113  std::cerr << "unrecognized database argument format: " << db_arg_str << ENDL;
114  return 1;
115  }
116 
117  std::string db_arg_str2 = db_args[1];
118  boost::split(db_args, db_arg_str2, boost::is_any_of(","));
119 
120  // optionally use a composite mode instead of individual flags
121  const std::unordered_set<std::string> db_modes {"safe", "fast", "fastest"};
122  std::string db_mode;
123  if (db_args.size() == 1)
124  {
125  if (db_modes.count(db_args[0]) > 0)
126  {
127  db_mode = db_args[0];
128  }
129  }
130  if (! db_mode.empty())
131  {
132  db_flags = get_db_flags_from_mode(db_mode);
133  }
134  return 0;
135 }
::std::string string
Definition: gtest-port.h:1097
int get_db_flags_from_mode(const std::string &db_mode)
std::string & trim(std::string &str)
Definition: string_tools.h:288
#define ENDL
Definition: misc_log_ex.h:149

◆ pop_blocks()

int pop_blocks ( cryptonote::core core,
int  num_blocks 
)

Definition at line 137 of file blockchain_import.cpp.

138 {
139  bool use_batch = opt_batch;
140 
141  if (use_batch)
143 
144  int quit = 0;
145  block popped_block;
146  std::vector<transaction> popped_txs;
147 
148  int i = 0;
149  boost::scoped_ptr<boost::thread> t(new boost::thread([&i, &num_blocks]() {
150  time_t pop_start_time = time(NULL);
151  time_t partial_pop_time;
152  double estimate_time;
153 
154  while(i < num_blocks) {
155  partial_pop_time = time(NULL);
156 
157  if(i != 0) {
158  estimate_time = (num_blocks - i) * (((partial_pop_time - pop_start_time)*1000) / i);
159  estimate_time = ceil(estimate_time/60000); //Minutes
160  }
161 
162 
163  std::cout << "\rPoping blocks from database (aprox. " << estimate_time << " minute(s) remaining): " << i + 1 << "/" << num_blocks << " ";
164  std::cout.flush();
165  boost::this_thread::sleep_for(boost::chrono::milliseconds{200});
166  }
167 
168  std::cout << "\r \r";
169  }));
170 
171  for (; i < num_blocks; ++i)
172  {
173  core.get_blockchain_storage().get_db().pop_block(popped_block, popped_txs);
174  quit = 1;
175  }
176 
177  t->join();
178 
179 
180  if (use_batch)
181  {
182  if (quit > 1)
183  {
184  // There was an error, so don't commit pending data.
185  // Destructor will abort write txn.
186  }
187  else
188  {
190  }
192  }
193 
194  return num_blocks;
195 }
uint64_t num_blocks(const std::vector< test_event_entry > &events)
Definition: chaingen.cpp:1044
void show_stats()
show profiling stats
time_t time
Definition: blockchain.cpp:93
handles core cryptonote functionality
virtual void batch_stop()=0
ends a batch transaction
const BlockchainDB & get_db() const
get a reference to the BlockchainDB in use by Blockchain
Definition: blockchain.h:963
Blockchain & get_blockchain_storage()
gets the Blockchain instance
virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0)=0
tells the BlockchainDB to start a new "batch" of blocks
Here is the call graph for this function:

◆ sha256_hash_string()

void sha256_hash_string ( unsigned char  hash[SHA256_DIGEST_LENGTH],
char  outputBuffer[65] 
)

Definition at line 599 of file blockchain_import.cpp.

600 {
601  int i = 0;
602 
603  for(i = 0; i < SHA256_DIGEST_LENGTH; i++)
604  {
605  sprintf(outputBuffer + (i * 2), "%02x", hash[i]);
606  }
607 
608  outputBuffer[64] = 0;
609 }
POD_CLASS hash
Definition: hash.h:50
Here is the caller graph for this function:

◆ validate_file_checksum_against_dns()

bool validate_file_checksum_against_dns ( std::string  import_file_path)

Definition at line 634 of file blockchain_import.cpp.

634  {
635 
636  // Ignore checksum verification if importing testnet blockchain.
637  // Return true right away.
638  if(opt_testnet) {
639  return true;
640  }
641 
642  std::vector<std::string> records;
643 
644  // All four ElectroneumPulse domains have DNSSEC on and valid
645  static const std::vector<std::string> dns_urls = {
646  "raw.electroneumpulse.com",
647  "raw.electroneumpulse.info",
648  "raw.electroneumpulse.net",
649  "raw.electroneumpulse.org"
650  };
651 
652  try {
653 
654  if (!tools::dns_utils::load_txt_records_from_dns(records, dns_urls, "checksum"))
655  return false;
656 
657  std::string checksum = records[0];
658 
659  // Calculate input-file SHA256 checksum
660  char calc_hash[65];
661  calc_sha256(import_file_path.c_str(), calc_hash);
662 
663  // Compare input-file checksum against the DNS hash. Return "false" if hashes differ.
664  if (strcmp(calc_hash, checksum.c_str()) != 0) {
665  MINFO("Invalid input-file checksum (" << calc_hash << "), expected: " << checksum);
666  return false;
667  }
668 
669  } catch(const std::exception &ex) {
670 
671  // Return false to display warning message if DNS Checksum verification fails for some reason
672  MINFO("Unable to verify input-file checksum due to: " << ex.what());
673  return false;
674  }
675 
676  return true;
677 }
#define MINFO(x)
Definition: misc_log_ex.h:75
::std::string string
Definition: gtest-port.h:1097
void calc_sha256(const char *path, char output[65])
bool load_txt_records_from_dns(std::vector< std::string > &good_records, const std::vector< std::string > &dns_urls, std::string type)
Definition: dns_utils.cpp:515
Here is the call graph for this function: