31 #include <boost/algorithm/string.hpp> 41 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY 42 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "bcutil" 44 #define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} 46 namespace po = boost::program_options;
53 static uint64_t records_per_sync = 128;
54 static const size_t slack = 512 * 1024 * 1024;
56 static std::error_code replace_file(
const boost::filesystem::path& replacement_name,
const boost::filesystem::path& replaced_name)
58 std::error_code ec =
tools::replace_file(replacement_name.string(), replaced_name.string());
60 MERROR(
"Error renaming " << replacement_name <<
" to " << replaced_name <<
": " << ec.message());
64 static void open(
MDB_env *&env,
const boost::filesystem::path &path,
uint64_t db_flags,
bool readonly)
80 dbr =
mdb_env_open(env, path.string().c_str(), flags, 0664);
81 if (dbr)
throw std::runtime_error(
"Failed to open database file '" 94 boost::filesystem::path path(db_path);
95 boost::filesystem::space_info si = boost::filesystem::space(path);
96 if(si.available < bytes)
98 MERROR(
"!! WARNING: Insufficient free space to extend database !!: " <<
99 (si.available >> 20L) <<
" MB available, " << (bytes >> 20L) <<
" MB needed");
106 MWARNING(
"Unable to query free disk space.");
115 new_mapsize += (new_mapsize % mst.
ms_psize);
121 MGINFO(
"LMDB Mapsize increased." <<
" Old: " << mei.
me_mapsize / (1024 * 1024) <<
"MiB" <<
", New: " << new_mapsize / (1024 * 1024) <<
"MiB");
124 static void check_resize(
MDB_env *env,
size_t bytes)
133 if (size_used + bytes + slack >= mei.
me_mapsize)
134 add_size(env, size_used + bytes + 2 * slack - mei.
me_mapsize);
137 static bool resize_point(
size_t nrecords,
MDB_env *env,
MDB_txn **txn,
size_t &bytes)
139 if (nrecords % records_per_sync && bytes <= slack / 2)
143 check_resize(env, bytes);
155 bool tx_active0 =
false, tx_active1 =
false;
204 size_t nrecords = 0, bytes = 0;
215 if (resize_point(++nrecords, env1, &txn1, bytes))
243 throw std::runtime_error(
"Invalid transaction pruned data");
249 MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties;
251 MDB_cursor *cur0_txs_pruned, *cur0_txs_prunable, *cur0_tx_indices, *cur1_txs_prunable, *cur1_txs_prunable_tip;
252 bool tx_active0 =
false, tx_active1 =
false;
255 MGINFO(
"Creating pruned txs_prunable");
283 mdb_set_dupsort(txn0, dbi0_tx_indices, BlockchainLMDB::compare_hash32);
296 dbr =
mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
299 dbr =
mdb_drop(txn1, dbi1_txs_prunable, 0);
301 dbr =
mdb_drop(txn1, dbi1_txs_prunable_tip, 0);
304 dbr =
mdb_dbi_open(txn1,
"properties", 0, &dbi1_properties);
309 static char pruning_seed_key[] =
"pruning_seed";
311 k.
mv_size = strlen(
"pruning_seed") + 1;
312 v.
mv_data = (
void*)&pruning_seed;
313 v.
mv_size =
sizeof(pruning_seed);
314 dbr =
mdb_put(txn1, dbi1_properties, &k, &v, 0);
320 dbr =
mdb_stat(txn0, dbi0_blocks, &stats);
324 size_t nrecords = 0, bytes = 0;
340 MDEBUG(block_height <<
"/" << blockchain_height <<
" is in tip");
344 bytes += kk.mv_size + vv.mv_size;
351 bytes += kk.mv_size + vv.
mv_size;
352 if (resize_point(++nrecords, env1, &txn1, bytes))
356 dbr =
mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
364 MDEBUG(
"" << block_height <<
"/" << blockchain_height <<
" should be pruned, dropping");
387 std::vector<std::string> options;
389 boost::split(options, db_sync_mode, boost::is_any_of(
" :"));
391 for(
const auto &
option : options)
399 if(options.size() == 0)
402 db_flags = DEFAULT_FLAGS;
405 bool safemode =
false;
406 if(options.size() >= 1)
408 if(options[0] ==
"safe")
413 else if(options[0] ==
"fast")
417 else if(options[0] ==
"fastest")
420 records_per_sync = 1000;
426 if(options.size() >= 2 && !safemode)
429 uint64_t bps = strtoull(options[1].c_str(), &endptr, 0);
432 records_per_sync = bps;
438 int main(
int argc,
char* argv[])
447 available_dbs =
"available: " + available_dbs;
453 boost::filesystem::path output_file_path;
455 po::options_description desc_cmd_only(
"Command line options");
456 po::options_description desc_cmd_sett(
"Command line options and settings options");
459 "database", available_dbs.c_str(), default_db_type
463 ,
"Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." 477 po::options_description desc_options(
"Allowed options");
478 desc_options.add(desc_cmd_only).add(desc_cmd_sett);
480 po::variables_map vm;
483 auto parser = po::command_line_parser(argc, argv).options(desc_options);
484 po::store(parser.run(), vm);
494 std::cout << desc_options << std::endl;
504 MINFO(
"Starting...");
517 MERROR(
"Invalid database type: " << db_type);
520 if (db_type !=
"lmdb")
522 MERROR(
"Unsupported database type: " << db_type <<
". Only lmdb is supported");
528 if (!parse_db_sync_mode(db_sync_mode, db_flags))
530 MERROR(
"Invalid db sync mode: " << db_sync_mode);
545 MINFO(
"Initializing source blockchain (BlockchainDB)");
546 std::array<std::unique_ptr<Blockchain>, 2> core_storage;
549 boost::filesystem::path paths[2];
550 bool already_pruned =
false;
551 for (
size_t n = 0; n < core_storage.size(); ++n)
553 core_storage[n].reset(
new Blockchain(m_mempool));
558 MERROR(
"Attempted to use non-existent database type: " << db_type);
559 throw std::runtime_error(
"Attempting to use non-existent database type");
561 MDEBUG(
"database: " << db_type);
566 if (boost::filesystem::exists(paths[1]))
568 if (!boost::filesystem::is_directory(paths[1]))
570 MERROR(
"LMDB needs a directory path, but a file was passed: " << paths[1].
string());
576 if (!boost::filesystem::create_directories(paths[1]))
578 MERROR(
"Failed to create directory: " << paths[1].
string());
582 db_path = paths[1].string();
589 MINFO(
"Loading blockchain from folder " << paths[n] <<
" ...");
595 catch (
const std::exception& e)
597 MERROR(
"Error opening database: " << e.what());
600 r = core_storage[n]->init(db, net_type);
602 std::string source_dest = n == 0 ?
"source" :
"pruned";
604 MINFO(source_dest <<
" blockchain storage initialized OK");
605 if (n == 0 && core_storage[0]->get_blockchain_pruning_seed())
607 if (!opt_copy_pruned_database)
609 MERROR(
"Blockchain is already pruned, use --" << arg_copy_pruned_database.name <<
" to copy it anyway");
612 already_pruned =
true;
615 core_storage[0]->deinit();
616 core_storage[0].reset(NULL);
617 core_storage[1]->deinit();
618 core_storage[1].reset(NULL);
621 MDB_env *env0 = NULL, *env1 = NULL;
622 open(env0, paths[0], db_flags,
true);
623 open(env1, paths[1], db_flags,
false);
636 copy_table(env0, env1,
"txpool_meta", 0,
MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
637 copy_table(env0, env1,
"txpool_blob", 0,
MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
639 copy_table(env0, env1,
"properties", 0, 0, BlockchainLMDB::compare_string);
652 MINFO(
"Swapping databases, pre-pruning blockchain will be left in " << paths[0].
string() +
"-old and can be removed if desired");
653 if (replace_file(paths[0].
string(), paths[0].
string() +
"-old") || replace_file(paths[1].
string(), paths[0].
string()))
655 MERROR(
"Blockchain pruned OK, but renaming failed");
659 MINFO(
"Blockchain pruned OK");
const char *const ELECTRONEUM_RELEASE_NAME
std::vector< std::vector< _variant_t > > table
int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
Set the size of the memory map to use for this environment.
void mdb_cursor_close(MDB_cursor *cursor)
Close a cursor handle.
virtual std::string get_db_name() const =0
gets the name of the folder the BlockchainDB's file(s) should be in
int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi)
Open a database in the environment.
Lightning memory-mapped database library.
int mdb_txn_commit(MDB_txn *txn)
Commit all the operations of a transaction into the database.
int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, unsigned int flags)
Store by cursor.
auto_scope_leave_caller create_scope_leave_handler(t_scope_leave_handler f)
int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat)
Retrieve statistics for a database.
#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message)
void mdb_dbi_close(MDB_env *env, MDB_dbi dbi)
Close a database handle. Normally unnecessary. Use with care:
void mlog_set_log(const char *log)
#define CATCH_ENTRY(location, return_val)
std::string mlog_get_default_log_path(const char *default_filename)
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size=MAX_LOG_FILE_SIZE, const std::size_t max_log_files=MAX_LOG_FILES)
#define CRYPTONOTE_PRUNING_LOG_STRIPES
bool is_v1_tx(const blobdata &tx_blob)
int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
Set a custom data comparison function for a MDB_DUPSORT database.
int mdb_env_info(MDB_env *env, MDB_envinfo *stat)
Return information about the LMDB environment.
const command_line::arg_descriptor< std::string > arg_db_sync_mode
const command_line::arg_descriptor< bool, false > arg_stagenet_on
int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs)
Set the maximum number of named databases for the environment.
const arg_descriptor< bool > arg_help
Non-owning sequence of data. Does not deep copy.
std::string blockchain_db_types(const std::string &sep)
struct MDB_env MDB_env
Opaque structure for a database environment.
mdb_size_t ms_branch_pages
boost::filesystem::path data_dir
virtual void open(const std::string &filename, const int db_flags=0)=0
open a db, or create it if necessary.
#define CRYPTONOTE_PRUNING_TIP_BLOCKS
struct MDB_txn MDB_txn
Opaque structure for a transaction handle.
Holds cryptonote related classes and helpers.
int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
Empty or delete+close a database.
const char *const ELECTRONEUM_VERSION_FULL
bool blockchain_valid_db_type(const std::string &db_type)
int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode)
Open an environment handle.
Statistics for a database in the environment.
Information about the environment.
const command_line::arg_descriptor< bool, false > arg_testnet_on
BlockchainDB * new_db(const std::string &db_type)
bool handle_error_helper(const boost::program_options::options_description &desc, F parser)
const command_line::arg_descriptor< std::string > arg_log_level
boost::shared_ptr< call_befor_die_base > auto_scope_leave_caller
#define MDB_val_set(var, val)
unsigned __int64 uint64_t
const command_line::arg_descriptor< std::string, false, true, 2 > arg_data_dir
int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
Set a custom key comparison function for a database.
mdb_size_t ms_overflow_pages
unsigned int MDB_dbi
A handle for an individual database in the DB environment.
int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
Create a transaction for use with the environment.
MDB_cursor_op
Cursor Get operations.
void add_arg(boost::program_options::options_description &description, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg, bool unique=true)
int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, MDB_cursor_op op)
Retrieve by cursor.
int mdb_env_create(MDB_env **env)
Create an LMDB environment handle.
void mdb_txn_abort(MDB_txn *txn)
Abandon all the operations of the transaction instead of saving them.
Transaction pool, handles transactions which are not part of a block.
struct MDB_cursor MDB_cursor
Opaque structure for navigating through a database.
The BlockchainDB backing store interface declaration/contract.
Generic structure used for passing keys and data in and out of the database.
T get_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, false, true > &arg)
int mdb_env_stat(MDB_env *env, MDB_stat *stat)
Return statistics about the LMDB environment.
char * mdb_strerror(int err)
Return a string describing a given error code.
std::string to_string(t_connection_type type)
int main(int argc, char *argv[])
int compare_uint64(const MDB_val *a, const MDB_val *b)
int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned int flags)
Store items into a database.
void mdb_env_close(MDB_env *env)
Close the environment and release the memory map.
bool is_arg_defaulted(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor)
Create a cursor handle.