31 #include <boost/filesystem.hpp> 32 #include <boost/format.hpp> 33 #include <boost/circular_buffer.hpp> 34 #include <boost/archive/text_oarchive.hpp> 35 #include <boost/archive/text_iarchive.hpp> 48 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY 49 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "blockchain.db.lmdb" 52 #if defined(__i386) || defined(__x86_64) 53 #define MISALIGNED_OK 1 67 struct pre_rct_output_data_t
76 inline void throw0(
const T &e)
83 inline void throw1(
const T &e)
89 #define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} 91 #define MDB_val_sized(var, val) MDB_val var = {val.size(), (void *)val.data()} 93 #define MDB_val_str(var, val) MDB_val var = {strlen(val) + 1, (void *)val} 96 struct MDB_val_copy:
public MDB_val 98 MDB_val_copy(
const T &t) :
101 mv_size =
sizeof (
T);
112 data(new char[bd.size()])
114 memcpy(data.get(), bd.data(), bd.size());
116 mv_data = data.get();
119 std::unique_ptr<char[]> data;
123 struct MDB_val_copy<const char*>:
public MDB_val 125 MDB_val_copy(
const char *s):
130 mv_data = data.get();
135 std::unique_ptr<char[]> data;
146 memcpy(&va,
a->mv_data,
sizeof(va));
148 return (va < vb) ? -1 : va > vb;
155 for (
int n = 7; n >= 0; n--)
159 return va[n] < vb[n] ? -1 : 1;
167 const char *va = (
const char*)
a->mv_data;
168 const char *vb = (
const char*) b->
mv_data;
169 return strcmp(va, vb);
174 size_t size = std::max(
a->mv_size, b->
mv_size);
178 for (
size_t n = 0; n < size; ++n)
182 return va[n] < vb[n] ? -1 : 1;
192 for (
int n = 0; n < 32; ++n)
196 return va[n] < vb[n] ? -1 : 1;
238 const char*
const LMDB_BLOCKS =
"blocks";
239 const char*
const LMDB_BLOCK_HEIGHTS =
"block_heights";
240 const char*
const LMDB_BLOCK_INFO =
"block_info";
242 const char*
const LMDB_TXS =
"txs";
243 const char*
const LMDB_TXS_PRUNED =
"txs_pruned";
244 const char*
const LMDB_TXS_PRUNABLE =
"txs_prunable";
245 const char*
const LMDB_TXS_PRUNABLE_HASH =
"txs_prunable_hash";
246 const char*
const LMDB_TXS_PRUNABLE_TIP =
"txs_prunable_tip";
247 const char*
const LMDB_TX_INDICES =
"tx_indices";
248 const char*
const LMDB_TX_OUTPUTS =
"tx_outputs";
250 const char*
const LMDB_OUTPUT_TXS =
"output_txs";
251 const char*
const LMDB_OUTPUT_AMOUNTS =
"output_amounts";
252 const char*
const LMDB_SPENT_KEYS =
"spent_keys";
254 const char*
const LMDB_TXPOOL_META =
"txpool_meta";
255 const char*
const LMDB_TXPOOL_BLOB =
"txpool_blob";
257 const char*
const LMDB_HF_STARTING_HEIGHTS =
"hf_starting_heights";
258 const char*
const LMDB_HF_VERSIONS =
"hf_versions";
259 const char*
const LMDB_VALIDATORS =
"validators";
260 const char*
const LMDB_PROPERTIES =
"properties";
261 const char*
const LMDB_UTXOS =
"unspent_txos";
262 const char*
const LMDB_ADDR_OUTPUTS =
"unspent_addr_outputs";
263 const char*
const LMDB_ADDR_TXS =
"addr_tx";
264 const char*
const LMDB_ADDR_TXS_OLD =
"addr_tx_map";
265 const char*
const LMDB_TX_INPUTS =
"tx_inputs";
267 const char zerokey[8] = {0};
268 const MDB_val zerokval = {
sizeof(zerokey), (
void *)zerokey };
285 #define CURSOR(name) \ 286 if (!m_cur_ ## name) { \ 287 int result = mdb_cursor_open(*m_write_txn, m_ ## name, &m_cur_ ## name); \ 289 throw0(DB_ERROR(lmdb_error("Failed to open cursor: ", result).c_str())); \ 292 #define RCURSOR(name) \ 293 if (!m_cur_ ## name) { \ 294 int result = mdb_cursor_open(m_txn, m_ ## name, (MDB_cursor **)&m_cur_ ## name); \ 296 throw0(DB_ERROR(lmdb_error("Failed to open cursor: ", result).c_str())); \ 297 if (m_cursors != &m_wcursors) \ 298 m_tinfo->m_ti_rflags.m_rf_ ## name = true; \ 299 } else if (m_cursors != &m_wcursors && !m_tinfo->m_ti_rflags.m_rf_ ## name) { \ 300 int result = mdb_cursor_renew(m_txn, m_cur_ ## name); \ 302 throw0(DB_ERROR(lmdb_error("Failed to renew cursor: ", result).c_str())); \ 303 m_tinfo->m_ti_rflags.m_rf_ ## name = true; \ 393 std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0};
394 std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT;
396 mdb_threadinfo::~mdb_threadinfo()
398 MDB_cursor **cur = &m_ti_rcursors.m_txc_blocks;
407 mdb_txn_safe::mdb_txn_safe(
const bool check) : m_txn(NULL), m_tinfo(NULL), m_check(check)
426 }
else if (
m_txn !=
nullptr)
430 LOG_PRINT_L0(
"WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
440 LOG_PRINT_L3(
"mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
457 message =
"Failed to commit a transaction to the db";
478 LOG_PRINT_L0(
"WARNING: mdb_txn_safe: abort() called, but m_txn is NULL");
506 MGINFO(
"LMDB map resize detected.");
517 throw0(
DB_ERROR(lmdb_error(
"Failed to set new mapsize: ", result).c_str()));
522 MGINFO(
"LMDB Mapsize increased." <<
" Old: " << old / (1024 * 1024) <<
"MiB" <<
", New: " << new_mapsize / (1024 * 1024) <<
"MiB");
547 inline void BlockchainLMDB::check_open()
const 550 throw0(DB_ERROR(
"DB operation attempted on a not-open DB instance"));
553 void BlockchainLMDB::do_resize(
uint64_t increase_size)
557 const uint64_t add_size = 1LL << 30;
562 boost::filesystem::path path(m_folder);
563 boost::filesystem::space_info si = boost::filesystem::space(path);
564 if(si.available < add_size)
566 MERROR(
"!! WARNING: Insufficient free space to extend database !!: " <<
567 (si.available >> 20L) <<
" MB available, " << (add_size >> 20L) <<
" MB needed");
574 MWARNING(
"Unable to query free disk space.");
591 if (increase_size > 0)
594 new_mapsize += (new_mapsize % mst.
ms_psize);
598 if (m_write_txn !=
nullptr)
602 throw0(DB_ERROR(
"lmdb resizing not yet supported when batch transactions enabled!"));
606 throw0(DB_ERROR(
"attempting resize with write transaction in progress, this should not happen!"));
614 throw0(DB_ERROR(lmdb_error(
"Failed to set new mapsize: ", result).c_str()));
616 MGINFO(
"LMDB Mapsize increased." <<
" Old: " << mei.
me_mapsize / (1024 * 1024) <<
"MiB" <<
", New: " << new_mapsize / (1024 * 1024) <<
"MiB");
622 bool BlockchainLMDB::need_resize(
uint64_t threshold_size)
const 625 #if defined(ENABLE_AUTO_RESIZE) 641 MDEBUG(
"Space used: " << size_used);
643 MDEBUG(
"Size threshold: " << threshold_size);
644 float resize_percent = RESIZE_PERCENT;
645 MDEBUG(boost::format(
"Percent used: %.04f Percent threshold: %.04f") % ((
double)size_used/mei.
me_mapsize) % resize_percent);
647 if (threshold_size > 0)
649 if (mei.
me_mapsize - size_used < threshold_size)
651 MINFO(
"Threshold met (size-based)");
658 if ((
double)size_used / mei.
me_mapsize > resize_percent)
660 MINFO(
"Threshold met (percent-based)");
669 void BlockchainLMDB::check_and_resize_for_batch(
uint64_t batch_num_blocks,
uint64_t batch_bytes)
672 MTRACE(
"[" << __func__ <<
"] " <<
"checking DB size");
673 const uint64_t min_increase_size = 512 * (1 << 20);
676 if (batch_num_blocks > 0)
678 threshold_size = get_estimated_batch_size(batch_num_blocks, batch_bytes);
679 MDEBUG(
"calculated batch size: " << threshold_size);
687 increase_size = (threshold_size > min_increase_size) ? threshold_size : min_increase_size;
688 MDEBUG(
"increase size: " << increase_size);
694 if (need_resize(threshold_size))
696 MGINFO(
"[batch] DB resize needed");
697 do_resize(increase_size);
708 float batch_safety_factor = 1.7f;
709 float batch_fudge_factor = batch_safety_factor * batch_num_blocks;
712 float db_expand_factor = 4.5f;
720 block_stop = m_height - 1;
722 if (block_stop >= num_prev_blocks)
723 block_start = block_stop - num_prev_blocks + 1;
726 MDEBUG(
"[" << __func__ <<
"] " <<
"m_height: " << m_height <<
" block_start: " << block_start <<
" block_stop: " << block_stop);
727 size_t avg_block_size = 0;
730 avg_block_size = batch_bytes / batch_num_blocks;
735 MDEBUG(
"No existing blocks to check for average block size");
737 else if (m_cum_count >= num_prev_blocks)
739 avg_block_size = m_cum_size / m_cum_count;
740 MDEBUG(
"average block size across recent " << m_cum_count <<
" blocks: " << avg_block_size);
749 for (
uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
755 total_block_size += block_weight;
761 avg_block_size = total_block_size / num_blocks_used;
762 MDEBUG(
"average block size across recent " << num_blocks_used <<
" blocks: " << avg_block_size);
765 if (avg_block_size < min_block_size)
766 avg_block_size = min_block_size;
767 MDEBUG(
"estimated average block size for batch: " << avg_block_size);
770 if (batch_fudge_factor < 5000.0)
771 batch_fudge_factor = 5000.0;
772 threshold_size = avg_block_size * db_expand_factor * batch_fudge_factor;
773 return threshold_size;
788 throw1(BLOCK_EXISTS(
"Attempting to add block that's already in the db"));
798 throw0(DB_ERROR(lmdb_error(
"Failed to get top block hash to check for new block's parent: ", result).c_str()));
801 if (prev->bh_height != m_height - 1)
802 throw0(BLOCK_PARENT_DNE(
"Top block is not new block's parent"));
817 throw0(DB_ERROR(lmdb_error(
"Failed to add block blob to db transaction: ", result).c_str()));
821 bi.bi_timestamp = blk.timestamp;
822 bi.bi_coins = coins_generated;
823 bi.bi_weight = block_weight;
824 bi.bi_diff_hi = ((cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to<
uint64_t>();
825 bi.bi_diff_lo = (cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>();
826 bi.bi_hash = blk_hash;
827 bi.bi_cum_rct = num_rct_outs;
828 if (blk.major_version >= 4)
833 throw1(BLOCK_DNE(lmdb_error(
"Failed to get block info: ", result).c_str()));
837 bi.bi_long_term_block_weight = long_term_block_weight;
842 throw0(DB_ERROR(lmdb_error(
"Failed to add block info to db transaction: ", result).c_str()));
846 throw0(DB_ERROR(lmdb_error(
"Failed to add block height by hash to db transaction: ", result).c_str()));
850 m_cum_size += block_weight;
854 void BlockchainLMDB::remove_block()
863 throw0(BLOCK_DNE (
"Attempting to remove block from an empty blockchain"));
869 MDB_val_copy<uint64_t> k(m_height - 1);
872 throw1(BLOCK_DNE(lmdb_error(
"Attempting to remove block that's not in the db: ", result).c_str()));
880 throw1(DB_ERROR(lmdb_error(
"Failed to locate block height by hash for removal: ", result).c_str()));
882 throw1(DB_ERROR(lmdb_error(
"Failed to add removal of block height by hash to db transaction: ", result).c_str()));
885 throw1(DB_ERROR(lmdb_error(
"Failed to add removal of block to db transaction: ", result).c_str()));
888 throw1(DB_ERROR(lmdb_error(
"Failed to add removal of block info to db transaction: ", result).c_str()));
912 throw1(TX_EXISTS(
std::string(
"Attempting to add transaction that's already in the db (tx id ").append(boost::lexical_cast<std::string>(tip->data.tx_id)).append(
")").c_str()));
920 ti.data.tx_id = tx_id;
922 ti.data.block_id = m_height;
924 val_h.mv_size =
sizeof(ti);
925 val_h.mv_data = (
void *)&ti;
929 throw0(DB_ERROR(lmdb_error(
"Failed to add tx data to db transaction: ", result).c_str()));
935 if (unprunable_size == 0)
937 std::stringstream ss;
941 throw0(DB_ERROR(
"Failed to serialize pruned tx"));
942 unprunable_size = ss.str().size();
945 if (unprunable_size > blob.size())
946 throw0(DB_ERROR(
"pruned tx size is larger than tx size"));
948 MDB_val pruned_blob = {unprunable_size, (
void*)blob.data()};
951 throw0(DB_ERROR(lmdb_error(
"Failed to add pruned tx blob to db transaction: ", result).c_str()));
953 MDB_val prunable_blob = {blob.size() - unprunable_size, (
void*)(blob.data() + unprunable_size)};
956 throw0(DB_ERROR(lmdb_error(
"Failed to add prunable tx blob to db transaction: ", result).c_str()));
958 if (get_blockchain_pruning_seed())
963 throw0(DB_ERROR(lmdb_error(
"Failed to add prunable tx id to db transaction: ", result).c_str()));
971 void BlockchainLMDB::remove_transaction_data(
const crypto::hash& tx_hash,
const transaction& tx)
989 throw1(TX_DNE(
"Attempting to remove transaction that isn't in the db"));
994 throw1(DB_ERROR(lmdb_error(
"Failed to locate pruned tx for removal: ", result).c_str()));
997 throw1(DB_ERROR(lmdb_error(
"Failed to add removal of pruned tx to db transaction: ", result).c_str()));
1004 throw1(DB_ERROR(lmdb_error(
"Failed to add removal of prunable tx to db transaction: ", result).c_str()));
1007 throw1(DB_ERROR(lmdb_error(
"Failed to locate prunable tx for removal: ", result).c_str()));
1011 throw1(DB_ERROR(lmdb_error(
"Failed to locate tx id for removal: ", result).c_str()));
1016 throw1(DB_ERROR(lmdb_error(
"Error adding removal of tx id to db transaction", result).c_str()));
1019 if (tx.version == 1) {
1020 remove_tx_outputs(tip->data.tx_id, tx);
1023 LOG_PRINT_L1(
"tx has no outputs to remove: " << tx_hash);
1025 throw1(DB_ERROR(lmdb_error(
"Failed to locate tx outputs for removal: ", result).c_str()));
1029 throw1(DB_ERROR(lmdb_error(
"Failed to add removal of tx outputs to db transaction: ", result).c_str()));
1035 throw1(DB_ERROR(
"Failed to add removal of tx index to db transaction"));
1039 const tx_out& tx_output,
1048 uint64_t m_num_outputs = num_outputs();
1055 if (tx_output.target.type() !=
typeid(txout_to_key))
1056 throw0(DB_ERROR(
"Wrong output type: expected txout_to_key"));
1057 if (tx_output.amount == 0 && !commitment)
1058 throw0(DB_ERROR(
"RCT output without commitment"));
1060 outtx ot = {m_num_outputs, tx_hash, local_index};
1065 throw0(DB_ERROR(lmdb_error(
"Failed to add output tx hash to db transaction: ", result).c_str()));
1069 MDB_val_copy<uint64_t> val_amount(tx_output.amount);
1076 throw0(DB_ERROR(
std::string(
"Failed to get number of outputs for amount: ").append(
mdb_strerror(result)).c_str()));
1077 ok.amount_index = num_elems;
1080 throw0(DB_ERROR(lmdb_error(
"Failed to get output amount in db transaction: ", result).c_str()));
1082 ok.amount_index = 0;
1083 ok.output_id = m_num_outputs;
1084 ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).
key;
1085 ok.data.unlock_time = unlock_time;
1086 ok.data.height = m_height;
1087 if (tx_output.amount == 0)
1089 ok.data.commitment = *commitment;
1099 throw0(DB_ERROR(lmdb_error(
"Failed to add output pubkey to db transaction: ", result).c_str()));
1101 return ok.amount_index;
1104 void BlockchainLMDB::add_tx_amount_output_indices(
const uint64_t tx_id,
1105 const std::vector<uint64_t>& amount_output_indices)
1114 size_t num_outputs = amount_output_indices.size();
1118 v.
mv_data = num_outputs ? (
void *)amount_output_indices.data() : (
void*)
"";
1124 throw0(DB_ERROR(
std::string(
"Failed to add <tx hash, amount output index array> to db transaction: ").append(
mdb_strerror(result)).c_str()));
1127 void BlockchainLMDB::remove_tx_outputs(
const uint64_t tx_id,
const transaction& tx)
1131 std::vector<std::vector<uint64_t>> amount_output_indices_set = get_tx_amount_output_indices(tx_id, 1);
1132 const std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.front();
1134 if (amount_output_indices.empty())
1136 if (tx.vout.empty())
1137 LOG_PRINT_L2(
"tx has no outputs, so no output indices");
1139 throw0(DB_ERROR(
"tx has outputs, but no output indices found"));
1142 for (
size_t i = tx.vout.size(); i-- > 0;)
1144 remove_output(tx.vout[i].amount, amount_output_indices[i]);
1148 void BlockchainLMDB::remove_output(
const uint64_t amount,
const uint64_t& out_index)
1161 throw1(OUTPUT_DNE(
"Attempting to get an output index by amount and amount index, but amount not found"));
1163 throw0(DB_ERROR(lmdb_error(
"DB error attempting to get an output", result).c_str()));
1170 throw0(DB_ERROR(
"Unexpected: global output index not found in m_output_txs"));
1174 throw1(DB_ERROR(lmdb_error(
"Error adding removal of output tx to db transaction", result).c_str()));
1178 throw0(DB_ERROR(lmdb_error(
std::string(
"Error deleting output index ").append(boost::lexical_cast<std::string>(out_index).append(
": ")).c_str(), result).c_str()));
1183 throw0(DB_ERROR(lmdb_error(
std::string(
"Error deleting amount for output index ").append(boost::lexical_cast<std::string>(out_index).append(
": ")).c_str(), result).c_str()));
1186 void BlockchainLMDB::prune_outputs(
uint64_t amount)
1194 MINFO(
"Pruning outputs for amount " << amount);
1202 throw0(DB_ERROR(lmdb_error(
"Error looking up outputs: ", result).c_str()));
1207 MINFO(num_elems <<
" outputs found");
1208 std::vector<uint64_t> output_ids;
1209 output_ids.reserve(num_elems);
1213 output_ids.push_back(okp->output_id);
1214 MDEBUG(
"output id " << okp->output_id);
1219 throw0(DB_ERROR(lmdb_error(
"Error counting outputs: ", result).c_str()));
1221 if (output_ids.size() != num_elems)
1222 throw0(DB_ERROR(
"Unexpected number of outputs"));
1226 throw0(DB_ERROR(lmdb_error(
"Error deleting outputs: ", result).c_str()));
1228 for (
uint64_t output_id: output_ids)
1233 throw0(DB_ERROR(lmdb_error(
"Error looking up output: ", result).c_str()));
1236 throw0(DB_ERROR(lmdb_error(
"Error deleting output: ", result).c_str()));
1248 MDB_val k = {
sizeof(k_image), (
void *)&k_image};
1251 throw1(KEY_IMAGE_EXISTS(
"Attempting to add spent key image that's already in the db"));
1253 throw1(DB_ERROR(lmdb_error(
"Error adding spent key image to db transaction: ", result).c_str()));
1265 MDB_val k = {
sizeof(k_image), (
void *)&k_image};
1268 throw1(DB_ERROR(lmdb_error(
"Error finding spent key to remove", result).c_str()));
1273 throw1(DB_ERROR(lmdb_error(
"Error adding removal of key image to db transaction", result).c_str()));
1277 blobdata BlockchainLMDB::output_to_blob(
const tx_out& output)
const 1282 throw1(DB_ERROR(
"Error serializing output to blob"));
1286 tx_out BlockchainLMDB::output_from_blob(
const blobdata& blob)
const 1289 std::stringstream ss;
1295 throw1(DB_ERROR(
"Error deserializing tx output blob"));
1300 blobdata BlockchainLMDB::validator_to_blob(
const validator_db& v)
const 1306 throw1(DB_ERROR(
"Error serializing validator to blob"));
1310 validator_db BlockchainLMDB::validator_from_blob(
const blobdata blob)
const 1313 std::stringstream ss;
1319 throw1(DB_ERROR(
"Error deserializing validator blob"));
1324 BlockchainLMDB::~BlockchainLMDB()
1331 try { batch_abort(); }
1343 m_folder =
"thishsouldnotexistbecauseitisgibberish";
1345 m_batch_transactions = batch_transactions;
1346 m_write_txn =
nullptr;
1347 m_write_batch_txn =
nullptr;
1348 m_batch_active =
false;
1365 throw0(
DB_OPEN_FAILURE(
"Attempted to open db, but it's already open"));
1367 boost::filesystem::path direc(filename);
1368 if (boost::filesystem::exists(direc))
1370 if (!boost::filesystem::is_directory(direc))
1371 throw0(
DB_OPEN_FAILURE(
"LMDB needs a directory path, but a file was passed"));
1375 if (!boost::filesystem::create_directories(direc))
1380 boost::filesystem::path old_files = direc.parent_path();
1384 LOG_PRINT_L0(
"Found existing LMDB files in " << old_files.string());
1386 throw DB_ERROR(
"Database could not be opened");
1389 boost::optional<bool> is_hdd_result =
tools::is_hdd(filename.c_str());
1392 if (is_hdd_result.value())
1396 m_folder = filename;
1406 throw0(
DB_ERROR(lmdb_error(
"Failed to create lmdb environment: ", result).c_str()));
1408 throw0(
DB_ERROR(lmdb_error(
"Failed to set max number of dbs: ", result).c_str()));
1411 if (threads > 110 &&
1413 throw0(
DB_ERROR(lmdb_error(
"Failed to set max number of readers: ", result).c_str()));
1415 size_t mapsize = DEFAULT_MAPSIZE;
1426 if (
auto result =
mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644))
1427 throw0(
DB_ERROR(lmdb_error(
"Failed to open lmdb environment: ", result).c_str()));
1433 if (cur_mapsize < mapsize)
1436 throw0(
DB_ERROR(lmdb_error(
"Failed to set max memory map size: ", result).c_str()));
1444 LOG_PRINT_L0(
"LMDB memory map needs to be resized, doing that now.");
1454 if (
auto mdb_res =
mdb_txn_begin(m_env, NULL, txn_flags, txn))
1455 throw0(
DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", mdb_res).c_str()));
1466 lmdb_db_open(txn, LMDB_TXS_PRUNED,
MDB_INTEGERKEY |
MDB_CREATE, m_txs_pruned,
"Failed to open db handle for m_txs_pruned");
1467 lmdb_db_open(txn, LMDB_TXS_PRUNABLE,
MDB_INTEGERKEY |
MDB_CREATE, m_txs_prunable,
"Failed to open db handle for m_txs_prunable");
1472 lmdb_db_open(txn, LMDB_TX_OUTPUTS,
MDB_INTEGERKEY |
MDB_CREATE, m_tx_outputs,
"Failed to open db handle for m_tx_outputs");
1479 lmdb_db_open(txn, LMDB_TXPOOL_META,
MDB_CREATE, m_txpool_meta,
"Failed to open db handle for m_txpool_meta");
1480 lmdb_db_open(txn, LMDB_TXPOOL_BLOB,
MDB_CREATE, m_txpool_blob,
"Failed to open db handle for m_txpool_blob");
1486 lmdb_db_open(txn, LMDB_HF_STARTING_HEIGHTS,
MDB_CREATE, m_hf_starting_heights,
"Failed to open db handle for m_hf_starting_heights");
1488 lmdb_db_open(txn, LMDB_HF_VERSIONS,
MDB_INTEGERKEY |
MDB_CREATE, m_hf_versions,
"Failed to open db handle for m_hf_versions");
1490 lmdb_db_open(txn, LMDB_VALIDATORS,
MDB_INTEGERKEY |
MDB_CREATE, m_validators,
"Failed to open db handle for m_validators");
1491 lmdb_db_open(txn, LMDB_UTXOS,
MDB_CREATE, m_utxos,
"Failed to open db handle for m_utxos");
1499 lmdb_db_open(txn, LMDB_TX_INPUTS,
MDB_CREATE, m_tx_inputs,
"Failed to open db handle for m_tx_inputs");
1500 lmdb_db_open(txn, LMDB_PROPERTIES,
MDB_CREATE, m_properties,
"Failed to open db handle for m_properties");
1532 result =
mdb_drop(txn, m_hf_starting_heights, 1);
1534 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_hf_starting_heights: ", result).c_str()));
1539 if ((result =
mdb_stat(txn, m_blocks, &db_stats)))
1540 throw0(
DB_ERROR(lmdb_error(
"Failed to query m_blocks: ", result).c_str()));
1544 bool compatible =
true;
1548 auto get_result =
mdb_get(txn, m_properties, &k, &v);
1554 MWARNING(
"Existing lmdb database was made by a later version (" << db_version <<
"). We don't know how it will change yet.");
1558 else if (db_version <
VERSION)
1565 MFATAL(
"Existing lmdb database needs to be converted, which cannot be done on a read-only database.");
1566 MFATAL(
"Please run electroneumd once to convert the database.");
1574 migrate(db_version);
1584 if (
VERSION > 0 && m_height > 0)
1593 MFATAL(
"Existing lmdb database is incompatible with this version.");
1594 MFATAL(
"Please delete the existing database and resync.");
1604 MDB_val_copy<uint32_t> v(
VERSION);
1605 auto put_result =
mdb_put(txn, m_properties, &k, &v, 0);
1611 MERROR(
"Failed to write version to database.");
1629 LOG_PRINT_L3(
"close() first calling batch_abort() due to active batch transaction");
1652 throw0(
DB_ERROR(lmdb_error(
"Failed to sync database: ", result).c_str()));
1658 MINFO(
"switching safe mode " << (onoff ?
"on" :
"off"));
1669 throw0(
DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
1671 if (
auto result =
mdb_drop(txn, m_blocks, 0))
1672 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_blocks: ", result).c_str()));
1673 if (
auto result =
mdb_drop(txn, m_block_info, 0))
1674 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_block_info: ", result).c_str()));
1675 if (
auto result =
mdb_drop(txn, m_block_heights, 0))
1676 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_block_heights: ", result).c_str()));
1677 if (
auto result =
mdb_drop(txn, m_txs_pruned, 0))
1678 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_txs_pruned: ", result).c_str()));
1679 if (
auto result =
mdb_drop(txn, m_txs_prunable, 0))
1680 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_txs_prunable: ", result).c_str()));
1681 if (
auto result =
mdb_drop(txn, m_txs_prunable_hash, 0))
1682 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_txs_prunable_hash: ", result).c_str()));
1683 if (
auto result =
mdb_drop(txn, m_txs_prunable_tip, 0))
1684 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_txs_prunable_tip: ", result).c_str()));
1685 if (
auto result =
mdb_drop(txn, m_tx_indices, 0))
1686 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_tx_indices: ", result).c_str()));
1687 if (
auto result =
mdb_drop(txn, m_tx_outputs, 0))
1688 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_tx_outputs: ", result).c_str()));
1689 if (
auto result =
mdb_drop(txn, m_output_txs, 0))
1690 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_output_txs: ", result).c_str()));
1691 if (
auto result =
mdb_drop(txn, m_output_amounts, 0))
1692 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_output_amounts: ", result).c_str()));
1693 if (
auto result =
mdb_drop(txn, m_spent_keys, 0))
1694 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_spent_keys: ", result).c_str()));
1695 (void)
mdb_drop(txn, m_hf_starting_heights, 0);
1696 if (
auto result =
mdb_drop(txn, m_hf_versions, 0))
1697 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_hf_versions: ", result).c_str()));
1698 if (
auto result =
mdb_drop(txn, m_validators, 0))
1699 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_validators: ", result).c_str()));
1700 if (
auto result =
mdb_drop(txn, m_utxos, 0))
1701 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_utxos: ", result).c_str()));
1702 if (
auto result =
mdb_drop(txn, m_addr_outputs, 0))
1703 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_addr_outputs: ", result).c_str()));
1704 if (
auto result =
mdb_drop(txn, m_addr_txs, 0))
1705 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_addr_txs: ", result).c_str()));
1706 if (
auto result =
mdb_drop(txn, m_addr_txs_old, 0))
1707 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_addr_txs_old: ", result).c_str()));
1708 if (
auto result =
mdb_drop(txn, m_tx_inputs, 0))
1709 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_tx_inputs: ", result).c_str()));
1710 if (
auto result =
mdb_drop(txn, m_properties, 0))
1711 throw0(
DB_ERROR(lmdb_error(
"Failed to drop m_properties: ", result).c_str()));
1715 MDB_val_copy<uint32_t> v(
VERSION);
1716 if (
auto result =
mdb_put(txn, m_properties, &k, &v, 0))
1717 throw0(
DB_ERROR(lmdb_error(
"Failed to write version to database: ", result).c_str()));
1727 std::vector<std::string> filenames;
1729 boost::filesystem::path datafile(m_folder);
1731 boost::filesystem::path lockfile(m_folder);
1734 filenames.push_back(datafile.string());
1735 filenames.push_back(lockfile.string());
1742 const std::string filename = folder +
"/data.mdb";
1745 boost::filesystem::remove(filename);
1747 catch (
const std::exception &e)
1749 MERROR(
"Failed to remove " << filename <<
": " << e.what());
1777 #define TXN_PREFIX(flags); \ 1778 mdb_txn_safe auto_txn; \ 1779 mdb_txn_safe* txn_ptr = &auto_txn; \ 1780 if (m_batch_active) \ 1781 txn_ptr = m_write_txn; \ 1784 if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \ 1785 throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \ 1788 #define TXN_PREFIX_RDONLY() \ 1790 mdb_txn_cursors *m_cursors; \ 1791 mdb_txn_safe auto_txn; \ 1792 bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); \ 1793 if (my_rtxn) auto_txn.m_tinfo = m_tinfo.get(); \ 1794 else auto_txn.uncheck() 1795 #define TXN_POSTFIX_RDONLY() 1797 #define TXN_POSTFIX_SUCCESS() \ 1799 if (! m_batch_active) \ 1800 auto_txn.commit(); \ 1812 #define TXN_BLOCK_PREFIX(flags); \ 1813 mdb_txn_safe auto_txn; \ 1814 mdb_txn_safe* txn_ptr = &auto_txn; \ 1815 if (m_batch_active || m_write_txn) \ 1816 txn_ptr = m_write_txn; \ 1819 if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \ 1820 throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \ 1823 #define TXN_BLOCK_POSTFIX_SUCCESS() \ 1825 if (! m_batch_active && ! m_write_txn) \ 1826 auto_txn.commit(); \ 1830 void BlockchainLMDB::add_chainstate_utxo(
const crypto::hash tx_hash,
const uint32_t relative_out_index,
1841 chainstate_key_t index;
1842 index.tx_hash = tx_hash;
1843 index.relative_out_index = relative_out_index;
1845 chainstate_value_t data;
1846 data.amount = amount;
1847 data.combined_key = combined_key;
1849 data.unlock_time = unlock_time;
1856 throw1(UTXO_EXISTS(
"Attempting to add utxo that's already in the db"));
1858 throw1(DB_ERROR(lmdb_error(
"Error adding utxo to db transaction: ", result).c_str()));
1862 bool BlockchainLMDB::check_chainstate_utxo(
const crypto::hash tx_hash,
const uint32_t relative_out_index)
1870 chainstate_key_t index;
1871 index.tx_hash = tx_hash;
1872 index.relative_out_index = relative_out_index;
1874 MDB_val k = {
sizeof(index), (
void *)&index};
1880 throw1(DB_ERROR(lmdb_error(
"Error finding utxo: ", result).c_str()));
1894 chainstate_key_t index;
1895 index.tx_hash = tx_hash;
1896 index.relative_out_index = relative_out_index;
1898 MDB_val k = {
sizeof(index), (
void *)&index};
1905 throw1(DB_ERROR(lmdb_error(
"Error finding utxo: ", result).c_str()));
1907 auto res = *(
const chainstate_value_t *) v.
mv_data;
1909 return res.unlock_time;
1913 void BlockchainLMDB::remove_chainstate_utxo(
const crypto::hash tx_hash,
const uint32_t relative_out_index)
1921 chainstate_key_t index;
1922 index.tx_hash = tx_hash;
1923 index.relative_out_index = relative_out_index;
1925 MDB_val k = {
sizeof(index), (
void *)&index};
1929 throw1(DB_ERROR(lmdb_error(
"Error finding utxo to remove", result).c_str()));
1934 throw1(DB_ERROR(lmdb_error(
"Error adding removal of utxo to db transaction", result).c_str()));
1954 chainstate_key_t
key;
1955 key.tx_hash = tx_hash;
1956 key.relative_out_index = relative_out_index;
1959 data.tx_hash = parent_tx_hash;
1960 data.in_index = in_index;
1967 throw1(UTXO_EXISTS(
"Attempting to add tx input that's already in the db"));
1969 throw1(DB_ERROR(lmdb_error(
"Error adding tx input to db transaction: ", result).c_str()));
1984 key.tx_hash = tx_hash;
1985 key.relative_out_index = relative_out_index;
1993 throw1(
DB_ERROR(lmdb_error(
"Error finding tx input: ", result).c_str()));
1999 void BlockchainLMDB::remove_tx_input(
const crypto::hash tx_hash,
const uint32_t relative_out_index)
2008 key.tx_hash = tx_hash;
2009 key.relative_out_index = relative_out_index;
2015 throw1(DB_ERROR(lmdb_error(
"Error finding tx input to remove", result).c_str()));
2020 throw1(DB_ERROR(lmdb_error(
"Error adding removal of tx input to db transaction", result).c_str()));
2024 void BlockchainLMDB::add_addr_output(
const crypto::hash tx_hash,
const uint32_t relative_out_index,
2035 MDB_val k = {
sizeof(combined_key), (
void *)&combined_key};
2039 throw1(DB_ERROR(lmdb_error(
"Error finding addr output to add: ", result).c_str()));
2047 throw0(DB_ERROR(
std::string(
"Failed to get number outputs for address: ").append(
mdb_strerror(result)).c_str()));
2050 num_elems =
res.db_index + 1;
2055 acc.tx_hash = tx_hash;
2056 acc.relative_out_index = relative_out_index;
2057 acc.amount = amount;
2058 acc.unlock_time = unlock_time;
2060 k = {
sizeof(combined_key), (
void *)&combined_key};
2061 MDB_val acc_v = {
sizeof(acc), (
void *)&acc};
2065 throw1(UTXO_EXISTS(
"Attempting to add addr output that's already in the db."));
2066 else if(result != 0)
2067 throw1(DB_ERROR(lmdb_error(
"Error adding addr output to db transaction: ", result).c_str()));
2082 MDB_val k = {
sizeof(combined_key), (
void *)&combined_key};
2092 throw0(
DB_ERROR(
"Failed to enumerate address outputs"));
2098 addr_out.tx_hash =
res.tx_hash;
2099 addr_out.relative_out_index =
res.relative_out_index;
2100 addr_out.amount =
res.amount;
2101 addr_out.spent = !check_chainstate_utxo(
res.tx_hash,
res.relative_out_index);
2122 MDB_val k = {
sizeof(combined_key), (
void *)&combined_key};
2133 throw1(
DB_ERROR(lmdb_error(
"Failed to enumerate address outputs", result).c_str()));
2136 std::set<std::string> tx_hashes;
2137 for(
size_t i = 0; i < batch_size + 1; ++i) {
2139 v =
MDB_val{
sizeof(start_db_index), (
void*)&start_db_index};
2146 throw0(
DB_ERROR(
"Failed to enumerate address outputs"));
2151 if(tx_hashes.find(tx_hash_hex) != tx_hashes.end())
2162 addr_out.
spent = !check_chainstate_utxo(
res.tx_hash,
res.relative_out_index);
2165 tx_hashes.emplace(tx_hash_hex);
2181 MDB_val k = {
sizeof(combined_key), (
void *)&combined_key};
2185 throw1(DB_ERROR(lmdb_error(
"Error finding addr tx to add: ", result).c_str()));
2196 num_elems =
res.db_index + 1;
2201 acc.tx_hash = tx_hash;
2203 k = {
sizeof(combined_key), (
void *)&combined_key};
2204 MDB_val acc_v = {
sizeof(acc), (
void *)&acc};
2208 throw1(UTXO_EXISTS(
"Attempting to add addr tx that's already in the db."));
2209 else if(result != 0)
2210 throw1(DB_ERROR(lmdb_error(
"Error adding addr tx to db transaction: ", result).c_str()));
2225 MDB_val k = {
sizeof(combined_key), (
void *)&combined_key};
2235 throw0(
DB_ERROR(
"Failed to enumerate address txs"));
2241 addr_tx.tx_hash =
res.tx_hash;
2259 MDB_val k = {
sizeof(combined_key), (
void *)&combined_key};
2270 throw1(
DB_ERROR(lmdb_error(
"Failed to enumerate address txs", result).c_str()));
2273 std::set<std::string> tx_hashes;
2274 for(
size_t i = 0; i < batch_size + 1; ++i) {
2276 v =
MDB_val{
sizeof(start_db_index), (
void*)&start_db_index};
2283 throw0(
DB_ERROR(
"Failed to enumerate address txs"));
2289 addr_tx.tx_hash =
res.tx_hash;
2308 MDB_val k = {
sizeof(combined_key), (
void *)&combined_key};
2318 throw0(
DB_ERROR(
"Failed to enumerate address outputs"));
2322 if(check_chainstate_utxo(
res.tx_hash,
res.relative_out_index))
2323 balance +=
res.amount;
2331 void BlockchainLMDB::remove_addr_output(
const crypto::hash tx_hash,
const uint32_t relative_out_index,
2343 MDB_val k = {
sizeof(combined_key), (
void *)&combined_key};
2348 throw1(DB_ERROR(lmdb_error(
"Failed to enumerate address outputs", result).c_str()));
2354 k = {
sizeof(combined_key), (
void *)&combined_key};
2360 throw0(DB_ERROR(
"Failed to enumerate outputs"));
2364 if(
res.tx_hash == tx_hash &&
res.relative_out_index == relative_out_index &&
res.amount == amount &&
res.unlock_time == unlock_time ) {
2367 throw1(DB_ERROR(lmdb_error(
"Error removing of addr output from db: ", result).c_str()));
2384 MDB_val k = {
sizeof(combined_key), (
void *)&combined_key};
2389 throw1(DB_ERROR(lmdb_error(
"Failed to enumerate address txs", result).c_str()));
2395 k = {
sizeof(combined_key), (
void *)&combined_key};
2401 throw0(DB_ERROR(
"Failed to enumerate outputs"));
2405 if(
res.tx_hash == tx_hash) {
2408 throw1(DB_ERROR(lmdb_error(
"Error removing of addr tx from db: ", result).c_str()));
2424 MDB_val k = {
sizeof(txid), (
void *)&txid};
2425 MDB_val v = {
sizeof(meta), (
void *)&meta};
2428 throw1(
DB_ERROR(
"Attempting to add txpool tx metadata that's already in the db"));
2430 throw1(
DB_ERROR(lmdb_error(
"Error adding txpool tx metadata to db transaction: ", result).c_str()));
2435 throw1(
DB_ERROR(
"Attempting to add txpool tx blob that's already in the db"));
2437 throw1(
DB_ERROR(lmdb_error(
"Error adding txpool tx blob to db transaction: ", result).c_str()));
2450 MDB_val k = {
sizeof(txid), (
void *)&txid};
2454 throw1(
DB_ERROR(lmdb_error(
"Error finding txpool tx meta to update: ", result).c_str()));
2457 throw1(
DB_ERROR(lmdb_error(
"Error adding removal of txpool tx metadata to db transaction: ", result).c_str()));
2458 v =
MDB_val({
sizeof(meta), (
void *)&meta});
2461 throw1(
DB_ERROR(
"Attempting to add txpool tx metadata that's already in the db"));
2463 throw1(
DB_ERROR(lmdb_error(
"Error adding txpool tx metadata to db transaction: ", result).c_str()));
2477 if (include_unrelayed_txes)
2481 if ((result =
mdb_stat(m_txn, m_txpool_meta, &db_stats)))
2482 throw0(
DB_ERROR(lmdb_error(
"Failed to query m_txpool_meta: ", result).c_str()));
2501 throw0(
DB_ERROR(lmdb_error(
"Failed to enumerate txpool tx metadata: ", result).c_str()));
2520 MDB_val k = {
sizeof(txid), (
void *)&txid};
2523 throw1(
DB_ERROR(lmdb_error(
"Error finding txpool tx meta: ", result).c_str()));
2537 MDB_val k = {
sizeof(txid), (
void *)&txid};
2540 throw1(
DB_ERROR(lmdb_error(
"Error finding txpool tx meta to remove: ", result).c_str()));
2545 throw1(
DB_ERROR(lmdb_error(
"Error adding removal of txpool tx metadata to db transaction: ", result).c_str()));
2549 throw1(
DB_ERROR(lmdb_error(
"Error finding txpool tx blob to remove: ", result).c_str()));
2554 throw1(
DB_ERROR(lmdb_error(
"Error adding removal of txpool tx blob to db transaction: ", result).c_str()));
2566 MDB_val k = {
sizeof(txid), (
void *)&txid};
2572 throw1(
DB_ERROR(lmdb_error(
"Error finding txpool tx meta: ", result).c_str()));
2587 MDB_val k = {
sizeof(txid), (
void *)&txid};
2593 throw1(
DB_ERROR(lmdb_error(
"Error finding txpool tx blob: ", result).c_str()));
2604 throw1(
DB_ERROR(
"Tx not found in txpool: "));
2621 throw0(
DB_ERROR(lmdb_error(
"Failed to retrieve pruning seed: ", result).c_str()));
2623 throw0(
DB_ERROR(
"Failed to retrieve or create pruning seed: unexpected value size"));
2627 return pruning_seed;
2635 throw0(
DB_ERROR(lmdb_error(
"Failed to find transaction pruned data: ", ret).c_str()));
2637 throw0(
DB_ERROR(
"Invalid transaction pruned data"));
2643 bool BlockchainLMDB::prune_worker(
int mode,
uint32_t pruning_seed)
2648 throw0(DB_ERROR(
"Pruning seed not in range"));
2651 throw0(DB_ERROR(
"Pruning seed not in range"));
2656 size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0, commit_counter = 0;
2662 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
2665 if ((result =
mdb_stat(txn, m_txs_prunable, &db_stats)))
2666 throw0(DB_ERROR(lmdb_error(
"Failed to query m_txs_prunable: ", result).c_str()));
2671 result =
mdb_get(txn, m_properties, &k, &v);
2672 bool prune_tip_table =
false;
2680 MDEBUG(
"Pruning not enabled, nothing to do");
2683 if (pruning_seed == 0)
2687 v.
mv_size =
sizeof(pruning_seed);
2688 result =
mdb_put(txn, m_properties, &k, &v, 0);
2690 throw0(DB_ERROR(
"Failed to save pruning seed"));
2691 prune_tip_table =
false;
2693 else if (result == 0)
2697 throw0(DB_ERROR(
"Failed to retrieve or create pruning seed: unexpected value size"));
2699 if (pruning_seed == 0)
2702 throw0(DB_ERROR(
"Blockchain already pruned with different seed"));
2704 throw0(DB_ERROR(
"Blockchain already pruned with different base"));
2710 throw0(DB_ERROR(lmdb_error(
"Failed to retrieve or create pruning seed: ", result).c_str()));
2714 MINFO(
"Checking blockchain pruning...");
2716 MINFO(
"Pruning blockchain...");
2718 MDB_cursor *c_txs_pruned, *c_txs_prunable, *c_txs_prunable_tip;
2721 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_pruned: ", result).c_str()));
2724 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_prunable: ", result).c_str()));
2725 result =
mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip);
2727 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_prunable_tip: ", result).c_str()));
2730 if (prune_tip_table)
2740 throw0(DB_ERROR(lmdb_error(
"Failed to enumerate transactions: ", ret).c_str()));
2749 ++n_prunable_records;
2752 MWARNING(
"Already pruned at height " << block_height <<
"/" << blockchain_height);
2754 throw0(DB_ERROR(lmdb_error(
"Failed to find transaction prunable data: ", result).c_str()));
2757 MDEBUG(
"Pruning at height " << block_height <<
"/" << blockchain_height);
2763 throw0(DB_ERROR(lmdb_error(
"Failed to delete transaction prunable data: ", result).c_str()));
2768 throw0(DB_ERROR(lmdb_error(
"Failed to delete transaction tip data: ", result).c_str()));
2772 MDEBUG(
"Committing txn at checkpoint...");
2776 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
2779 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_pruned: ", result).c_str()));
2782 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_prunable: ", result).c_str()));
2783 result =
mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip);
2785 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_prunable_tip: ", result).c_str()));
2796 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for tx_indices: ", result).c_str()));
2805 throw0(DB_ERROR(lmdb_error(
"Failed to enumerate transactions: ", ret).c_str()));
2811 const uint64_t block_height = ti.data.block_id;
2820 throw0(DB_ERROR(lmdb_error(
"Error looking for transaction prunable data: ", result).c_str()));
2822 MERROR(
"Transaction not found in prunable tip table for height " << block_height <<
"/" << blockchain_height <<
2829 throw0(DB_ERROR(lmdb_error(
"Error looking for transaction prunable data: ", result).c_str()));
2837 throw0(DB_ERROR(lmdb_error(
"Error looking for transaction prunable data: ", result).c_str()));
2841 MERROR(
"Prunable data found for pruned height " << block_height <<
"/" << blockchain_height <<
2846 ++n_prunable_records;
2848 MWARNING(
"Already pruned at height " << block_height <<
"/" << blockchain_height);
2851 MDEBUG(
"Pruning at height " << block_height <<
"/" << blockchain_height);
2853 n_bytes += kp.mv_size + v.
mv_size;
2856 throw0(DB_ERROR(lmdb_error(
"Failed to delete transaction prunable data: ", result).c_str()));
2868 throw0(DB_ERROR(lmdb_error(
"Error looking for transaction prunable data: ", result).c_str()));
2870 MERROR(
"Prunable data not found for unpruned height " << block_height <<
"/" << blockchain_height <<
2877 MDEBUG(
"Committing txn at checkpoint...");
2881 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
2884 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_pruned: ", result).c_str()));
2887 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_prunable: ", result).c_str()));
2888 result =
mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip);
2890 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_prunable_tip: ", result).c_str()));
2893 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for tx_indices: ", result).c_str()));
2899 throw0(DB_ERROR(lmdb_error(
"Failed to restore cursor for tx_indices: ", result).c_str()));
2906 if ((result =
mdb_stat(txn, m_txs_prunable, &db_stats)))
2907 throw0(DB_ERROR(lmdb_error(
"Failed to query m_txs_prunable: ", result).c_str()));
2909 const size_t db_bytes = (pages0 - pages1) * db_stats.
ms_psize;
2920 t <<
" ms: " << (n_bytes/1024.0f/1024.0f) <<
" MB (" << db_bytes/1024.0f/1024.0f <<
" MB) pruned in " <<
2921 n_pruned_records <<
" records (" << pages0 - pages1 <<
"/" << pages0 <<
" " << db_stats.
ms_psize <<
" byte pages), " <<
2922 n_prunable_records <<
"/" << n_total_records <<
" pruned records");
2962 throw0(
DB_ERROR(lmdb_error(
"Failed to enumerate txpool tx metadata: ", result).c_str()));
2965 if (!include_unrelayed_txes && meta.do_not_relay)
2975 throw0(
DB_ERROR(
"Failed to find txpool tx blob to match metadata"));
2977 throw0(
DB_ERROR(lmdb_error(
"Failed to enumerate txpool tx blob: ", result).c_str()));
2982 if (!f(txid, meta, passed_bd)) {
3008 else if (get_result)
3009 throw0(
DB_ERROR(lmdb_error(
"DB error attempting to fetch block index from hash", get_result).c_str()));
3043 throw1(
BLOCK_DNE(
"Attempted to retrieve non-existent block height"));
3044 else if (get_result)
3045 throw0(
DB_ERROR(
"Error attempting to retrieve a block height from the db"));
3075 throw0(
BLOCK_DNE(
std::string(
"Attempt to get block from height ").append(boost::lexical_cast<std::string>(
height)).append(
" failed -- block not in db").c_str()));
3077 else if (get_result)
3078 throw0(
DB_ERROR(
"Error attempting to retrieve a block from the db"));
3081 bd.assign(reinterpret_cast<char*>(result.
mv_data), result.
mv_size);
3100 throw0(
BLOCK_DNE(
std::string(
"Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(
height)).append(
" failed -- timestamp not in db").c_str()));
3102 else if (get_result)
3103 throw0(
DB_ERROR(
"Error attempting to retrieve a timestamp from the db"));
3115 std::vector<uint64_t>
res;
3118 if (heights.empty())
3120 res.reserve(heights.size());
3126 if ((result =
mdb_stat(m_txn, m_blocks, &db_stats)))
3127 throw0(
DB_ERROR(lmdb_error(
"Failed to query m_blocks: ", result).c_str()));
3128 for (
size_t i = 0; i < heights.size(); ++i)
3135 uint64_t range_begin = 0, range_end = 0;
3144 if (
height == prev_height + 1)
3150 if (height < range_begin || height >= range_end)
3159 range_end = range_begin + 1;
3162 throw0(
DB_ERROR(lmdb_error(
"Error attempting to retrieve rct distribution from the db: ", result).c_str()));
3200 throw0(
BLOCK_DNE(
std::string(
"Attempt to get block size from height ").append(boost::lexical_cast<std::string>(
height)).append(
" failed -- block size not in db").c_str()));
3202 else if (get_result)
3203 throw0(
DB_ERROR(
"Error attempting to retrieve a block size from the db"));
3211 std::vector<uint64_t> BlockchainLMDB::get_block_info_64bit_fields(
uint64_t start_height,
size_t count, off_t offset)
const 3220 if (start_height >= h)
3223 std::vector<uint64_t> ret;
3227 uint64_t range_begin = 0, range_end = 0;
3243 if (height < range_begin || height >= range_end)
3252 range_end = range_begin + 1;
3255 throw0(DB_ERROR(lmdb_error(
"Error attempting to retrieve block_info from the db: ", result).c_str()));
3258 ret.push_back(*(
const uint64_t*)(((
const char*)bi) + offset));
3265 uint64_t BlockchainLMDB::get_max_block_size()
3276 return std::numeric_limits<uint64_t>::max();
3278 throw0(DB_ERROR(lmdb_error(
"Failed to retrieve max block size: ", result).c_str()));
3280 throw0(DB_ERROR(
"Failed to retrieve or create max block size: unexpected value size"));
3284 return max_block_size;
3287 void BlockchainLMDB::add_max_block_size(
uint64_t sz)
3299 throw0(DB_ERROR(lmdb_error(
"Failed to retrieve max block size: ", result).c_str()));
3304 throw0(DB_ERROR(
"Failed to retrieve or create max block size: unexpected value size"));
3307 if (sz > max_block_size)
3308 max_block_size = sz;
3309 v.
mv_data = (
void*)&max_block_size;
3310 v.
mv_size =
sizeof(max_block_size);
3313 throw0(DB_ERROR(lmdb_error(
"Failed to set max_block_size: ", result).c_str()));
3319 return get_block_info_64bit_fields(start_height,
count, offsetof(
mdb_block_info, bi_weight));
3324 return get_block_info_64bit_fields(start_height,
count, offsetof(
mdb_block_info, bi_long_term_block_weight));
3341 throw0(
BLOCK_DNE(
std::string(
"Attempt to set cumulative difficulty from height ").append(boost::lexical_cast<std::string>(
height)).append(
" failed -- difficulty not in db").c_str()));
3344 throw0(
DB_ERROR(
"Error attempting to set a cumulative difficulty"));
3354 bi.bi_hash = result_bi->
bi_hash;
3359 throw0(
DB_ERROR(lmdb_error(
"Failed to set cumulative difficulty to db transaction: ", result).c_str()));
3375 throw0(
BLOCK_DNE(
std::string(
"Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(
height)).append(
" failed -- difficulty not in db").c_str()));
3377 else if (get_result)
3378 throw0(
DB_ERROR(
"Error attempting to retrieve a cumulative difficulty from the db"));
3402 return diff1 - diff2;
3417 throw0(
BLOCK_DNE(
std::string(
"Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(
height)).append(
" failed -- block size not in db").c_str()));
3419 else if (get_result)
3420 throw0(
DB_ERROR(
"Error attempting to retrieve a total generated coins from the db"));
3440 throw0(
BLOCK_DNE(
std::string(
"Attempt to get block long term weight from height ").append(boost::lexical_cast<std::string>(
height)).append(
" failed -- block info not in db").c_str()));
3442 else if (get_result)
3443 throw0(
DB_ERROR(
"Error attempting to retrieve a long term block weight from the db"));
3463 throw0(
BLOCK_DNE(
std::string(
"Attempt to get hash from height ").append(boost::lexical_cast<std::string>(
height)).append(
" failed -- hash not in db").c_str()));
3465 else if (get_result)
3466 throw0(
DB_ERROR(lmdb_error(
"Error attempting to retrieve a block hash from the db: ", get_result).c_str()));
3478 std::vector<block> v;
3492 std::vector<crypto::hash> v;
3508 *block_height = m_height - 1;
3541 if ((result =
mdb_stat(m_txn, m_blocks, &db_stats)))
3542 throw0(
DB_ERROR(lmdb_error(
"Failed to query m_blocks: ", result).c_str()));
3546 uint64_t BlockchainLMDB::num_outputs()
const 3560 else if (result == 0)
3563 throw0(
DB_ERROR(lmdb_error(
"Failed to query m_output_txs: ", result).c_str()));
3577 bool tx_found =
false;
3581 if (get_result == 0)
3626 else if (get_result)
3627 throw0(
DB_ERROR(lmdb_error(
"DB error attempting to fetch transaction from hash", get_result).c_str()));
3646 else if (get_result)
3647 throw0(
DB_ERROR(lmdb_error(
"DB error attempting to fetch tx data from hash: ", get_result).c_str()));
3668 if (get_result == 0)
3673 if (get_result == 0)
3680 else if (get_result)
3681 throw0(
DB_ERROR(lmdb_error(
"DB error attempting to fetch tx from hash", get_result).c_str()));
3683 bd.assign(reinterpret_cast<char*>(result0.
mv_data), result0.
mv_size);
3684 bd.append(reinterpret_cast<char*>(result1.
mv_data), result1.
mv_size);
3703 if (get_result == 0)
3711 else if (get_result)
3712 throw0(
DB_ERROR(lmdb_error(
"DB error attempting to fetch tx from hash", get_result).c_str()));
3714 bd.assign(reinterpret_cast<char*>(result.
mv_data), result.
mv_size);
3733 if (get_result == 0)
3741 else if (get_result)
3742 throw0(
DB_ERROR(lmdb_error(
"DB error attempting to fetch tx from hash", get_result).c_str()));
3744 bd.assign(reinterpret_cast<char*>(result.
mv_data), result.
mv_size);
3761 MDB_val result, val_tx_prunable_hash;
3763 if (get_result == 0)
3771 else if (get_result)
3772 throw0(
DB_ERROR(lmdb_error(
"DB error attempting to fetch tx prunable hash from tx hash", get_result).c_str()));
3790 if ((result =
mdb_stat(m_txn, m_txs_pruned, &db_stats)))
3791 throw0(
DB_ERROR(lmdb_error(
"Failed to query m_txs_pruned: ", result).c_str()));
3802 std::vector<transaction> v;
3804 for (
auto& h : hlist)
3826 else if (get_result)
3827 throw0(
DB_ERROR(lmdb_error(
"DB error attempting to fetch tx height from hash", get_result).c_str()));
3843 MDB_val_copy<uint64_t> k(amount);
3852 throw0(
DB_ERROR(
"DB error attempting to get number of outputs of an amount"));
3871 throw1(
OUTPUT_DNE(
std::string(
"Attempting to get output pubkey by index, but key does not exist: amount " +
3873 else if (get_result)
3874 throw0(
DB_ERROR(
"Error attempting to retrieve an output pubkey from the db"));
3885 memcpy(&ret, &okp->
data,
sizeof(pre_rct_output_data_t));;
3886 if (include_commitmemt)
3905 throw1(
OUTPUT_DNE(
"output with given index not in db"));
3906 else if (get_result)
3907 throw0(
DB_ERROR(
"DB error attempting to fetch output tx hash"));
3919 std::vector < uint64_t > offsets;
3920 std::vector<tx_out_index> indices;
3921 offsets.push_back(index);
3923 if (!indices.size())
3924 throw1(
OUTPUT_DNE(
"Attempting to get an output index by amount and amount index, but amount not found"));
3940 std::vector<std::vector<uint64_t>> amount_output_indices_set;
3941 amount_output_indices_set.reserve(n_txes);
3944 while (n_txes-- > 0)
3948 LOG_PRINT_L0(
"WARNING: Unexpected: tx has no amount indices stored in " 3949 "tx_outputs, but it should have an empty entry even if it's a tx without " 3952 throw0(
DB_ERROR(lmdb_error(
"DB error attempting to get data for tx_outputs[tx_index]", result).c_str()));
3959 amount_output_indices_set.resize(amount_output_indices_set.size() + 1);
3960 std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.back();
3961 amount_output_indices.reserve(num_outputs);
3962 for (
size_t i = 0; i < num_outputs; ++i)
3964 amount_output_indices.push_back(indices[i]);
3969 return amount_output_indices_set;
3982 MDB_val k = {
sizeof(img), (
void *)&img};
4009 throw0(
DB_ERROR(
"Failed to enumerate key images"));
4037 k =
MDB_val{
sizeof(h1), (
void*)&h1};
4050 throw0(
DB_ERROR(
"Failed to enumerate blocks"));
4056 throw0(
DB_ERROR(
"Failed to parse block from blob retrieved from the db"));
4059 throw0(
DB_ERROR(
"Failed to get block hash from blob retrieved from the db"));
4095 throw0(
DB_ERROR(lmdb_error(
"Failed to enumerate transactions: ", ret).c_str()));
4106 throw0(
DB_ERROR(lmdb_error(
"Failed to enumerate transactions: ", ret).c_str()));
4113 throw0(
DB_ERROR(
"Failed to parse tx from blob retrieved from the db"));
4119 throw0(
DB_ERROR(lmdb_error(
"Failed to get prunable tx data the db: ", ret).c_str()));
4122 throw0(
DB_ERROR(
"Failed to parse tx from blob retrieved from the db"));
4155 throw0(
DB_ERROR(
"Failed to enumerate outputs"));
4159 if (!f(amount, toi.first, ok->data.height, toi.second)) {
4190 throw0(
DB_ERROR(
"Failed to enumerate outputs"));
4192 if (amount != out_amount)
4194 MERROR(
"Amount is not the expected amount");
4214 if (! m_batch_transactions)
4215 throw0(
DB_ERROR(
"batch transactions not enabled"));
4218 if (m_write_batch_txn !=
nullptr)
4221 throw0(
DB_ERROR(
"batch transaction attempted, but m_write_txn already in use"));
4224 m_writer = boost::this_thread::get_id();
4225 check_and_resize_for_batch(batch_num_blocks, batch_bytes);
4230 if (
auto mdb_res =
lmdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn))
4232 delete m_write_batch_txn;
4233 m_write_batch_txn =
nullptr;
4234 throw0(
DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", mdb_res).c_str()));
4239 m_write_txn = m_write_batch_txn;
4241 m_batch_active =
true;
4242 memset(&m_wcursors, 0,
sizeof(m_wcursors));
4245 if (m_tinfo->m_ti_rflags.m_rf_txn)
4247 memset(&m_tinfo->m_ti_rflags, 0,
sizeof(m_tinfo->m_ti_rflags));
4257 if (! m_batch_transactions)
4258 throw0(
DB_ERROR(
"batch transactions not enabled"));
4259 if (! m_batch_active)
4260 throw1(
DB_ERROR(
"batch transaction not in progress"));
4261 if (m_write_batch_txn ==
nullptr)
4262 throw1(
DB_ERROR(
"batch transaction not in progress"));
4263 if (m_writer != boost::this_thread::get_id())
4264 throw1(
DB_ERROR(
"batch transaction owned by other thread"));
4275 m_write_txn =
nullptr;
4276 delete m_write_batch_txn;
4277 m_write_batch_txn =
nullptr;
4278 memset(&m_wcursors, 0,
sizeof(m_wcursors));
4281 void BlockchainLMDB::cleanup_batch()
4284 m_write_txn =
nullptr;
4285 delete m_write_batch_txn;
4286 m_write_batch_txn =
nullptr;
4287 m_batch_active =
false;
4288 memset(&m_wcursors, 0,
sizeof(m_wcursors));
4294 if (! m_batch_transactions)
4295 throw0(
DB_ERROR(
"batch transactions not enabled"));
4296 if (! m_batch_active)
4297 throw1(
DB_ERROR(
"batch transaction not in progress"));
4298 if (m_write_batch_txn ==
nullptr)
4299 throw1(
DB_ERROR(
"batch transaction not in progress"));
4300 if (m_writer != boost::this_thread::get_id())
4301 throw1(
DB_ERROR(
"batch transaction owned by other thread"));
4312 catch (
const std::exception &e)
4323 if (! m_batch_transactions)
4324 throw0(
DB_ERROR(
"batch transactions not enabled"));
4325 if (! m_batch_active)
4326 throw1(
DB_ERROR(
"batch transaction not in progress"));
4327 if (m_write_batch_txn ==
nullptr)
4328 throw1(
DB_ERROR(
"batch transaction not in progress"));
4329 if (m_writer != boost::this_thread::get_id())
4330 throw1(
DB_ERROR(
"batch transaction owned by other thread"));
4333 m_write_txn =
nullptr;
4335 m_write_batch_txn->
abort();
4336 delete m_write_batch_txn;
4337 m_write_batch_txn =
nullptr;
4338 m_batch_active =
false;
4339 memset(&m_wcursors, 0,
sizeof(m_wcursors));
4346 if ((batch_transactions) && (m_batch_transactions))
4348 MINFO(
"batch transaction mode already enabled, but asked to enable batch mode");
4350 m_batch_transactions = batch_transactions;
4351 MINFO(
"batch transactions " << (m_batch_transactions ?
"enabled" :
"disabled"));
4359 if (m_write_txn && m_writer == boost::this_thread::get_id()) {
4360 *mtxn = m_write_txn->
m_txn;
4370 m_tinfo.reset(tinfo);
4374 throw0(
DB_ERROR_TXN_START(lmdb_error(
"Failed to create a read transaction for the db: ", mdb_res).c_str()));
4379 throw0(
DB_ERROR_TXN_START(lmdb_error(
"Failed to renew a read transaction for the db: ", mdb_res).c_str()));
4396 memset(&m_tinfo->m_ti_rflags, 0,
sizeof(m_tinfo->m_ti_rflags));
4416 if (! m_batch_active && m_write_txn)
4418 if (! m_batch_active)
4420 m_writer = boost::this_thread::get_id();
4425 m_write_txn =
nullptr;
4426 throw0(
DB_ERROR_TXN_START(lmdb_error(
"Failed to create a transaction for the db: ", mdb_res).c_str()));
4428 memset(&m_wcursors, 0,
sizeof(m_wcursors));
4431 if (m_tinfo->m_ti_rflags.m_rf_txn)
4433 memset(&m_tinfo->m_ti_rflags, 0,
sizeof(m_tinfo->m_ti_rflags));
4435 }
else if (m_writer != boost::this_thread::get_id())
4444 if (m_writer != boost::this_thread::get_id())
4447 if (! m_batch_active)
4455 m_write_txn =
nullptr;
4456 memset(&m_wcursors, 0,
sizeof(m_wcursors));
4466 if (m_writer != boost::this_thread::get_id())
4469 if (! m_batch_active)
4472 m_write_txn =
nullptr;
4473 memset(&m_wcursors, 0,
sizeof(m_wcursors));
4481 memset(&m_tinfo->m_ti_rflags, 0,
sizeof(m_tinfo->m_ti_rflags));
4485 const std::vector<std::pair<transaction, blobdata>>& txs)
4491 if (m_height % 1024 == 0)
4494 if (! m_batch_active && need_resize())
4496 LOG_PRINT_L0(
"LMDB memory map needs to be resized, doing that now.");
4503 BlockchainDB::add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs);
4513 void BlockchainLMDB::pop_block(
block& blk, std::vector<transaction>& txs)
4522 BlockchainDB::pop_block(blk, txs);
4533 std::vector<tx_out_index> &tx_out_indices)
const 4537 tx_out_indices.clear();
4538 tx_out_indices.reserve(global_indices.size());
4543 for (
const uint64_t &output_id : global_indices)
4549 throw1(
OUTPUT_DNE(
"output with given index not in db"));
4550 else if (get_result)
4551 throw0(
DB_ERROR(
"DB error attempting to fetch output tx hash"));
4562 if (amounts.
size() != 1 && amounts.
size() != offsets.size())
4563 throw0(
DB_ERROR(
"Invalid sizes of amounts and offets"));
4569 outputs.reserve(offsets.size());
4575 for (
size_t i = 0; i < offsets.size(); ++i)
4577 const uint64_t amount = amounts.
size() == 1 ? amounts[0] : amounts[i];
4586 MDEBUG(
"Partial result: " << outputs.size() <<
"/" << offsets.size());
4589 throw1(
OUTPUT_DNE((
std::string(
"Attempting to get output pubkey by global index (amount ") + boost::lexical_cast<std::string>(amount) +
", index " + boost::lexical_cast<std::string>(offsets[i]) +
", count " + boost::lexical_cast<std::string>(
get_num_outputs(amount)) +
"), but key does not exist (current height " + boost::lexical_cast<std::string>(
height()) +
")").c_str()));
4591 else if (get_result)
4592 throw0(
DB_ERROR(lmdb_error(
"Error attempting to retrieve an output pubkey from the db", get_result).c_str()));
4597 outputs.push_back(okp->
data);
4602 outputs.resize(outputs.size() + 1);
4604 memcpy(&data, &okp->
data,
sizeof(pre_rct_output_data_t));
4621 std::vector <uint64_t> tx_indices;
4622 tx_indices.reserve(offsets.size());
4628 for (
const uint64_t &index : offsets)
4634 throw1(
OUTPUT_DNE(
"Attempting to get output by index, but key does not exist"));
4635 else if (get_result)
4636 throw0(
DB_ERROR(lmdb_error(
"Error attempting to retrieve an output from the db", get_result).c_str()));
4643 if(tx_indices.size() > 0)
4659 std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> histogram;
4663 if (amounts.empty())
4673 throw0(
DB_ERROR(lmdb_error(
"Failed to enumerate outputs: ", ret).c_str()));
4677 if (num_elems >= min_count)
4683 for (
const auto &amount: amounts)
4685 MDB_val_copy<uint64_t> k(amount);
4696 if (num_elems >= min_count)
4701 throw0(
DB_ERROR(lmdb_error(
"Failed to enumerate outputs: ", ret).c_str()));
4706 if (unlocked || recent_cutoff > 0) {
4708 for (std::map<
uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>::iterator i = histogram.begin(); i != histogram.end(); ++i) {
4710 uint64_t num_elems = std::get<0>(i->second);
4711 while (num_elems > 0) {
4719 std::get<1>(i->second) = num_elems;
4721 if (recent_cutoff > 0)
4724 while (num_elems > 0) {
4728 if (ts < recent_cutoff)
4734 std::get<2>(i->second) = recent;
4752 distribution.clear();
4754 if (from_height >= db_height)
4756 distribution.resize(db_height - from_height, 0);
4770 throw0(
DB_ERROR(
"Failed to enumerate outputs"));
4774 distribution[
height - from_height]++;
4777 if (to_height > 0 &&
height > to_height)
4781 distribution[0] += base;
4782 for (
size_t n = 1; n < distribution.size(); ++n)
4783 distribution[n] += distribution[n - 1];
4791 void BlockchainLMDB::check_hard_fork_info()
4795 void BlockchainLMDB::drop_hard_fork_info()
4802 auto result =
mdb_drop(*txn_ptr, m_hf_starting_heights, 1);
4804 throw1(DB_ERROR(lmdb_error(
"Error dropping hard fork starting heights db: ", result).c_str()));
4805 result =
mdb_drop(*txn_ptr, m_hf_versions, 1);
4807 throw1(DB_ERROR(lmdb_error(
"Error dropping hard fork versions db: ", result).c_str()));
4812 void BlockchainLMDB::set_validator_list(
std::string validator_list,
uint32_t expiration_date) {
4819 v.validators = std::vector<uint8_t>(validator_list.begin(), validator_list.end());
4820 v.expiration_date = expiration_date;
4822 MDB_val_copy<uint64_t> val_key(0);
4823 MDB_val_copy<blobdata> val_value(validator_to_blob(v));
4827 result =
mdb_put(*txn_ptr, m_validators, &val_key, &val_value, 0);
4829 throw1(DB_ERROR(lmdb_error(
"Error adding validator list to db transaction: ", result).c_str()));
4834 std::string BlockchainLMDB::get_validator_list()
const {
4841 MDB_val_copy<uint64_t> val_key(0);
4845 LOG_PRINT_L1(
"Error attempting to retrieve the list of validators from the db.");
4851 ret.assign(reinterpret_cast<const char*>(val_ret.
mv_data), val_ret.
mv_size);
4853 validator_db v = validator_from_blob(ret);
4855 if((v.expiration_date) -
time(
nullptr) <= 0) {
4861 return std::string(v.validators.begin(), v.validators.end());
4871 MDB_val_copy<uint64_t> val_key(
height);
4872 MDB_val_copy<uint8_t> val_value(
version);
4876 result =
mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, 0);
4878 throw1(DB_ERROR(lmdb_error(
"Error adding hard fork version to db transaction: ", result).c_str()));
4891 MDB_val_copy<uint64_t> val_key(
height);
4895 throw0(DB_ERROR(lmdb_error(
"Error attempting to retrieve a hard fork version at height " + boost::lexical_cast<std::string>(
height) +
" from the db: ", result).c_str()));
4902 bool BlockchainLMDB::is_read_only()
const 4907 throw0(DB_ERROR(lmdb_error(
"Error getting database environment info: ", result).c_str()));
4915 uint64_t BlockchainLMDB::get_database_size()
const 4918 boost::filesystem::path datafile(m_folder);
4925 void BlockchainLMDB::fixup()
4932 #define RENAME_DB(name) do { \ 4935 n2[sizeof(n2)-2]--; \ 4937 result = mdb_dbi_open(txn, n2, MDB_CREATE, &tdbi); \ 4939 throw0(DB_ERROR(lmdb_error("Failed to create " + std::string(n2) + ": ", result).c_str())); \ 4940 result = mdb_drop(txn, tdbi, 1); \ 4942 throw0(DB_ERROR(lmdb_error("Failed to delete " + std::string(n2) + ": ", result).c_str())); \ 4943 k.mv_data = (void *)name; \ 4944 k.mv_size = sizeof(name)-1; \ 4945 result = mdb_cursor_open(txn, 1, &c_cur); \ 4947 throw0(DB_ERROR(lmdb_error("Failed to open a cursor for " name ": ", result).c_str())); \ 4948 result = mdb_cursor_get(c_cur, &k, NULL, MDB_SET_KEY); \ 4950 throw0(DB_ERROR(lmdb_error("Failed to get DB record for " name ": ", result).c_str())); \ 4951 ptr = (char *)k.mv_data; \ 4952 ptr[sizeof(name)-2]++; } while(0) 4954 #define LOGIF(y) if (ELPP->vRegistry()->allowed(y, "global")) 4956 void BlockchainLMDB::migrate_0_1()
4961 mdb_txn_safe txn(
false);
4965 MGINFO_YELLOW(
"Migrating blockchain from DB version 0 to 1 - this may take a while:");
4966 MINFO(
"updating blocks, hf_versions, outputs, txs, and spent_keys tables...");
4971 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
4974 if ((result =
mdb_stat(txn, m_blocks, &db_stats)))
4975 throw0(DB_ERROR(lmdb_error(
"Failed to query m_blocks: ", result).c_str()));
4977 MINFO(
"Total number of blocks: " << m_height);
4978 MINFO(
"block migration will update block_heights, block_info, and hf_versions...");
4980 MINFO(
"migrating block_heights:");
4986 throw0(DB_ERROR(lmdb_error(
"Failed to retrieve block_heights flags: ", result).c_str()));
4999 o_heights = m_block_heights;
5016 std::cout << i <<
" / " << z <<
" \r" << std::flush;
5021 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5025 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_heightr: ", result).c_str()));
5028 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_heights: ", result).c_str()));
5031 result =
mdb_stat(txn, m_block_heights, &ms);
5033 throw0(DB_ERROR(lmdb_error(
"Failed to query block_heights table: ", result).c_str()));
5043 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from block_heights: ", result).c_str()));
5048 throw0(DB_ERROR(lmdb_error(
"Failed to put a record into block_heightr: ", result).c_str()));
5055 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from block_heights: ", result).c_str()));
5061 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5063 result =
mdb_drop(txn, o_heights, 1);
5065 throw0(DB_ERROR(lmdb_error(
"Failed to delete old block_heights table: ", result).c_str()));
5086 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5097 lmdb_db_open(txn,
"block_diffs", 0, diffs,
"Failed to open db handle for block_diffs");
5098 lmdb_db_open(txn,
"block_hashes", 0,
hashes,
"Failed to open db handle for block_hashes");
5099 lmdb_db_open(txn,
"block_sizes", 0, sizes,
"Failed to open db handle for block_sizes");
5100 lmdb_db_open(txn,
"block_timestamps", 0, timestamps,
"Failed to open db handle for block_timestamps");
5101 MDB_cursor *c_cur, *c_coins, *c_diffs, *c_hashes, *c_sizes, *c_timestamps;
5109 std::cout << i <<
" / " << z <<
" \r" << std::flush;
5114 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5118 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_info: ", result).c_str()));
5121 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_coins: ", result).c_str()));
5124 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_diffs: ", result).c_str()));
5127 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_hashes: ", result).c_str()));
5130 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_coins: ", result).c_str()));
5133 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_timestamps: ", result).c_str()));
5136 result =
mdb_stat(txn, m_block_info, &ms);
5138 throw0(DB_ERROR(lmdb_error(
"Failed to query block_info table: ", result).c_str()));
5146 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from block_coins: ", result).c_str()));
5151 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from block_diffs: ", result).c_str()));
5155 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from block_hashes: ", result).c_str()));
5159 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from block_sizes: ", result).c_str()));
5166 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from block_timestamps: ", result).c_str()));
5170 throw0(DB_ERROR(lmdb_error(
"Failed to put a record into block_info: ", result).c_str()));
5173 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from block_coins: ", result).c_str()));
5176 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from block_diffs: ", result).c_str()));
5179 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from block_hashes: ", result).c_str()));
5182 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from block_sizes: ", result).c_str()));
5185 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from block_timestamps: ", result).c_str()));
5193 result =
mdb_drop(txn, timestamps, 1);
5195 throw0(DB_ERROR(lmdb_error(
"Failed to delete block_timestamps from the db: ", result).c_str()));
5198 throw0(DB_ERROR(lmdb_error(
"Failed to delete block_sizes from the db: ", result).c_str()));
5201 throw0(DB_ERROR(lmdb_error(
"Failed to delete block_hashes from the db: ", result).c_str()));
5204 throw0(DB_ERROR(lmdb_error(
"Failed to delete block_diffs from the db: ", result).c_str()));
5207 throw0(DB_ERROR(lmdb_error(
"Failed to delete block_coins from the db: ", result).c_str()));
5218 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5221 throw0(DB_ERROR(lmdb_error(
"Failed to retrieve hf_versions flags: ", result).c_str()));
5232 o_hfv = m_hf_versions;
5233 lmdb_db_open(txn,
"hf_versionr",
MDB_INTEGERKEY |
MDB_CREATE, m_hf_versions,
"Failed to open db handle for hf_versionr");
5243 std::cout << i <<
" / " << z <<
" \r" << std::flush;
5248 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5252 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for spent_keyr: ", result).c_str()));
5255 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for spent_keys: ", result).c_str()));
5258 result =
mdb_stat(txn, m_hf_versions, &ms);
5260 throw0(DB_ERROR(lmdb_error(
"Failed to query hf_versions table: ", result).c_str()));
5270 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from hf_versions: ", result).c_str()));
5273 throw0(DB_ERROR(lmdb_error(
"Failed to put a record into hf_versionr: ", result).c_str()));
5276 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from hf_versions: ", result).c_str()));
5282 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5286 throw0(DB_ERROR(lmdb_error(
"Failed to delete old hf_versions table: ", result).c_str()));
5289 lmdb_db_open(txn,
"hf_versions",
MDB_INTEGERKEY, m_hf_versions,
"Failed to open db handle for hf_versions");
5301 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5311 #define DELETE_DB(x) do { \ 5312 LOG_PRINT_L1(" " x ":"); \ 5313 result = mdb_txn_begin(m_env, NULL, 0, txn); \ 5315 throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); \ 5316 result = mdb_dbi_open(txn, x, 0, &dbi); \ 5318 result = mdb_drop(txn, dbi, 1); \ 5320 throw0(DB_ERROR(lmdb_error("Failed to delete " x ": ", result).c_str())); \ 5336 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5339 lmdb_db_open(txn, LMDB_TX_OUTPUTS,
MDB_INTEGERKEY |
MDB_CREATE, m_tx_outputs,
"Failed to open db handle for m_tx_outputs");
5353 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5356 throw0(DB_ERROR(lmdb_error(
"Failed to retrieve txs flags: ", result).c_str()));
5375 MDB_cursor *c_blocks, *c_txs, *c_props, *c_cur;
5382 txn.m_txn = m_write_txn->
m_txn;
5389 std::cout << i <<
" / " << z <<
" \r" << std::flush;
5395 throw0(DB_ERROR(lmdb_error(
"Failed to update txblk property: ", result).c_str()));
5399 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5400 m_write_txn->
m_txn = txn.m_txn;
5401 m_write_batch_txn->
m_txn = txn.m_txn;
5402 memset(&m_wcursors, 0,
sizeof(m_wcursors));
5406 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for blocks: ", result).c_str()));
5409 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for properties: ", result).c_str()));
5412 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs: ", result).c_str()));
5415 result =
mdb_stat(txn, m_txs, &ms);
5417 throw0(DB_ERROR(lmdb_error(
"Failed to query txs table: ", result).c_str()));
5423 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from properties: ", result).c_str()));
5430 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from blocks: ", result).c_str()));
5438 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from props: ", result).c_str()));
5441 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from props: ", result).c_str()));
5445 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from blocks: ", result).c_str()));
5449 throw0(DB_ERROR(
"Failed to parse block from blob retrieved from the db"));
5452 for (
unsigned int j = 0; j<b.tx_hashes.size(); j++) {
5457 throw0(DB_ERROR(lmdb_error(
"Failed to get record from txs: ", result).c_str()));
5460 throw0(DB_ERROR(
"Failed to parse tx from blob retrieved from the db"));
5464 throw0(DB_ERROR(lmdb_error(
"Failed to get record from txs: ", result).c_str()));
5471 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5474 throw0(DB_ERROR(lmdb_error(
"Failed to delete txs from the db: ", result).c_str()));
5480 lmdb_db_open(txn,
"txs",
MDB_INTEGERKEY, m_txs,
"Failed to open db handle for txs");
5491 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5492 result =
mdb_put(txn, m_properties, &vk, &v, 0);
5494 throw0(DB_ERROR(lmdb_error(
"Failed to update version for the db: ", result).c_str()));
5498 void BlockchainLMDB::migrate_1_2()
5503 mdb_txn_safe txn(
false);
5507 MGINFO_YELLOW(
"Migrating blockchain from DB version 1 to 2 - this may take a while:");
5508 MINFO(
"updating txs_pruned and txs_prunable tables...");
5513 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5518 MDB_stat db_stats_txs_prunable_hash;
5519 if ((result =
mdb_stat(txn, m_txs, &db_stats_txs)))
5520 throw0(DB_ERROR(lmdb_error(
"Failed to query m_txs: ", result).c_str()));
5521 if ((result =
mdb_stat(txn, m_txs_pruned, &db_stats_txs_pruned)))
5522 throw0(DB_ERROR(lmdb_error(
"Failed to query m_txs_pruned: ", result).c_str()));
5523 if ((result =
mdb_stat(txn, m_txs_prunable, &db_stats_txs_prunable)))
5524 throw0(DB_ERROR(lmdb_error(
"Failed to query m_txs_prunable: ", result).c_str()));
5525 if ((result =
mdb_stat(txn, m_txs_prunable_hash, &db_stats_txs_prunable_hash)))
5526 throw0(DB_ERROR(lmdb_error(
"Failed to query m_txs_prunable_hash: ", result).c_str()));
5528 throw0(DB_ERROR(
"Mismatched sizes for txs_pruned and txs_prunable"));
5532 MINFO(
"txs already migrated");
5536 MINFO(
"updating txs tables:");
5538 MDB_cursor *c_old, *c_cur0, *c_cur1, *c_cur2;
5544 result =
mdb_stat(txn, m_txs, &db_stats_txs);
5546 throw0(DB_ERROR(lmdb_error(
"Failed to query m_txs: ", result).c_str()));
5548 std::cout << i <<
" / " << (i + db_stats_txs.
ms_entries) <<
" \r" << std::flush;
5553 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5557 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_pruned: ", result).c_str()));
5560 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_prunable: ", result).c_str()));
5563 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs_prunable_hash: ", result).c_str()));
5566 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for txs: ", result).c_str()));
5578 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from txs: ", result).c_str()));
5584 throw0(DB_ERROR(
"Failed to parse tx from blob retrieved from the db"));
5585 std::stringstream ss;
5587 bool r = tx.serialize_base(ba);
5589 throw0(DB_ERROR(
"Failed to serialize pruned tx"));
5592 if (pruned.size() > bd.size())
5593 throw0(DB_ERROR(
"Pruned tx is larger than raw tx"));
5594 if (memcmp(pruned.data(), bd.data(), pruned.size()))
5595 throw0(DB_ERROR(
"Pruned tx is not a prefix of the raw tx"));
5598 nv.
mv_data = (
void*)pruned.data();
5602 throw0(DB_ERROR(lmdb_error(
"Failed to put a record into txs_pruned: ", result).c_str()));
5604 nv.
mv_data = (
void*)(bd.data() + pruned.size());
5605 nv.
mv_size = bd.size() - pruned.size();
5608 throw0(DB_ERROR(lmdb_error(
"Failed to put a record into txs_prunable: ", result).c_str()));
5612 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from txs: ", result).c_str()));
5624 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5625 result =
mdb_put(txn, m_properties, &vk, &v, 0);
5627 throw0(DB_ERROR(lmdb_error(
"Failed to update version for the db: ", result).c_str()));
5631 void BlockchainLMDB::migrate_2_3()
5636 mdb_txn_safe txn(
false);
5640 MGINFO_YELLOW(
"Migrating blockchain from DB version 2 to 3 - this may take a while:");
5647 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5650 if ((result =
mdb_stat(txn, m_blocks, &db_stats)))
5651 throw0(DB_ERROR(lmdb_error(
"Failed to query m_blocks: ", result).c_str()));
5654 MDEBUG(
"enumerating rct outputs...");
5655 std::vector<uint64_t> distribution(blockchain_height, 0);
5657 if (
height >= blockchain_height)
5659 MERROR(
"Output found claiming height >= blockchain height");
5666 throw0(DB_ERROR(
"Failed to build rct output distribution"));
5667 for (
size_t i = 1; i < distribution.size(); ++i)
5668 distribution[i] += distribution[i - 1];
5674 MDB_dbi o_block_info = m_block_info;
5684 std::cout << i <<
" / " << blockchain_height <<
" \r" << std::flush;
5689 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5693 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_infn: ", result).c_str()));
5696 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_info: ", result).c_str()));
5699 result =
mdb_stat(txn, m_block_info, &db_stats);
5701 throw0(DB_ERROR(lmdb_error(
"Failed to query m_block_info: ", result).c_str()));
5711 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from block_info: ", result).c_str()));
5715 bi.bi_timestamp = bi_old->bi_timestamp;
5716 bi.bi_coins = bi_old->bi_coins;
5717 bi.bi_weight = bi_old->bi_weight;
5718 bi.bi_diff = bi_old->bi_diff;
5719 bi.bi_hash = bi_old->bi_hash;
5720 if (bi_old->bi_height >= distribution.size())
5721 throw0(DB_ERROR(
"Bad height in block_info record"));
5722 bi.bi_cum_rct = distribution[bi_old->bi_height];
5726 throw0(DB_ERROR(lmdb_error(
"Failed to put a record into block_infn: ", result).c_str()));
5733 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from block_info: ", result).c_str()));
5739 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5741 result =
mdb_drop(txn, o_block_info, 1);
5743 throw0(DB_ERROR(lmdb_error(
"Failed to delete old block_info table: ", result).c_str()));
5760 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5761 result =
mdb_put(txn, m_properties, &vk, &v, 0);
5763 throw0(DB_ERROR(lmdb_error(
"Failed to update version for the db: ", result).c_str()));
5767 void BlockchainLMDB::migrate_3_4()
5772 mdb_txn_safe txn(
false);
5775 bool past_long_term_weight =
false;
5777 MGINFO_YELLOW(
"Migrating blockchain from DB version 3 to 4 - this may take a while:");
5784 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5787 if ((result =
mdb_stat(txn, m_blocks, &db_stats)))
5788 throw0(DB_ERROR(lmdb_error(
"Failed to query m_blocks: ", result).c_str()));
5797 MDB_dbi o_block_info = m_block_info;
5805 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for blocks: ", result).c_str()));
5813 std::cout << i <<
" / " << blockchain_height <<
" \r" << std::flush;
5818 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5822 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_infn: ", result).c_str()));
5825 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_info: ", result).c_str()));
5828 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for blocks: ", result).c_str()));
5831 result =
mdb_stat(txn, m_block_info, &db_stats);
5833 throw0(DB_ERROR(lmdb_error(
"Failed to query m_block_info: ", result).c_str()));
5843 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from block_info: ", result).c_str()));
5847 bi.bi_timestamp = bi_old->bi_timestamp;
5848 bi.bi_coins = bi_old->bi_coins;
5849 bi.bi_weight = bi_old->bi_weight;
5850 bi.bi_diff = bi_old->bi_diff;
5851 bi.bi_hash = bi_old->bi_hash;
5852 bi.bi_cum_rct = bi_old->bi_cum_rct;
5855 if (!past_long_term_weight)
5857 MDB_val_copy<uint64_t> kb(bi.bi_height);
5861 throw0(DB_ERROR(lmdb_error(
"Failed to query m_blocks: ", result).c_str()));
5863 throw0(DB_ERROR(
"Invalid data from m_blocks"));
5866 past_long_term_weight =
true;
5870 if (past_long_term_weight)
5872 std::vector<uint64_t> weights(long_term_block_weights.begin(), long_term_block_weights.end());
5874 long_term_block_weight = std::min<uint64_t>(bi.bi_weight, long_term_effective_block_median_weight + long_term_effective_block_median_weight * 2 / 5);
5878 long_term_block_weight = bi.bi_weight;
5880 long_term_block_weights.push_back(long_term_block_weight);
5881 bi.bi_long_term_block_weight = long_term_block_weight;
5886 throw0(DB_ERROR(lmdb_error(
"Failed to put a record into block_infn: ", result).c_str()));
5893 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from block_info: ", result).c_str()));
5899 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5901 result =
mdb_drop(txn, o_block_info, 1);
5903 throw0(DB_ERROR(lmdb_error(
"Failed to delete old block_info table: ", result).c_str()));
5920 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5921 result =
mdb_put(txn, m_properties, &vk, &v, 0);
5923 throw0(DB_ERROR(lmdb_error(
"Failed to update version for the db: ", result).c_str()));
5927 void BlockchainLMDB::migrate_4_5()
5932 mdb_txn_safe txn(
false);
5936 MGINFO_YELLOW(
"Migrating blockchain from DB version 4 to 5 - this may take a while:");
5943 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5946 if ((result =
mdb_stat(txn, m_blocks, &db_stats)))
5947 throw0(DB_ERROR(lmdb_error(
"Failed to query m_blocks: ", result).c_str()));
5954 MDB_dbi o_block_info = m_block_info;
5962 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for blocks: ", result).c_str()));
5970 std::cout << i <<
" / " << blockchain_height <<
" \r" << std::flush;
5975 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
5979 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_infn: ", result).c_str()));
5982 throw0(DB_ERROR(lmdb_error(
"Failed to open a cursor for block_info: ", result).c_str()));
5985 result =
mdb_stat(txn, m_block_info, &db_stats);
5987 throw0(DB_ERROR(lmdb_error(
"Failed to query m_block_info: ", result).c_str()));
5997 throw0(DB_ERROR(lmdb_error(
"Failed to get a record from block_info: ", result).c_str()));
6001 bi.bi_timestamp = bi_old->bi_timestamp;
6002 bi.bi_coins = bi_old->bi_coins;
6003 bi.bi_weight = bi_old->bi_weight;
6004 bi.bi_diff_lo = bi_old->bi_diff;
6006 bi.bi_hash = bi_old->bi_hash;
6007 bi.bi_cum_rct = bi_old->bi_cum_rct;
6008 bi.bi_long_term_block_weight = bi_old->bi_long_term_block_weight;
6013 throw0(DB_ERROR(lmdb_error(
"Failed to put a record into block_infn: ", result).c_str()));
6020 throw0(DB_ERROR(lmdb_error(
"Failed to delete a record from block_info: ", result).c_str()));
6026 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
6028 result =
mdb_drop(txn, o_block_info, 1);
6030 throw0(DB_ERROR(lmdb_error(
"Failed to delete old block_info table: ", result).c_str()));
6047 throw0(DB_ERROR(lmdb_error(
"Failed to create a transaction for the db: ", result).c_str()));
6048 result =
mdb_put(txn, m_properties, &vk, &v, 0);
6050 throw0(DB_ERROR(lmdb_error(
"Failed to update version for the db: ", result).c_str()));
6054 void BlockchainLMDB::migrate(
const uint32_t oldversion)
#define TXN_POSTFIX_SUCCESS()
virtual void block_wtxn_start()
virtual void safesyncmode(const bool onoff)
toggle safe syncs for the DB
#define m_cur_block_heights
virtual bool get_tx_blob(const crypto::hash &h, cryptonote::blobdata &tx) const
fetches the transaction blob with the given hash
virtual void set_batch_transactions(bool batch_transactions)
sets whether or not to batch transactions
bool is_coinbase(const transaction &tx)
#define CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE
int lmdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
Set the size of the memory map to use for this environment.
void mdb_cursor_close(MDB_cursor *cursor)
Close a cursor handle.
virtual std::vector< std::string > get_filenames() const
get all files used by the BlockchainDB (if any)
virtual std::vector< address_outputs > get_addr_output_all(const crypto::public_key &combined_key)
struct cryptonote::mdb_block_info_4 mdb_block_info_4
struct cryptonote::acc_addr_tx_t acc_addr_tx_t
int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp)
Return count of duplicates for current key.
virtual uint64_t get_tx_unlock_time(const crypto::hash &h) const
fetch a transaction's unlock time/height
int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi)
Open a database in the environment.
virtual std::vector< transaction > get_tx_list(const std::vector< crypto::hash > &hlist) const
fetches a list of transactions based on their hashes
virtual void block_rtxn_abort() const
#define CRYPTONOTE_BLOCKCHAINDATA_LOCK_FILENAME
virtual uint64_t get_block_long_term_weight(const uint64_t &height) const
fetch a block's long term weight
struct cryptonote::mdb_block_info_2 mdb_block_info_2
static int compare_hash32(const MDB_val *a, const MDB_val *b)
uint64_t relative_out_index
virtual void set_block_cumulative_difficulty(uint64_t height, difficulty_type diff)
sets a block's cumulative difficulty
struct cryptonote::txindex txindex
int mdb_txn_commit(MDB_txn *txn)
Commit all the operations of a transaction into the database.
int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, unsigned int flags)
Store by cursor.
int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat)
Retrieve statistics for a database.
virtual std::string get_db_name() const
gets the name of the folder the BlockchainDB's file(s) should be in
#define TXN_POSTFIX_RDONLY()
void mdb_dbi_close(MDB_env *env, MDB_dbi dbi)
Close a database handle. Normally unnecessary. Use with care:
#define m_cur_hf_versions
virtual uint64_t get_balance(const crypto::public_key &combined_key)
virtual block_header get_block_header(const crypto::hash &h) const
fetch a block header
virtual bool for_all_outputs(std::function< bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const
runs a function over all outputs stored
#define CRYPTONOTE_PRUNING_LOG_STRIPES
pre_rct_output_data_t data
virtual std::vector< address_txs > get_addr_tx_all(const crypto::public_key &combined_key)
virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0)
tells the BlockchainDB to start a new "batch" of blocks
#define ETN_DEFAULT_TX_SPENDABLE_AGE_V8
virtual bool update_pruning()
prunes recent blockchain changes as needed, iff pruning is enabled
epee::critical_section m_synchronization_lock
A lock, currently for when BlockchainLMDB needs to resize the backing db file.
bool is_v1_tx(const blobdata &tx_blob)
int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
Set a custom data comparison function for a MDB_DUPSORT database.
#define MCLOG_RED(level, cat, x)
int mdb_env_info(MDB_env *env, MDB_envinfo *stat)
Return information about the LMDB environment.
thrown when there is an error starting a DB transaction
#define m_cur_txpool_blob
int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs)
Set the maximum number of named databases for the environment.
Non-owning sequence of data. Does not deep copy.
#define CRYPTONOTE_BLOCKCHAINDATA_FILENAME
struct MDB_env MDB_env
Opaque structure for a database environment.
mdb_size_t ms_branch_pages
virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta)
add a txpool transaction
virtual std::vector< std::vector< uint64_t > > get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes) const
gets output indices (amount-specific) for a transaction's outputs
MDB_env * mdb_txn_env(MDB_txn *txn)
Returns the transaction's MDB_env.
static int compare_data(const MDB_val *a, const MDB_val *b)
struct MDB_val MDB_val
Generic structure used for passing keys and data in and out of the database.
#define m_cur_txs_prunable_hash
virtual uint64_t get_block_height(const crypto::hash &h) const
gets the height of the block with a given hash
struct hash_func hashes[]
virtual bool block_rtxn_start() const
virtual bool tx_exists(const crypto::hash &h) const
check if a transaction with a given hash exists
virtual bool get_txpool_tx_meta(const crypto::hash &txid, txpool_tx_meta_t &meta) const
get a txpool transaction's metadata
struct cryptonote::blk_height blk_height
struct cryptonote::mdb_block_info_3 mdb_block_info_3
#define CRYPTONOTE_PRUNING_TIP_BLOCKS
void add_transaction(const crypto::hash &blk_hash, const std::pair< transaction, blobdata > &tx, const crypto::hash *tx_hash_ptr=NULL, const crypto::hash *tx_prunable_hash_ptr=NULL)
helper function for add_transactions, to add each individual transaction
#define TXN_BLOCK_POSTFIX_SUCCESS()
struct MDB_txn MDB_txn
Opaque structure for a transaction handle.
virtual tx_input_t get_tx_input(const crypto::hash tx_hash, const uint32_t relative_out_index)
thrown when a requested transaction does not exist
virtual bool txpool_has_tx(const crypto::hash &txid) const
check whether a txid is in the txpool
constexpr std::size_t size() const noexcept
virtual difficulty_type get_block_difficulty(const uint64_t &height) const
fetch a block's difficulty
Holds cryptonote related classes and helpers.
int mdb_env_get_flags(MDB_env *env, unsigned int *flags)
Get environment flags.
virtual bool get_prunable_tx_hash(const crypto::hash &tx_hash, crypto::hash &prunable_hash) const
fetches the prunable transaction hash
virtual std::vector< block > get_blocks_range(const uint64_t &h1, const uint64_t &h2) const
fetch a list of blocks
int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
Empty or delete+close a database.
#define HF_VERSION_LONG_TERM_BLOCK_WEIGHT
void lmdb_resized(MDB_env *env)
virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t &index) const
gets an output's tx hash and index
blobdata tx_to_blob(const transaction &tx)
#define MDB_val_set(var, val)
int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode)
Open an environment handle.
mdb_size_t count(MDB_cursor *cur)
Statistics for a database in the environment.
#define TXN_BLOCK_PREFIX(flags)
Information about the environment.
uint64_t bi_long_term_block_weight
virtual uint32_t get_blockchain_pruning_seed() const
get the blockchain pruning seed
static std::atomic_flag creation_gate
virtual uint64_t get_block_already_generated_coins(const uint64_t &height) const
fetch a block's already generated coins
bool serialize(Archive &ar, T &v)
#define TXN_PREFIX(flags)
virtual void block_rtxn_stop() const
void commit(std::string message="")
virtual block get_block_from_height(const uint64_t &height) const
fetch a block by height
virtual bool block_exists(const crypto::hash &h, uint64_t *height=NULL) const
checks if a block exists
void mdb_txn_reset(MDB_txn *txn)
Reset a read-only transaction.
virtual bool for_all_key_images(std::function< bool(const crypto::key_image &)>) const
runs a function over all key images stored
virtual bool get_prunable_tx_blob(const crypto::hash &h, cryptonote::blobdata &tx) const
fetches the prunable transaction blob with the given hash
int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags)
Retrieve the DB flags for a database handle.
bool get_block_hash(const block &b, crypto::hash &res)
uint64_t time_tx_exists
a performance metric
static int compare_uint64(const MDB_val *a, const MDB_val *b)
virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta)
update a txpool transaction's metadata
virtual size_t get_block_weight(const uint64_t &height) const
fetch a block's weight
virtual void close()
close the BlockchainDB
#define m_cur_txpool_meta
#define m_cur_output_amounts
int mdb_txn_renew(MDB_txn *txn)
Renew a read-only transaction.
virtual transaction get_tx(const crypto::hash &h) const
fetches the transaction with the given hash
struct cryptonote::acc_outs_t acc_outs_t
virtual bool get_txpool_tx_blob(const crypto::hash &txid, cryptonote::blobdata &bd) const
get a txpool transaction's blob
unsigned __int64 uint64_t
virtual uint64_t add_block(const std::pair< block, blobdata > &blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type &cumulative_difficulty, const uint64_t &coins_generated, const std::vector< std::pair< transaction, blobdata >> &txs)
handles the addition of a new block to BlockchainDB
virtual std::vector< address_txs > get_addr_tx_batch(const crypto::public_key &combined_key, uint64_t start_db_index=0, uint64_t batch_size=100, bool desc=false)
static void wait_no_active_txns()
#define CRITICAL_REGION_LOCAL(x)
bool t_serializable_object_to_blob(const t_object &to, blobdata &b_blob)
virtual void batch_stop()
ends a batch transaction
virtual bool remove_data_file(const std::string &folder) const
remove file(s) storing the database
uint64_t height
the height of the block which created the output
int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
Set a custom key comparison function for a database.
mdb_size_t ms_overflow_pages
virtual uint64_t get_block_timestamp(const uint64_t &height) const
fetch a block's timestamp
struct cryptonote::outtx outtx
thrown when a requested block does not exist
unsigned int MDB_dbi
A handle for an individual database in the DB environment.
int mdb_env_sync(MDB_env *env, int force)
Flush the data buffers to disk.
std::map< uint64_t, std::tuple< uint64_t, uint64_t, uint64_t > > get_output_histogram(const std::vector< uint64_t > &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const
return a histogram of outputs on the blockchain
virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes=true) const
get the number of transactions in the txpool
bool parse_and_validate_tx_base_from_blob(const blobdata &tx_blob, transaction &tx)
bool m_open
Whether or not the BlockchainDB is open/ready for use.
virtual bool lock()
acquires the BlockchainDB lock
bool parse_and_validate_tx_from_blob(const blobdata &tx_blob, transaction &tx)
virtual void unlock()
This function releases the BlockchainDB lock.
int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
Create a transaction for use with the environment.
MDB_cursor_op
Cursor Get operations.
Useful when application has potentially harmful situtaions.
version
Supported socks variants.
rct::key commitment
the output's amount commitment (for spend verification)
int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, MDB_cursor_op op)
Retrieve by cursor.
virtual difficulty_type get_block_cumulative_difficulty(const uint64_t &height) const
fetch a block's cumulative difficulty
std::string message("Message requiring signing")
virtual void open(const std::string &filename, const int mdb_flags=0)
open a db, or create it if necessary.
int lmdb_txn_renew(MDB_txn *txn)
mdb_txn_cursors m_ti_rcursors
int mdb_env_create(MDB_env **env)
Create an LMDB environment handle.
void mdb_txn_abort(MDB_txn *txn)
Abandon all the operations of the transaction instead of saving them.
const GenericPointer< typename T::ValueType > T2 T::AllocatorType & a
type_vec_type median(std::vector< type_vec_type > &v)
mdb_block_info_4 mdb_block_info
virtual bool for_all_txpool_txes(std::function< bool(const crypto::hash &, const txpool_tx_meta_t &, const cryptonote::blobdata *)> f, bool include_blob=false, bool include_unrelayed_txes=true) const
runs a function over all txpool transactions
A generic BlockchainDB exception.
Mainly useful to represent current progress of application.
struct MDB_cursor MDB_cursor
Opaque structure for navigating through a database.
virtual void remove_txpool_tx(const crypto::hash &txid)
remove a txpool transaction
The BlockchainDB backing store interface declaration/contract.
#define m_cur_txs_prunable_tip
Generic structure used for passing keys and data in and out of the database.
uint64_t bi_long_term_block_weight
#define TXN_PREFIX_RDONLY()
static int compare_publickey(const MDB_val *a, const MDB_val *b)
const T & move(const T &t)
struct cryptonote::outkey outkey
boost::multiprecision::uint128_t difficulty_type
void * memcpy(void *a, const void *b, size_t c)
#define CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE
#define m_cur_addr_outputs
virtual bool prune_blockchain(uint32_t pruning_seed=0)
prunes the blockchain
virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t &height) const
fetch a block blob by height
#define MDB_val_sized(var, val)
virtual void batch_abort()
aborts a batch transaction
virtual bool for_all_transactions(std::function< bool(const crypto::hash &, const cryptonote::transaction &)>, bool pruned) const
runs a function over all transactions stored
virtual void fixup()
fix up anything that may be wrong due to past bugs
virtual std::vector< crypto::hash > get_hashes_range(const uint64_t &h1, const uint64_t &h2) const
fetch a list of block hashes
virtual block get_top_block() const
fetch the top block
int mdb_env_stat(MDB_env *env, MDB_stat *stat)
Return statistics about the LMDB environment.
char * mdb_strerror(int err)
Return a string describing a given error code.
int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers)
Set the maximum number of threads/reader slots for the environment.
virtual uint64_t get_top_block_timestamp() const
fetch the top block's timestamp
virtual bool for_blocks_range(const uint64_t &h1, const uint64_t &h2, std::function< bool(uint64_t, const crypto::hash &, const cryptonote::block &)>) const
runs a function over a range of blocks
key zeroCommit(etn_amount amount)
uint64_t relative_out_index
int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff)
Set environment flags.
thrown when a requested output does not exist
int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags)
Delete current key/data pair.
virtual crypto::hash get_block_hash_from_height(const uint64_t &height) const
fetch a block's hash
virtual void batch_commit()
#define MDB_val_str(var, val)
std::pair< crypto::hash, uint64_t > tx_out_index
std::string to_string(t_connection_type type)
uint64_t num_active_tx() const
uint64_t time_commit1
a performance metric
bool get_file_size(const std::string &path_to_file, uint64_t &size)
struct cryptonote::mdb_block_info_1 mdb_block_info_1
thrown when opening the BlockchainDB fails
struct cryptonote::mdb_txn_cursors mdb_txn_cursors
blobdata block_to_blob(const block &b)
static void prevent_new_txns()
int compare_uint64(const MDB_val *a, const MDB_val *b)
virtual void sync()
sync the BlockchainDB with disk
#define m_cur_txs_prunable
virtual cryptonote::blobdata get_block_blob(const crypto::hash &h) const
fetches the block with the given hash
virtual block get_block(const crypto::hash &h) const
fetches the block with the given hash
#define DBF_ADDR_TX_SALVAGE
virtual std::vector< uint64_t > get_long_term_block_weights(uint64_t start_height, size_t count) const
fetch the last N blocks' long term weights
virtual uint64_t get_tx_block_height(const crypto::hash &h) const
fetches the height of a transaction's block
virtual uint64_t get_num_outputs(const uint64_t &amount) const
fetches the number of outputs of a given amount
int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data)
Get items from a database.
virtual std::vector< address_outputs > get_addr_output_batch(const crypto::public_key &combined_key, uint64_t start_db_index=0, uint64_t batch_size=100, bool desc=false)
virtual bool has_key_image(const crypto::key_image &img) const
check if a key image is stored as spent
a struct containing output metadata
struct cryptonote::mdb_threadinfo mdb_threadinfo
virtual output_data_t get_output_key(const uint64_t &amount, const uint64_t &index, bool include_commitmemt) const
get some of an output's data
bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector< uint64_t > &distribution, uint64_t &base) const
virtual void reset()
Remove everything from the BlockchainDB.
virtual void block_wtxn_stop()
int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned int flags)
Store items into a database.
#define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5
void mdb_env_close(MDB_env *env)
Close the environment and release the memory map.
std::atomic< unsigned int > unprunable_size
int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor)
Create a cursor handle.
static int compare_string(const MDB_val *a, const MDB_val *b)
virtual uint64_t get_tx_count() const
fetches the total number of transactions ever
bool parse_and_validate_block_from_blob(const blobdata &b_blob, block &b, crypto::hash *block_hash)
virtual uint64_t height() const
fetch the current blockchain height
virtual std::vector< uint64_t > get_block_weights(uint64_t start_height, size_t count) const
fetch the last N blocks' weights
virtual std::vector< uint64_t > get_block_cumulative_rct_outputs(const std::vector< uint64_t > &heights) const
fetch a block's cumulative number of rct outputs
static std::atomic< uint64_t > num_active_txns
static void allow_new_txns()
virtual bool get_pruned_tx_blob(const crypto::hash &h, cryptonote::blobdata &tx) const
fetches the pruned transaction blob with the given hash
virtual bool check_pruning()
checks pruning was done correctly, iff enabled
virtual void block_wtxn_abort()
struct cryptonote::pre_rct_outkey pre_rct_outkey
virtual tx_out_index get_output_tx_and_index(const uint64_t &amount, const uint64_t &index) const
gets an output's tx hash and index
virtual crypto::hash top_block_hash(uint64_t *block_height=NULL) const
fetch the top block's hash