Electroneum
db_lmdb.cpp
Go to the documentation of this file.
1 // Copyrights(c) 2017-2021, The Electroneum Project
2 // Copyrights(c) 2014-2019, The Monero Project
3 // All rights reserved.
4 //
5 // Redistribution and use in source and binary forms, with or without modification, are
6 // permitted provided that the following conditions are met:
7 //
8 // 1. Redistributions of source code must retain the above copyright notice, this list of
9 // conditions and the following disclaimer.
10 //
11 // 2. Redistributions in binary form must reproduce the above copyright notice, this list
12 // of conditions and the following disclaimer in the documentation and/or other
13 // materials provided with the distribution.
14 //
15 // 3. Neither the name of the copyright holder nor the names of its contributors may be
16 // used to endorse or promote products derived from this software without specific
17 // prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
20 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
21 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
22 // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
27 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 
29 #include "db_lmdb.h"
30 
31 #include <boost/filesystem.hpp>
32 #include <boost/format.hpp>
33 #include <boost/circular_buffer.hpp>
34 #include <boost/archive/text_oarchive.hpp>
35 #include <boost/archive/text_iarchive.hpp>
36 #include <memory> // std::unique_ptr
37 #include <cstring> // memcpy
38 
39 #include "string_tools.h"
40 #include "file_io_utils.h"
41 #include "common/util.h"
42 #include "common/pruning.h"
44 #include "crypto/crypto.h"
45 #include "profile_tools.h"
46 #include "ringct/rctOps.h"
47 
48 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY
49 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "blockchain.db.lmdb"
50 
51 
52 #if defined(__i386) || defined(__x86_64)
53 #define MISALIGNED_OK 1
54 #endif
55 
57 using namespace crypto;
58 
59 // Increase when the DB structure changes
60 #define VERSION 5
61 
62 namespace
63 {
64 
65 #pragma pack(push, 1)
66 // This MUST be identical to output_data_t, without the extra rct data at the end
67 struct pre_rct_output_data_t
68 {
69  crypto::public_key pubkey;
70  uint64_t unlock_time;
72 };
73 #pragma pack(pop)
74 
75 template <typename T>
76 inline void throw0(const T &e)
77 {
78  LOG_PRINT_L0(e.what());
79  throw e;
80 }
81 
82 template <typename T>
83 inline void throw1(const T &e)
84 {
85  LOG_PRINT_L1(e.what());
86  throw e;
87 }
88 
89 #define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val}
90 
91 #define MDB_val_sized(var, val) MDB_val var = {val.size(), (void *)val.data()}
92 
93 #define MDB_val_str(var, val) MDB_val var = {strlen(val) + 1, (void *)val}
94 
95 template<typename T>
96 struct MDB_val_copy: public MDB_val
97 {
98  MDB_val_copy(const T &t) :
99  t_copy(t)
100  {
101  mv_size = sizeof (T);
102  mv_data = &t_copy;
103  }
104 private:
105  T t_copy;
106 };
107 
108 template<>
109 struct MDB_val_copy<cryptonote::blobdata>: public MDB_val
110 {
111  MDB_val_copy(const cryptonote::blobdata &bd) :
112  data(new char[bd.size()])
113  {
114  memcpy(data.get(), bd.data(), bd.size());
115  mv_size = bd.size();
116  mv_data = data.get();
117  }
118 private:
119  std::unique_ptr<char[]> data;
120 };
121 
122 template<>
123 struct MDB_val_copy<const char*>: public MDB_val
124 {
125  MDB_val_copy(const char *s):
126  size(strlen(s)+1), // include the NUL, makes it easier for compares
127  data(new char[size])
128  {
129  mv_size = size;
130  mv_data = data.get();
131  memcpy(mv_data, s, size);
132  }
133 private:
134  size_t size;
135  std::unique_ptr<char[]> data;
136 };
137 
138 }
139 
140 namespace cryptonote
141 {
142 
144 {
145  uint64_t va, vb;
146  memcpy(&va, a->mv_data, sizeof(va));
147  memcpy(&vb, b->mv_data, sizeof(vb));
148  return (va < vb) ? -1 : va > vb;
149 }
150 
151 int BlockchainLMDB::compare_hash32(const MDB_val *a, const MDB_val *b)
152 {
153  uint32_t *va = (uint32_t*) a->mv_data;
154  uint32_t *vb = (uint32_t*) b->mv_data;
155  for (int n = 7; n >= 0; n--)
156  {
157  if (va[n] == vb[n])
158  continue;
159  return va[n] < vb[n] ? -1 : 1;
160  }
161 
162  return 0;
163 }
164 
165 int BlockchainLMDB::compare_string(const MDB_val *a, const MDB_val *b)
166 {
167  const char *va = (const char*) a->mv_data;
168  const char *vb = (const char*) b->mv_data;
169  return strcmp(va, vb);
170 }
171 
172 int BlockchainLMDB::compare_data(const MDB_val *a, const MDB_val *b)
173 {
174  size_t size = std::max(a->mv_size, b->mv_size);
175 
176  uint8_t *va = (uint8_t*) a->mv_data;
177  uint8_t *vb = (uint8_t*) b->mv_data;
178  for (size_t n = 0; n < size; ++n)
179  {
180  if (va[n] == vb[n])
181  continue;
182  return va[n] < vb[n] ? -1 : 1;
183  }
184 
185  return 0;
186 }
187 
188 int BlockchainLMDB::compare_publickey(const MDB_val *a, const MDB_val *b)
189 {
190  uint8_t *va = (uint8_t*) a->mv_data;
191  uint8_t *vb = (uint8_t*) b->mv_data;
192  for (int n = 0; n < 32; ++n)
193  {
194  if (va[n] == vb[n])
195  continue;
196  return va[n] < vb[n] ? -1 : 1;
197  }
198 
199  return 0;
200 }
201 
202 }
203 
204 namespace
205 {
206 
207 /* DB schema:
208  *
209  * Table Key Data
210  * ----- --- ----
211  * blocks block ID block blob
212  * block_heights block hash block height
213  * block_info block ID {block metadata}
214  *
215  * txs_pruned txn ID pruned txn blob
216  * txs_prunable txn ID prunable txn blob
217  * txs_prunable_hash txn ID prunable txn hash
218  * txs_prunable_tip txn ID height
219  * tx_indices txn hash {txn ID, metadata}
220  * tx_outputs txn ID [txn amount output indices]
221  *
222  * output_txs output ID {txn hash, local index}
223  * output_amounts amount [{amount output index, metadata}...]
224  *
225  * spent_keys input hash -
226  *
227  * txpool_meta txn hash txn metadata
228  * txpool_blob txn hash txn blob
229  *
230  * Note: where the data items are of uniform size, DUPFIXED tables have
231  * been used to save space. In most of these cases, a dummy "zerokval"
232  * key is used when accessing the table; the Key listed above will be
233  * attached as a prefix on the Data to serve as the DUPSORT key.
234  * (DUPFIXED saves 8 bytes per record.)
235  *
236  * The output_amounts table doesn't use a dummy key, but uses DUPSORT.
237  */
238 const char* const LMDB_BLOCKS = "blocks";
239 const char* const LMDB_BLOCK_HEIGHTS = "block_heights";
240 const char* const LMDB_BLOCK_INFO = "block_info";
241 
242 const char* const LMDB_TXS = "txs";
243 const char* const LMDB_TXS_PRUNED = "txs_pruned";
244 const char* const LMDB_TXS_PRUNABLE = "txs_prunable";
245 const char* const LMDB_TXS_PRUNABLE_HASH = "txs_prunable_hash";
246 const char* const LMDB_TXS_PRUNABLE_TIP = "txs_prunable_tip";
247 const char* const LMDB_TX_INDICES = "tx_indices";
248 const char* const LMDB_TX_OUTPUTS = "tx_outputs";
249 
250 const char* const LMDB_OUTPUT_TXS = "output_txs";
251 const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts";
252 const char* const LMDB_SPENT_KEYS = "spent_keys";
253 
254 const char* const LMDB_TXPOOL_META = "txpool_meta";
255 const char* const LMDB_TXPOOL_BLOB = "txpool_blob";
256 
257 const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights";
258 const char* const LMDB_HF_VERSIONS = "hf_versions";
259 const char* const LMDB_VALIDATORS = "validators";
260 const char* const LMDB_PROPERTIES = "properties";
261 const char* const LMDB_UTXOS = "unspent_txos";
262 const char* const LMDB_ADDR_OUTPUTS = "unspent_addr_outputs";
263 const char* const LMDB_ADDR_TXS = "addr_tx";
264 const char* const LMDB_ADDR_TXS_OLD = "addr_tx_map";
265 const char* const LMDB_TX_INPUTS = "tx_inputs";
266 
267 const char zerokey[8] = {0};
268 const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey };
269 
270 const std::string lmdb_error(const std::string& error_string, int mdb_res)
271 {
272  const std::string full_string = error_string + mdb_strerror(mdb_res);
273  return full_string;
274 }
275 
276 inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string)
277 {
278  if (auto res = mdb_dbi_open(txn, name, flags, &dbi))
279  throw0(cryptonote::DB_OPEN_FAILURE((lmdb_error(error_string + " : ", res) + std::string(" - you may want to start with --db-salvage")).c_str()));
280 }
281 
282 
283 } // anonymous namespace
284 
285 #define CURSOR(name) \
286  if (!m_cur_ ## name) { \
287  int result = mdb_cursor_open(*m_write_txn, m_ ## name, &m_cur_ ## name); \
288  if (result) \
289  throw0(DB_ERROR(lmdb_error("Failed to open cursor: ", result).c_str())); \
290  }
291 
292 #define RCURSOR(name) \
293  if (!m_cur_ ## name) { \
294  int result = mdb_cursor_open(m_txn, m_ ## name, (MDB_cursor **)&m_cur_ ## name); \
295  if (result) \
296  throw0(DB_ERROR(lmdb_error("Failed to open cursor: ", result).c_str())); \
297  if (m_cursors != &m_wcursors) \
298  m_tinfo->m_ti_rflags.m_rf_ ## name = true; \
299  } else if (m_cursors != &m_wcursors && !m_tinfo->m_ti_rflags.m_rf_ ## name) { \
300  int result = mdb_cursor_renew(m_txn, m_cur_ ## name); \
301  if (result) \
302  throw0(DB_ERROR(lmdb_error("Failed to renew cursor: ", result).c_str())); \
303  m_tinfo->m_ti_rflags.m_rf_ ## name = true; \
304  }
305 
306 namespace cryptonote
307 {
308 
309 typedef struct mdb_block_info_1
310 {
314  uint64_t bi_weight; // a size_t really but we need 32-bit compat
318 
319 typedef struct mdb_block_info_2
320 {
324  uint64_t bi_weight; // a size_t really but we need 32-bit compat
329 
330 typedef struct mdb_block_info_3
331 {
335  uint64_t bi_weight; // a size_t really but we need 32-bit compat
341 
342 typedef struct mdb_block_info_4
343 {
347  uint64_t bi_weight; // a size_t really but we need 32-bit compat
354 
356 
357 typedef struct blk_height {
360 } blk_height;
361 
362 typedef struct pre_rct_outkey {
365  pre_rct_output_data_t data;
367 
368 typedef struct outkey {
372 } outkey;
373 
374 typedef struct outtx {
378 } outtx;
379 
380 typedef struct acc_outs_t {
386 }acc_outs_t;
387 
388 typedef struct acc_addr_tx_t {
392 
393 std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0};
394 std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT;
395 
396 mdb_threadinfo::~mdb_threadinfo()
397 {
398  MDB_cursor **cur = &m_ti_rcursors.m_txc_blocks;
399  unsigned i;
400  for (i=0; i<sizeof(mdb_txn_cursors)/sizeof(MDB_cursor *); i++)
401  if (cur[i])
402  mdb_cursor_close(cur[i]);
403  if (m_ti_rtxn)
404  mdb_txn_abort(m_ti_rtxn);
405 }
406 
407 mdb_txn_safe::mdb_txn_safe(const bool check) : m_txn(NULL), m_tinfo(NULL), m_check(check)
408 {
409  if (check)
410  {
411  while (creation_gate.test_and_set());
412  num_active_txns++;
413  creation_gate.clear();
414  }
415 }
416 
418 {
419  if (!m_check)
420  return;
421  LOG_PRINT_L3("mdb_txn_safe: destructor");
422  if (m_tinfo != nullptr)
423  {
425  memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
426  } else if (m_txn != nullptr)
427  {
428  if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
429  {
430  LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
431  }
432  else
433  {
434  // Example of when this occurs: a lookup fails, so a read-only txn is
435  // aborted through this destructor. However, successful read-only txns
436  // ideally should have been committed when done and not end up here.
437  //
438  // NOTE: not sure if this is ever reached for a non-batch write
439  // transaction, but it's probably not ideal if it did.
440  LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
441  }
443  }
444  num_active_txns--;
445 }
446 
448 {
449  num_active_txns--;
450  m_check = false;
451 }
452 
454 {
455  if (message.size() == 0)
456  {
457  message = "Failed to commit a transaction to the db";
458  }
459 
460  if (auto result = mdb_txn_commit(m_txn))
461  {
462  m_txn = nullptr;
463  throw0(DB_ERROR(lmdb_error(message + ": ", result).c_str()));
464  }
465  m_txn = nullptr;
466 }
467 
469 {
470  LOG_PRINT_L3("mdb_txn_safe: abort()");
471  if(m_txn != nullptr)
472  {
474  m_txn = nullptr;
475  }
476  else
477  {
478  LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL");
479  }
480 }
481 
483 {
484  return num_active_txns;
485 }
486 
488 {
489  while (creation_gate.test_and_set());
490 }
491 
493 {
494  while (num_active_txns > 0);
495 }
496 
498 {
499  creation_gate.clear();
500 }
501 
503 {
505 
506  MGINFO("LMDB map resize detected.");
507 
508  MDB_envinfo mei;
509 
510  mdb_env_info(env, &mei);
511  uint64_t old = mei.me_mapsize;
512 
514 
515  int result = mdb_env_set_mapsize(env, 0);
516  if (result)
517  throw0(DB_ERROR(lmdb_error("Failed to set new mapsize: ", result).c_str()));
518 
519  mdb_env_info(env, &mei);
520  uint64_t new_mapsize = mei.me_mapsize;
521 
522  MGINFO("LMDB Mapsize increased." << " Old: " << old / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
523 
525 }
526 
527 inline int lmdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
528 {
529  int res = mdb_txn_begin(env, parent, flags, txn);
530  if (res == MDB_MAP_RESIZED) {
531  lmdb_resized(env);
532  res = mdb_txn_begin(env, parent, flags, txn);
533  }
534  return res;
535 }
536 
537 inline int lmdb_txn_renew(MDB_txn *txn)
538 {
539  int res = mdb_txn_renew(txn);
540  if (res == MDB_MAP_RESIZED) {
542  res = mdb_txn_renew(txn);
543  }
544  return res;
545 }
546 
547 inline void BlockchainLMDB::check_open() const
548 {
549  if (!m_open)
550  throw0(DB_ERROR("DB operation attempted on a not-open DB instance"));
551 }
552 
553 void BlockchainLMDB::do_resize(uint64_t increase_size)
554 {
555  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
557  const uint64_t add_size = 1LL << 30;
558 
559  // check disk capacity
560  try
561  {
562  boost::filesystem::path path(m_folder);
563  boost::filesystem::space_info si = boost::filesystem::space(path);
564  if(si.available < add_size)
565  {
566  MERROR("!! WARNING: Insufficient free space to extend database !!: " <<
567  (si.available >> 20L) << " MB available, " << (add_size >> 20L) << " MB needed");
568  return;
569  }
570  }
571  catch(...)
572  {
573  // print something but proceed.
574  MWARNING("Unable to query free disk space.");
575  }
576 
577  MDB_envinfo mei;
578 
579  mdb_env_info(m_env, &mei);
580 
581  MDB_stat mst;
582 
583  mdb_env_stat(m_env, &mst);
584 
585  // add 1Gb per resize, instead of doing a percentage increase
586  uint64_t new_mapsize = (uint64_t) mei.me_mapsize + add_size;
587 
588  // If given, use increase_size instead of above way of resizing.
589  // This is currently used for increasing by an estimated size at start of new
590  // batch txn.
591  if (increase_size > 0)
592  new_mapsize = mei.me_mapsize + increase_size;
593 
594  new_mapsize += (new_mapsize % mst.ms_psize);
595 
597 
598  if (m_write_txn != nullptr)
599  {
600  if (m_batch_active)
601  {
602  throw0(DB_ERROR("lmdb resizing not yet supported when batch transactions enabled!"));
603  }
604  else
605  {
606  throw0(DB_ERROR("attempting resize with write transaction in progress, this should not happen!"));
607  }
608  }
609 
611 
612  int result = mdb_env_set_mapsize(m_env, new_mapsize);
613  if (result)
614  throw0(DB_ERROR(lmdb_error("Failed to set new mapsize: ", result).c_str()));
615 
616  MGINFO("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
617 
619 }
620 
621 // threshold_size is used for batch transactions
622 bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
623 {
624  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
625 #if defined(ENABLE_AUTO_RESIZE)
626  MDB_envinfo mei;
627 
628  mdb_env_info(m_env, &mei);
629 
630  MDB_stat mst;
631 
632  mdb_env_stat(m_env, &mst);
633 
634  // size_used doesn't include data yet to be committed, which can be
635  // significant size during batch transactions. For that, we estimate the size
636  // needed at the beginning of the batch transaction and pass in the
637  // additional size needed.
638  uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
639 
640  MDEBUG("DB map size: " << mei.me_mapsize);
641  MDEBUG("Space used: " << size_used);
642  MDEBUG("Space remaining: " << mei.me_mapsize - size_used);
643  MDEBUG("Size threshold: " << threshold_size);
644  float resize_percent = RESIZE_PERCENT;
645  MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent);
646 
647  if (threshold_size > 0)
648  {
649  if (mei.me_mapsize - size_used < threshold_size)
650  {
651  MINFO("Threshold met (size-based)");
652  return true;
653  }
654  else
655  return false;
656  }
657 
658  if ((double)size_used / mei.me_mapsize > resize_percent)
659  {
660  MINFO("Threshold met (percent-based)");
661  return true;
662  }
663  return false;
664 #else
665  return false;
666 #endif
667 }
668 
669 void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks, uint64_t batch_bytes)
670 {
671  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
672  MTRACE("[" << __func__ << "] " << "checking DB size");
673  const uint64_t min_increase_size = 512 * (1 << 20);
674  uint64_t threshold_size = 0;
675  uint64_t increase_size = 0;
676  if (batch_num_blocks > 0)
677  {
678  threshold_size = get_estimated_batch_size(batch_num_blocks, batch_bytes);
679  MDEBUG("calculated batch size: " << threshold_size);
680 
681  // The increased DB size could be a multiple of threshold_size, a fixed
682  // size increase (> threshold_size), or other variations.
683  //
684  // Currently we use the greater of threshold size and a minimum size. The
685  // minimum size increase is used to avoid frequent resizes when the batch
686  // size is set to a very small numbers of blocks.
687  increase_size = (threshold_size > min_increase_size) ? threshold_size : min_increase_size;
688  MDEBUG("increase size: " << increase_size);
689  }
690 
691  // if threshold_size is 0 (i.e. number of blocks for batch not passed in), it
692  // will fall back to the percent-based threshold check instead of the
693  // size-based check
694  if (need_resize(threshold_size))
695  {
696  MGINFO("[batch] DB resize needed");
697  do_resize(increase_size);
698  }
699 }
700 
701 uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uint64_t batch_bytes) const
702 {
703  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
704  uint64_t threshold_size = 0;
705 
706  // batch size estimate * batch safety factor = final size estimate
707  // Takes into account "reasonable" block size increases in batch.
708  float batch_safety_factor = 1.7f;
709  float batch_fudge_factor = batch_safety_factor * batch_num_blocks;
710  // estimate of stored block expanded from raw block, including denormalization and db overhead.
711  // Note that this probably doesn't grow linearly with block size.
712  float db_expand_factor = 4.5f;
713  uint64_t num_prev_blocks = 500;
714  // For resizing purposes, allow for at least 4k average block size.
715  uint64_t min_block_size = 4 * 1024;
716 
717  uint64_t block_stop = 0;
718  uint64_t m_height = height();
719  if (m_height > 1)
720  block_stop = m_height - 1;
721  uint64_t block_start = 0;
722  if (block_stop >= num_prev_blocks)
723  block_start = block_stop - num_prev_blocks + 1;
724  uint32_t num_blocks_used = 0;
725  uint64_t total_block_size = 0;
726  MDEBUG("[" << __func__ << "] " << "m_height: " << m_height << " block_start: " << block_start << " block_stop: " << block_stop);
727  size_t avg_block_size = 0;
728  if (batch_bytes)
729  {
730  avg_block_size = batch_bytes / batch_num_blocks;
731  goto estim;
732  }
733  if (m_height == 0)
734  {
735  MDEBUG("No existing blocks to check for average block size");
736  }
737  else if (m_cum_count >= num_prev_blocks)
738  {
739  avg_block_size = m_cum_size / m_cum_count;
740  MDEBUG("average block size across recent " << m_cum_count << " blocks: " << avg_block_size);
741  m_cum_size = 0;
742  m_cum_count = 0;
743  }
744  else
745  {
746  MDB_txn *rtxn;
747  mdb_txn_cursors *rcurs;
748  bool my_rtxn = block_rtxn_start(&rtxn, &rcurs);
749  for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
750  {
751  // we have access to block weight, which will be greater or equal to block size,
752  // so use this as a proxy. If it's too much off, we might have to check actual size,
753  // which involves reading more data, so is not really wanted
754  size_t block_weight = get_block_weight(block_num);
755  total_block_size += block_weight;
756  // Track number of blocks being totalled here instead of assuming, in case
757  // some blocks were to be skipped for being outliers.
758  ++num_blocks_used;
759  }
760  if (my_rtxn) block_rtxn_stop();
761  avg_block_size = total_block_size / num_blocks_used;
762  MDEBUG("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size);
763  }
764 estim:
765  if (avg_block_size < min_block_size)
766  avg_block_size = min_block_size;
767  MDEBUG("estimated average block size for batch: " << avg_block_size);
768 
769  // bigger safety margin on smaller block sizes
770  if (batch_fudge_factor < 5000.0)
771  batch_fudge_factor = 5000.0;
772  threshold_size = avg_block_size * db_expand_factor * batch_fudge_factor;
773  return threshold_size;
774 }
775 
776 void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
777  uint64_t num_rct_outs, const crypto::hash& blk_hash)
778 {
779  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
780  check_open();
781  mdb_txn_cursors *m_cursors = &m_wcursors;
782  uint64_t m_height = height();
783 
784  CURSOR(block_heights)
785  blk_height bh = {blk_hash, m_height};
786  MDB_val_set(val_h, bh);
787  if (mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH) == 0)
788  throw1(BLOCK_EXISTS("Attempting to add block that's already in the db"));
789 
790  if (m_height > 0)
791  {
792  MDB_val_set(parent_key, blk.prev_id);
793  int result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &parent_key, MDB_GET_BOTH);
794  if (result)
795  {
796  LOG_PRINT_L3("m_height: " << m_height);
797  LOG_PRINT_L3("parent_key: " << blk.prev_id);
798  throw0(DB_ERROR(lmdb_error("Failed to get top block hash to check for new block's parent: ", result).c_str()));
799  }
800  blk_height *prev = (blk_height *)parent_key.mv_data;
801  if (prev->bh_height != m_height - 1)
802  throw0(BLOCK_PARENT_DNE("Top block is not new block's parent"));
803  }
804 
805  int result = 0;
806 
807  MDB_val_set(key, m_height);
808 
809  CURSOR(blocks)
810  CURSOR(block_info)
811 
812  // this call to mdb_cursor_put will change height()
813  cryptonote::blobdata block_blob(block_to_blob(blk));
814  MDB_val_sized(blob, block_blob);
815  result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND);
816  if (result)
817  throw0(DB_ERROR(lmdb_error("Failed to add block blob to db transaction: ", result).c_str()));
818 
819  mdb_block_info bi;
820  bi.bi_height = m_height;
821  bi.bi_timestamp = blk.timestamp;
822  bi.bi_coins = coins_generated;
823  bi.bi_weight = block_weight;
824  bi.bi_diff_hi = ((cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to<uint64_t>();
825  bi.bi_diff_lo = (cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>();
826  bi.bi_hash = blk_hash;
827  bi.bi_cum_rct = num_rct_outs;
828  if (blk.major_version >= 4)
829  {
830  uint64_t last_height = m_height-1;
831  MDB_val_set(h, last_height);
832  if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH)))
833  throw1(BLOCK_DNE(lmdb_error("Failed to get block info: ", result).c_str()));
834  const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data;
835  bi.bi_cum_rct += bi_prev->bi_cum_rct;
836  }
837  bi.bi_long_term_block_weight = long_term_block_weight;
838 
839  MDB_val_set(val, bi);
840  result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP);
841  if (result)
842  throw0(DB_ERROR(lmdb_error("Failed to add block info to db transaction: ", result).c_str()));
843 
844  result = mdb_cursor_put(m_cur_block_heights, (MDB_val *)&zerokval, &val_h, 0);
845  if (result)
846  throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str()));
847 
848  // we use weight as a proxy for size, since we don't have size but weight is >= size
849  // and often actually equal
850  m_cum_size += block_weight;
851  m_cum_count++;
852 }
853 
854 void BlockchainLMDB::remove_block()
855 {
856  int result;
857 
858  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
859  check_open();
860  uint64_t m_height = height();
861 
862  if (m_height == 0)
863  throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain"));
864 
865  mdb_txn_cursors *m_cursors = &m_wcursors;
866  CURSOR(block_info)
867  CURSOR(block_heights)
868  CURSOR(blocks)
869  MDB_val_copy<uint64_t> k(m_height - 1);
870  MDB_val h = k;
871  if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH)))
872  throw1(BLOCK_DNE(lmdb_error("Attempting to remove block that's not in the db: ", result).c_str()));
873 
874  // must use h now; deleting from m_block_info will invalidate it
876  blk_height bh = {bi->bi_hash, 0};
877  h.mv_data = (void *)&bh;
878  h.mv_size = sizeof(bh);
879  if ((result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &h, MDB_GET_BOTH)))
880  throw1(DB_ERROR(lmdb_error("Failed to locate block height by hash for removal: ", result).c_str()));
881  if ((result = mdb_cursor_del(m_cur_block_heights, 0)))
882  throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str()));
883 
884  if ((result = mdb_cursor_del(m_cur_blocks, 0)))
885  throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str()));
886 
887  if ((result = mdb_cursor_del(m_cur_block_info, 0)))
888  throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str()));
889 }
890 
891 uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash)
892 {
893  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
894  check_open();
895  mdb_txn_cursors *m_cursors = &m_wcursors;
896  uint64_t m_height = height();
897 
898  int result;
899  uint64_t tx_id = get_tx_count();
900 
901  CURSOR(txs_pruned)
902  CURSOR(txs_prunable)
903  CURSOR(txs_prunable_hash)
904  CURSOR(txs_prunable_tip)
905  CURSOR(tx_indices)
906 
907  MDB_val_set(val_tx_id, tx_id);
908  MDB_val_set(val_h, tx_hash);
909  result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH);
910  if (result == 0) {
911  txindex *tip = (txindex *)val_h.mv_data;
912  throw1(TX_EXISTS(std::string("Attempting to add transaction that's already in the db (tx id ").append(boost::lexical_cast<std::string>(tip->data.tx_id)).append(")").c_str()));
913  } else if (result != MDB_NOTFOUND) {
914  throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str()));
915  }
916 
917  const cryptonote::transaction &tx = txp.first;
918  txindex ti;
919  ti.key = tx_hash;
920  ti.data.tx_id = tx_id;
921  ti.data.unlock_time = tx.unlock_time;
922  ti.data.block_id = m_height; // we don't need blk_hash since we know m_height
923 
924  val_h.mv_size = sizeof(ti);
925  val_h.mv_data = (void *)&ti;
926 
927  result = mdb_cursor_put(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, 0);
928  if (result)
929  throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str()));
930 
931  const cryptonote::blobdata &blob = txp.second;
932  MDB_val_sized(blobval, blob);
933 
934  unsigned int unprunable_size = tx.unprunable_size;
935  if (unprunable_size == 0)
936  {
937  std::stringstream ss;
938  binary_archive<true> ba(ss);
939  bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba);
940  if (!r)
941  throw0(DB_ERROR("Failed to serialize pruned tx"));
942  unprunable_size = ss.str().size();
943  }
944 
945  if (unprunable_size > blob.size())
946  throw0(DB_ERROR("pruned tx size is larger than tx size"));
947 
948  MDB_val pruned_blob = {unprunable_size, (void*)blob.data()};
949  result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND);
950  if (result)
951  throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str()));
952 
953  MDB_val prunable_blob = {blob.size() - unprunable_size, (void*)(blob.data() + unprunable_size)};
954  result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND);
955  if (result)
956  throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str()));
957 
958  if (get_blockchain_pruning_seed())
959  {
960  MDB_val_set(val_height, m_height);
961  result = mdb_cursor_put(m_cur_txs_prunable_tip, &val_tx_id, &val_height, 0);
962  if (result)
963  throw0(DB_ERROR(lmdb_error("Failed to add prunable tx id to db transaction: ", result).c_str()));
964  }
965 
966  return tx_id;
967 }
968 
969 // TODO: compare pros and cons of looking up the tx hash's tx index once and
970 // passing it in to functions like this
971 void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx)
972 {
973  int result;
974 
975  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
976  check_open();
977 
978  mdb_txn_cursors *m_cursors = &m_wcursors;
979  CURSOR(tx_indices)
980  CURSOR(txs_pruned)
981  CURSOR(txs_prunable)
982  CURSOR(txs_prunable_hash)
983  CURSOR(txs_prunable_tip)
984  CURSOR(tx_outputs)
985 
986  MDB_val_set(val_h, tx_hash);
987 
988  if (mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH))
989  throw1(TX_DNE("Attempting to remove transaction that isn't in the db"));
990  txindex *tip = (txindex *)val_h.mv_data;
991  MDB_val_set(val_tx_id, tip->data.tx_id);
992 
993  if ((result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, NULL, MDB_SET)))
994  throw1(DB_ERROR(lmdb_error("Failed to locate pruned tx for removal: ", result).c_str()));
995  result = mdb_cursor_del(m_cur_txs_pruned, 0);
996  if (result)
997  throw1(DB_ERROR(lmdb_error("Failed to add removal of pruned tx to db transaction: ", result).c_str()));
998 
999  result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET);
1000  if (result == 0)
1001  {
1002  result = mdb_cursor_del(m_cur_txs_prunable, 0);
1003  if (result)
1004  throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str()));
1005  }
1006  else if (result != MDB_NOTFOUND)
1007  throw1(DB_ERROR(lmdb_error("Failed to locate prunable tx for removal: ", result).c_str()));
1008 
1009  result = mdb_cursor_get(m_cur_txs_prunable_tip, &val_tx_id, NULL, MDB_SET);
1010  if (result && result != MDB_NOTFOUND)
1011  throw1(DB_ERROR(lmdb_error("Failed to locate tx id for removal: ", result).c_str()));
1012  if (result == 0)
1013  {
1015  if (result)
1016  throw1(DB_ERROR(lmdb_error("Error adding removal of tx id to db transaction", result).c_str()));
1017  }
1018 
1019  if (tx.version == 1) {
1020  remove_tx_outputs(tip->data.tx_id, tx);
1021  result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_id, NULL, MDB_SET);
1022  if (result == MDB_NOTFOUND)
1023  LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash);
1024  else if (result)
1025  throw1(DB_ERROR(lmdb_error("Failed to locate tx outputs for removal: ", result).c_str()));
1026  if (!result) {
1027  result = mdb_cursor_del(m_cur_tx_outputs, 0);
1028  if (result)
1029  throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str()));
1030  }
1031  }
1032 
1033  // Don't delete the tx_indices entry until the end, after we're done with val_tx_id
1035  throw1(DB_ERROR("Failed to add removal of tx index to db transaction"));
1036 }
1037 
1038 uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash,
1039  const tx_out& tx_output,
1040  const uint64_t& local_index,
1041  const uint64_t unlock_time,
1042  const rct::key *commitment)
1043 {
1044  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1045  check_open();
1046  mdb_txn_cursors *m_cursors = &m_wcursors;
1047  uint64_t m_height = height();
1048  uint64_t m_num_outputs = num_outputs();
1049 
1050  int result = 0;
1051 
1052  CURSOR(output_txs)
1053  CURSOR(output_amounts)
1054 
1055  if (tx_output.target.type() != typeid(txout_to_key))
1056  throw0(DB_ERROR("Wrong output type: expected txout_to_key"));
1057  if (tx_output.amount == 0 && !commitment)
1058  throw0(DB_ERROR("RCT output without commitment"));
1059 
1060  outtx ot = {m_num_outputs, tx_hash, local_index};
1061  MDB_val_set(vot, ot);
1062 
1063  result = mdb_cursor_put(m_cur_output_txs, (MDB_val *)&zerokval, &vot, MDB_APPENDDUP);
1064  if (result)
1065  throw0(DB_ERROR(lmdb_error("Failed to add output tx hash to db transaction: ", result).c_str()));
1066 
1067  outkey ok;
1068  MDB_val data;
1069  MDB_val_copy<uint64_t> val_amount(tx_output.amount);
1070  result = mdb_cursor_get(m_cur_output_amounts, &val_amount, &data, MDB_SET);
1071  if (!result)
1072  {
1073  mdb_size_t num_elems = 0;
1074  result = mdb_cursor_count(m_cur_output_amounts, &num_elems);
1075  if (result)
1076  throw0(DB_ERROR(std::string("Failed to get number of outputs for amount: ").append(mdb_strerror(result)).c_str()));
1077  ok.amount_index = num_elems;
1078  }
1079  else if (result != MDB_NOTFOUND)
1080  throw0(DB_ERROR(lmdb_error("Failed to get output amount in db transaction: ", result).c_str()));
1081  else
1082  ok.amount_index = 0;
1083  ok.output_id = m_num_outputs;
1084  ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).key;
1085  ok.data.unlock_time = unlock_time;
1086  ok.data.height = m_height;
1087  if (tx_output.amount == 0)
1088  {
1089  ok.data.commitment = *commitment;
1090  data.mv_size = sizeof(ok);
1091  }
1092  else
1093  {
1094  data.mv_size = sizeof(pre_rct_outkey);
1095  }
1096  data.mv_data = &ok;
1097 
1098  if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP)))
1099  throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str()));
1100 
1101  return ok.amount_index;
1102 }
1103 
1104 void BlockchainLMDB::add_tx_amount_output_indices(const uint64_t tx_id,
1105  const std::vector<uint64_t>& amount_output_indices)
1106 {
1107  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1108  check_open();
1109  mdb_txn_cursors *m_cursors = &m_wcursors;
1110  CURSOR(tx_outputs)
1111 
1112  int result = 0;
1113 
1114  size_t num_outputs = amount_output_indices.size();
1115 
1116  MDB_val_set(k_tx_id, tx_id);
1117  MDB_val v;
1118  v.mv_data = num_outputs ? (void *)amount_output_indices.data() : (void*)"";
1119  v.mv_size = sizeof(uint64_t) * num_outputs;
1120  // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size);
1121 
1122  result = mdb_cursor_put(m_cur_tx_outputs, &k_tx_id, &v, MDB_APPEND);
1123  if (result)
1124  throw0(DB_ERROR(std::string("Failed to add <tx hash, amount output index array> to db transaction: ").append(mdb_strerror(result)).c_str()));
1125 }
1126 
1127 void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction& tx)
1128 {
1129  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1130 
1131  std::vector<std::vector<uint64_t>> amount_output_indices_set = get_tx_amount_output_indices(tx_id, 1);
1132  const std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.front();
1133 
1134  if (amount_output_indices.empty())
1135  {
1136  if (tx.vout.empty())
1137  LOG_PRINT_L2("tx has no outputs, so no output indices");
1138  else
1139  throw0(DB_ERROR("tx has outputs, but no output indices found"));
1140  }
1141 
1142  for (size_t i = tx.vout.size(); i-- > 0;)
1143  {
1144  remove_output(tx.vout[i].amount, amount_output_indices[i]);
1145  }
1146 }
1147 
1148 void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_index)
1149 {
1150  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1151  check_open();
1152  mdb_txn_cursors *m_cursors = &m_wcursors;
1153  CURSOR(output_amounts);
1154  CURSOR(output_txs);
1155 
1156  MDB_val_set(k, amount);
1157  MDB_val_set(v, out_index);
1158 
1159  auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH);
1160  if (result == MDB_NOTFOUND)
1161  throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
1162  else if (result)
1163  throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str()));
1164 
1165  const pre_rct_outkey *ok = (const pre_rct_outkey *)v.mv_data;
1166  MDB_val_set(otxk, ok->output_id);
1167  result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &otxk, MDB_GET_BOTH);
1168  if (result == MDB_NOTFOUND)
1169  {
1170  throw0(DB_ERROR("Unexpected: global output index not found in m_output_txs"));
1171  }
1172  else if (result)
1173  {
1174  throw1(DB_ERROR(lmdb_error("Error adding removal of output tx to db transaction", result).c_str()));
1175  }
1176  result = mdb_cursor_del(m_cur_output_txs, 0);
1177  if (result)
1178  throw0(DB_ERROR(lmdb_error(std::string("Error deleting output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str()));
1179 
1180  // now delete the amount
1181  result = mdb_cursor_del(m_cur_output_amounts, 0);
1182  if (result)
1183  throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str()));
1184 }
1185 
1186 void BlockchainLMDB::prune_outputs(uint64_t amount)
1187 {
1188  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1189  check_open();
1190  mdb_txn_cursors *m_cursors = &m_wcursors;
1191  CURSOR(output_amounts);
1192  CURSOR(output_txs);
1193 
1194  MINFO("Pruning outputs for amount " << amount);
1195 
1196  MDB_val v;
1197  MDB_val_set(k, amount);
1198  int result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET);
1199  if (result == MDB_NOTFOUND)
1200  return;
1201  if (result)
1202  throw0(DB_ERROR(lmdb_error("Error looking up outputs: ", result).c_str()));
1203 
1204  // gather output ids
1205  mdb_size_t num_elems;
1207  MINFO(num_elems << " outputs found");
1208  std::vector<uint64_t> output_ids;
1209  output_ids.reserve(num_elems);
1210  while (1)
1211  {
1212  const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data;
1213  output_ids.push_back(okp->output_id);
1214  MDEBUG("output id " << okp->output_id);
1216  if (result == MDB_NOTFOUND)
1217  break;
1218  if (result)
1219  throw0(DB_ERROR(lmdb_error("Error counting outputs: ", result).c_str()));
1220  }
1221  if (output_ids.size() != num_elems)
1222  throw0(DB_ERROR("Unexpected number of outputs"));
1223 
1225  if (result)
1226  throw0(DB_ERROR(lmdb_error("Error deleting outputs: ", result).c_str()));
1227 
1228  for (uint64_t output_id: output_ids)
1229  {
1230  MDB_val_set(v, output_id);
1231  result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
1232  if (result)
1233  throw0(DB_ERROR(lmdb_error("Error looking up output: ", result).c_str()));
1234  result = mdb_cursor_del(m_cur_output_txs, 0);
1235  if (result)
1236  throw0(DB_ERROR(lmdb_error("Error deleting output: ", result).c_str()));
1237  }
1238 }
1239 
1240 void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image)
1241 {
1242  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1243  check_open();
1244  mdb_txn_cursors *m_cursors = &m_wcursors;
1245 
1246  CURSOR(spent_keys)
1247 
1248  MDB_val k = {sizeof(k_image), (void *)&k_image};
1249  if (auto result = mdb_cursor_put(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_NODUPDATA)) {
1250  if (result == MDB_KEYEXIST)
1251  throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db"));
1252  else
1253  throw1(DB_ERROR(lmdb_error("Error adding spent key image to db transaction: ", result).c_str()));
1254  }
1255 }
1256 
1257 void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image)
1258 {
1259  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1260  check_open();
1261  mdb_txn_cursors *m_cursors = &m_wcursors;
1262 
1263  CURSOR(spent_keys)
1264 
1265  MDB_val k = {sizeof(k_image), (void *)&k_image};
1266  auto result = mdb_cursor_get(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_GET_BOTH);
1267  if (result != 0 && result != MDB_NOTFOUND)
1268  throw1(DB_ERROR(lmdb_error("Error finding spent key to remove", result).c_str()));
1269  if (!result)
1270  {
1271  result = mdb_cursor_del(m_cur_spent_keys, 0);
1272  if (result)
1273  throw1(DB_ERROR(lmdb_error("Error adding removal of key image to db transaction", result).c_str()));
1274  }
1275 }
1276 
1277 blobdata BlockchainLMDB::output_to_blob(const tx_out& output) const
1278 {
1279  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1280  blobdata b;
1281  if (!t_serializable_object_to_blob(output, b))
1282  throw1(DB_ERROR("Error serializing output to blob"));
1283  return b;
1284 }
1285 
1286 tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const
1287 {
1288  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1289  std::stringstream ss;
1290  ss << blob;
1291  binary_archive<false> ba(ss);
1292  tx_out o;
1293 
1294  if (!(::serialization::serialize(ba, o)))
1295  throw1(DB_ERROR("Error deserializing tx output blob"));
1296 
1297  return o;
1298 }
1299 
1300 blobdata BlockchainLMDB::validator_to_blob(const validator_db& v) const
1301 {
1302  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1303  blobdata b;
1304 
1305  if (!t_serializable_object_to_blob(v, b))
1306  throw1(DB_ERROR("Error serializing validator to blob"));
1307  return b;
1308 }
1309 
1310 validator_db BlockchainLMDB::validator_from_blob(const blobdata blob) const
1311 {
1312  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1313  std::stringstream ss;
1314  ss << blob;
1315  binary_archive<false> ba(ss);
1316  validator_db o = AUTO_VAL_INIT(o);
1317 
1318  if (!(::serialization::serialize(ba, o)))
1319  throw1(DB_ERROR("Error deserializing validator blob"));
1320 
1321  return o;
1322 }
1323 
1324 BlockchainLMDB::~BlockchainLMDB()
1325 {
1326  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1327 
1328  // batch transaction shouldn't be active at this point. If it is, consider it aborted.
1329  if (m_batch_active)
1330  {
1331  try { batch_abort(); }
1332  catch (...) { /* ignore */ }
1333  }
1334  if (m_open)
1335  close();
1336 }
1337 
1338 BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB()
1339 {
1340  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1341  // initialize folder to something "safe" just in case
1342  // someone accidentally misuses this class...
1343  m_folder = "thishsouldnotexistbecauseitisgibberish";
1344 
1345  m_batch_transactions = batch_transactions;
1346  m_write_txn = nullptr;
1347  m_write_batch_txn = nullptr;
1348  m_batch_active = false;
1349  m_cum_size = 0;
1350  m_cum_count = 0;
1351 
1352  // reset may also need changing when initialize things here
1353 
1354  m_hardfork = nullptr;
1355 }
1356 
1357 void BlockchainLMDB::open(const std::string& filename, const int db_flags)
1358 {
1359  int result;
1360  int mdb_flags = MDB_NORDAHEAD;
1361 
1362  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1363 
1364  if (m_open)
1365  throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open"));
1366 
1367  boost::filesystem::path direc(filename);
1368  if (boost::filesystem::exists(direc))
1369  {
1370  if (!boost::filesystem::is_directory(direc))
1371  throw0(DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed"));
1372  }
1373  else
1374  {
1375  if (!boost::filesystem::create_directories(direc))
1376  throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str()));
1377  }
1378 
1379  // check for existing LMDB files in base directory
1380  boost::filesystem::path old_files = direc.parent_path();
1381  if (boost::filesystem::exists(old_files / CRYPTONOTE_BLOCKCHAINDATA_FILENAME)
1382  || boost::filesystem::exists(old_files / CRYPTONOTE_BLOCKCHAINDATA_LOCK_FILENAME))
1383  {
1384  LOG_PRINT_L0("Found existing LMDB files in " << old_files.string());
1385  LOG_PRINT_L0("Move " << CRYPTONOTE_BLOCKCHAINDATA_FILENAME << " and/or " << CRYPTONOTE_BLOCKCHAINDATA_LOCK_FILENAME << " to " << filename << ", or delete them, and then restart");
1386  throw DB_ERROR("Database could not be opened");
1387  }
1388 
1389  boost::optional<bool> is_hdd_result = tools::is_hdd(filename.c_str());
1390  if (is_hdd_result)
1391  {
1392  if (is_hdd_result.value())
1393  MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use an SSD if possible");
1394  }
1395 
1396  m_folder = filename;
1397 
1398 #ifdef __OpenBSD__
1399  if ((mdb_flags & MDB_WRITEMAP) == 0) {
1400  MCLOG_RED(el::Level::Info, "global", "Running on OpenBSD: forcing WRITEMAP");
1401  mdb_flags |= MDB_WRITEMAP;
1402  }
1403 #endif
1404  // set up lmdb environment
1405  if ((result = mdb_env_create(&m_env)))
1406  throw0(DB_ERROR(lmdb_error("Failed to create lmdb environment: ", result).c_str()));
1407  if ((result = mdb_env_set_maxdbs(m_env, 26)))
1408  throw0(DB_ERROR(lmdb_error("Failed to set max number of dbs: ", result).c_str()));
1409 
1410  int threads = tools::get_max_concurrency();
1411  if (threads > 110 && /* maxreaders default is 126, leave some slots for other read processes */
1412  (result = mdb_env_set_maxreaders(m_env, threads+16)))
1413  throw0(DB_ERROR(lmdb_error("Failed to set max number of readers: ", result).c_str()));
1414 
1415  size_t mapsize = DEFAULT_MAPSIZE;
1416 
1417  if (db_flags & DBF_FAST)
1418  mdb_flags |= MDB_NOSYNC;
1419  if (db_flags & DBF_FASTEST)
1420  mdb_flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC;
1421  if (db_flags & DBF_RDONLY)
1422  mdb_flags = MDB_RDONLY;
1423  if (db_flags & DBF_SALVAGE)
1424  mdb_flags |= MDB_PREVSNAPSHOT;
1425 
1426  if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644))
1427  throw0(DB_ERROR(lmdb_error("Failed to open lmdb environment: ", result).c_str()));
1428 
1429  MDB_envinfo mei;
1430  mdb_env_info(m_env, &mei);
1431  uint64_t cur_mapsize = (uint64_t)mei.me_mapsize;
1432 
1433  if (cur_mapsize < mapsize)
1434  {
1435  if (auto result = mdb_env_set_mapsize(m_env, mapsize))
1436  throw0(DB_ERROR(lmdb_error("Failed to set max memory map size: ", result).c_str()));
1437  mdb_env_info(m_env, &mei);
1438  cur_mapsize = (uint64_t)mei.me_mapsize;
1439  LOG_PRINT_L1("LMDB memory map size: " << cur_mapsize);
1440  }
1441 
1442  if (need_resize())
1443  {
1444  LOG_PRINT_L0("LMDB memory map needs to be resized, doing that now.");
1445  do_resize();
1446  }
1447 
1448  int txn_flags = 0;
1449  if (mdb_flags & MDB_RDONLY)
1450  txn_flags |= MDB_RDONLY;
1451 
1452  // get a read/write MDB_txn, depending on mdb_flags
1453  mdb_txn_safe txn;
1454  if (auto mdb_res = mdb_txn_begin(m_env, NULL, txn_flags, txn))
1455  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str()));
1456 
1457  // open necessary databases, and set properties as needed
1458  // uses macros to avoid having to change things too many places
1459  // also change blockchain_prune.cpp to match
1460  lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks");
1461 
1462  lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for m_block_info");
1463  lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for m_block_heights");
1464 
1465  lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs");
1466  lmdb_db_open(txn, LMDB_TXS_PRUNED, MDB_INTEGERKEY | MDB_CREATE, m_txs_pruned, "Failed to open db handle for m_txs_pruned");
1467  lmdb_db_open(txn, LMDB_TXS_PRUNABLE, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable, "Failed to open db handle for m_txs_prunable");
1468  lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash");
1469  if (!(mdb_flags & MDB_RDONLY))
1470  lmdb_db_open(txn, LMDB_TXS_PRUNABLE_TIP, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_tip, "Failed to open db handle for m_txs_prunable_tip");
1471  lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices");
1472  lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs");
1473 
1474  lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_txs, "Failed to open db handle for m_output_txs");
1475  lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
1476 
1477  lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys");
1478 
1479  lmdb_db_open(txn, LMDB_TXPOOL_META, MDB_CREATE, m_txpool_meta, "Failed to open db handle for m_txpool_meta");
1480  lmdb_db_open(txn, LMDB_TXPOOL_BLOB, MDB_CREATE, m_txpool_blob, "Failed to open db handle for m_txpool_blob");
1481 
1482  // this subdb is dropped on sight, so it may not be present when we open the DB.
1483  // Since we use MDB_CREATE, we'll get an exception if we open read-only and it does not exist.
1484  // So we don't open for read-only, and also not drop below. It is not used elsewhere.
1485  if (!(mdb_flags & MDB_RDONLY))
1486  lmdb_db_open(txn, LMDB_HF_STARTING_HEIGHTS, MDB_CREATE, m_hf_starting_heights, "Failed to open db handle for m_hf_starting_heights");
1487 
1488  lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_INTEGERKEY | MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions");
1489 
1490  lmdb_db_open(txn, LMDB_VALIDATORS, MDB_INTEGERKEY | MDB_CREATE, m_validators, "Failed to open db handle for m_validators");
1491  lmdb_db_open(txn, LMDB_UTXOS, MDB_CREATE, m_utxos, "Failed to open db handle for m_utxos");
1492  lmdb_db_open(txn, LMDB_ADDR_OUTPUTS, MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_addr_outputs, "Failed to open db handle for m_addr_outputs");
1493  lmdb_db_open(txn, LMDB_ADDR_TXS, MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_addr_txs, "Failed to open db handle for m_addr_txs");
1494  lmdb_db_open(txn, LMDB_ADDR_TXS_OLD, MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_addr_txs_old, "Failed to open db handle for m_addr_txs_old");
1495  if(db_flags & DBF_ADDR_TX_SALVAGE) {
1496  mdb_drop(txn, m_addr_txs, 0);
1497  mdb_drop(txn, m_addr_txs_old, 1);
1498  }
1499  lmdb_db_open(txn, LMDB_TX_INPUTS, MDB_CREATE, m_tx_inputs, "Failed to open db handle for m_tx_inputs");
1500  lmdb_db_open(txn, LMDB_PROPERTIES, MDB_CREATE, m_properties, "Failed to open db handle for m_properties");
1501 
1502  mdb_set_dupsort(txn, m_spent_keys, compare_hash32);
1503  mdb_set_dupsort(txn, m_block_heights, compare_hash32);
1504  mdb_set_dupsort(txn, m_tx_indices, compare_hash32);
1505  mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
1506  mdb_set_dupsort(txn, m_output_txs, compare_uint64);
1507  mdb_set_dupsort(txn, m_block_info, compare_uint64);
1508  if (!(mdb_flags & MDB_RDONLY))
1509  mdb_set_dupsort(txn, m_txs_prunable_tip, compare_uint64);
1510  mdb_set_compare(txn, m_txs_prunable, compare_uint64);
1511  mdb_set_dupsort(txn, m_txs_prunable_hash, compare_uint64);
1512 
1513  mdb_set_compare(txn, m_utxos, compare_data);
1514  mdb_set_compare(txn, m_txpool_meta, compare_hash32);
1515  mdb_set_compare(txn, m_txpool_blob, compare_hash32);
1516  mdb_set_compare(txn, m_properties, compare_string);
1517 
1518  mdb_set_dupsort(txn, m_addr_outputs, compare_uint64);
1519  mdb_set_compare(txn, m_addr_outputs, compare_publickey);
1520 
1521  mdb_set_dupsort(txn, m_addr_txs, compare_uint64);
1522  mdb_set_compare(txn, m_addr_txs, compare_publickey);
1523 
1524  mdb_set_dupsort(txn, m_addr_txs_old, compare_uint64);
1525  mdb_set_compare(txn, m_addr_txs_old, compare_publickey);
1526 
1527  mdb_set_compare(txn, m_tx_inputs, compare_data);
1528 
1529 
1530  if (!(mdb_flags & MDB_RDONLY))
1531  {
1532  result = mdb_drop(txn, m_hf_starting_heights, 1);
1533  if (result && result != MDB_NOTFOUND)
1534  throw0(DB_ERROR(lmdb_error("Failed to drop m_hf_starting_heights: ", result).c_str()));
1535  }
1536 
1537  // get and keep current height
1538  MDB_stat db_stats;
1539  if ((result = mdb_stat(txn, m_blocks, &db_stats)))
1540  throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
1541  LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries);
1542  uint64_t m_height = db_stats.ms_entries;
1543 
1544  bool compatible = true;
1545 
1546  MDB_val_str(k, "version");
1547  MDB_val v;
1548  auto get_result = mdb_get(txn, m_properties, &k, &v);
1549  if(get_result == MDB_SUCCESS)
1550  {
1551  const uint32_t db_version = *(const uint32_t*)v.mv_data;
1552  if (db_version > VERSION)
1553  {
1554  MWARNING("Existing lmdb database was made by a later version (" << db_version << "). We don't know how it will change yet.");
1555  compatible = false;
1556  }
1557 #if VERSION > 0
1558  else if (db_version < VERSION)
1559  {
1560  if (mdb_flags & MDB_RDONLY)
1561  {
1562  txn.abort();
1563  mdb_env_close(m_env);
1564  m_open = false;
1565  MFATAL("Existing lmdb database needs to be converted, which cannot be done on a read-only database.");
1566  MFATAL("Please run electroneumd once to convert the database.");
1567  return;
1568  }
1569  // Note that there was a schema change within version 0 as well.
1570  // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10
1571  // We don't handle the old format previous to that commit.
1572  txn.commit();
1573  m_open = true;
1574  migrate(db_version);
1575  return;
1576  }
1577 #endif
1578  }
1579  else
1580  {
1581  // if not found, and the DB is non-empty, this is probably
1582  // an "old" version 0, which we don't handle. If the DB is
1583  // empty it's fine.
1584  if (VERSION > 0 && m_height > 0)
1585  compatible = false;
1586  }
1587 
1588  if (!compatible)
1589  {
1590  txn.abort();
1591  mdb_env_close(m_env);
1592  m_open = false;
1593  MFATAL("Existing lmdb database is incompatible with this version.");
1594  MFATAL("Please delete the existing database and resync.");
1595  return;
1596  }
1597 
1598  if (!(mdb_flags & MDB_RDONLY))
1599  {
1600  // only write version on an empty DB
1601  if (m_height == 0)
1602  {
1603  MDB_val_str(k, "version");
1604  MDB_val_copy<uint32_t> v(VERSION);
1605  auto put_result = mdb_put(txn, m_properties, &k, &v, 0);
1606  if (put_result != MDB_SUCCESS)
1607  {
1608  txn.abort();
1609  mdb_env_close(m_env);
1610  m_open = false;
1611  MERROR("Failed to write version to database.");
1612  return;
1613  }
1614  }
1615  }
1616 
1617  // commit the transaction
1618  txn.commit();
1619 
1620  m_open = true;
1621  // from here, init should be finished
1622 }
1623 
1625 {
1626  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1627  if (m_batch_active)
1628  {
1629  LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction");
1630  batch_abort();
1631  }
1632  this->sync();
1633  m_tinfo.reset();
1634 
1635  // FIXME: not yet thread safe!!! Use with care.
1636  mdb_env_close(m_env);
1637  m_open = false;
1638 }
1639 
1641 {
1642  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1643  check_open();
1644 
1645  if (is_read_only())
1646  return;
1647 
1648  // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part
1649  // MDB_NOMETASYNC. Force flush to be synchronous.
1650  if (auto result = mdb_env_sync(m_env, true))
1651  {
1652  throw0(DB_ERROR(lmdb_error("Failed to sync database: ", result).c_str()));
1653  }
1654 }
1655 
1656 void BlockchainLMDB::safesyncmode(const bool onoff)
1657 {
1658  MINFO("switching safe mode " << (onoff ? "on" : "off"));
1659  mdb_env_set_flags(m_env, MDB_NOSYNC|MDB_MAPASYNC, !onoff);
1660 }
1661 
1663 {
1664  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1665  check_open();
1666 
1667  mdb_txn_safe txn;
1668  if (auto result = lmdb_txn_begin(m_env, NULL, 0, txn))
1669  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
1670 
1671  if (auto result = mdb_drop(txn, m_blocks, 0))
1672  throw0(DB_ERROR(lmdb_error("Failed to drop m_blocks: ", result).c_str()));
1673  if (auto result = mdb_drop(txn, m_block_info, 0))
1674  throw0(DB_ERROR(lmdb_error("Failed to drop m_block_info: ", result).c_str()));
1675  if (auto result = mdb_drop(txn, m_block_heights, 0))
1676  throw0(DB_ERROR(lmdb_error("Failed to drop m_block_heights: ", result).c_str()));
1677  if (auto result = mdb_drop(txn, m_txs_pruned, 0))
1678  throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_pruned: ", result).c_str()));
1679  if (auto result = mdb_drop(txn, m_txs_prunable, 0))
1680  throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable: ", result).c_str()));
1681  if (auto result = mdb_drop(txn, m_txs_prunable_hash, 0))
1682  throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_hash: ", result).c_str()));
1683  if (auto result = mdb_drop(txn, m_txs_prunable_tip, 0))
1684  throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_tip: ", result).c_str()));
1685  if (auto result = mdb_drop(txn, m_tx_indices, 0))
1686  throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_indices: ", result).c_str()));
1687  if (auto result = mdb_drop(txn, m_tx_outputs, 0))
1688  throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_outputs: ", result).c_str()));
1689  if (auto result = mdb_drop(txn, m_output_txs, 0))
1690  throw0(DB_ERROR(lmdb_error("Failed to drop m_output_txs: ", result).c_str()));
1691  if (auto result = mdb_drop(txn, m_output_amounts, 0))
1692  throw0(DB_ERROR(lmdb_error("Failed to drop m_output_amounts: ", result).c_str()));
1693  if (auto result = mdb_drop(txn, m_spent_keys, 0))
1694  throw0(DB_ERROR(lmdb_error("Failed to drop m_spent_keys: ", result).c_str()));
1695  (void)mdb_drop(txn, m_hf_starting_heights, 0); // this one is dropped in new code
1696  if (auto result = mdb_drop(txn, m_hf_versions, 0))
1697  throw0(DB_ERROR(lmdb_error("Failed to drop m_hf_versions: ", result).c_str()));
1698  if (auto result = mdb_drop(txn, m_validators, 0))
1699  throw0(DB_ERROR(lmdb_error("Failed to drop m_validators: ", result).c_str()));
1700  if (auto result = mdb_drop(txn, m_utxos, 0))
1701  throw0(DB_ERROR(lmdb_error("Failed to drop m_utxos: ", result).c_str()));
1702  if (auto result = mdb_drop(txn, m_addr_outputs, 0))
1703  throw0(DB_ERROR(lmdb_error("Failed to drop m_addr_outputs: ", result).c_str()));
1704  if (auto result = mdb_drop(txn, m_addr_txs, 0))
1705  throw0(DB_ERROR(lmdb_error("Failed to drop m_addr_txs: ", result).c_str()));
1706  if (auto result = mdb_drop(txn, m_addr_txs_old, 0))
1707  throw0(DB_ERROR(lmdb_error("Failed to drop m_addr_txs_old: ", result).c_str()));
1708  if (auto result = mdb_drop(txn, m_tx_inputs, 0))
1709  throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_inputs: ", result).c_str()));
1710  if (auto result = mdb_drop(txn, m_properties, 0))
1711  throw0(DB_ERROR(lmdb_error("Failed to drop m_properties: ", result).c_str()));
1712 
1713  // init with current version
1714  MDB_val_str(k, "version");
1715  MDB_val_copy<uint32_t> v(VERSION);
1716  if (auto result = mdb_put(txn, m_properties, &k, &v, 0))
1717  throw0(DB_ERROR(lmdb_error("Failed to write version to database: ", result).c_str()));
1718 
1719  txn.commit();
1720  m_cum_size = 0;
1721  m_cum_count = 0;
1722 }
1723 
1724 std::vector<std::string> BlockchainLMDB::get_filenames() const
1725 {
1726  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1727  std::vector<std::string> filenames;
1728 
1729  boost::filesystem::path datafile(m_folder);
1731  boost::filesystem::path lockfile(m_folder);
1733 
1734  filenames.push_back(datafile.string());
1735  filenames.push_back(lockfile.string());
1736 
1737  return filenames;
1738 }
1739 
1741 {
1742  const std::string filename = folder + "/data.mdb";
1743  try
1744  {
1745  boost::filesystem::remove(filename);
1746  }
1747  catch (const std::exception &e)
1748  {
1749  MERROR("Failed to remove " << filename << ": " << e.what());
1750  return false;
1751  }
1752  return true;
1753 }
1754 
1756 {
1757  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1758 
1759  return std::string("lmdb");
1760 }
1761 
1762 // TODO: this?
1764 {
1765  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1766  check_open();
1767  return false;
1768 }
1769 
1770 // TODO: this?
1772 {
1773  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1774  check_open();
1775 }
1776 
1777 #define TXN_PREFIX(flags); \
1778  mdb_txn_safe auto_txn; \
1779  mdb_txn_safe* txn_ptr = &auto_txn; \
1780  if (m_batch_active) \
1781  txn_ptr = m_write_txn; \
1782  else \
1783  { \
1784  if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \
1785  throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \
1786  } \
1787 
1788 #define TXN_PREFIX_RDONLY() \
1789  MDB_txn *m_txn; \
1790  mdb_txn_cursors *m_cursors; \
1791  mdb_txn_safe auto_txn; \
1792  bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); \
1793  if (my_rtxn) auto_txn.m_tinfo = m_tinfo.get(); \
1794  else auto_txn.uncheck()
1795 #define TXN_POSTFIX_RDONLY()
1796 
1797 #define TXN_POSTFIX_SUCCESS() \
1798  do { \
1799  if (! m_batch_active) \
1800  auto_txn.commit(); \
1801  } while(0)
1802 
1803 
1804 // The below two macros are for DB access within block add/remove, whether
1805 // regular batch txn is in use or not. m_write_txn is used as a batch txn, even
1806 // if it's only within block add/remove.
1807 //
1808 // DB access functions that may be called both within block add/remove and
1809 // without should use these. If the function will be called ONLY within block
1810 // add/remove, m_write_txn alone may be used instead of these macros.
1811 
1812 #define TXN_BLOCK_PREFIX(flags); \
1813  mdb_txn_safe auto_txn; \
1814  mdb_txn_safe* txn_ptr = &auto_txn; \
1815  if (m_batch_active || m_write_txn) \
1816  txn_ptr = m_write_txn; \
1817  else \
1818  { \
1819  if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \
1820  throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \
1821  } \
1822 
1823 #define TXN_BLOCK_POSTFIX_SUCCESS() \
1824  do { \
1825  if (! m_batch_active && ! m_write_txn) \
1826  auto_txn.commit(); \
1827  } while(0)
1828 
1829 
1830 void BlockchainLMDB::add_chainstate_utxo(const crypto::hash tx_hash, const uint32_t relative_out_index,
1831  const crypto::public_key combined_key, uint64_t amount, uint64_t unlock_time, bool is_coinbase)
1832 {
1833  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1834  check_open();
1835 
1836  mdb_txn_cursors *m_cursors = &m_wcursors;
1837  CURSOR(utxos)
1838 
1839  int result = 0;
1840 
1841  chainstate_key_t index;
1842  index.tx_hash = tx_hash;
1843  index.relative_out_index = relative_out_index;
1844 
1845  chainstate_value_t data;
1846  data.amount = amount;
1847  data.combined_key = combined_key;
1848  data.is_coinbase = is_coinbase;
1849  data.unlock_time = unlock_time;
1850 
1851  MDB_val_set(k, index);
1852  MDB_val_set(v, data);
1853 
1854  if (auto result = mdb_cursor_put(m_cur_utxos, &k, &v, MDB_NODUPDATA)) {
1855  if (result == MDB_KEYEXIST)
1856  throw1(UTXO_EXISTS("Attempting to add utxo that's already in the db"));
1857  else
1858  throw1(DB_ERROR(lmdb_error("Error adding utxo to db transaction: ", result).c_str()));
1859  }
1860 }
1861 
1862 bool BlockchainLMDB::check_chainstate_utxo(const crypto::hash tx_hash, const uint32_t relative_out_index)
1863 {
1864  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1865  check_open();
1866 
1868  RCURSOR(utxos)
1869 
1870  chainstate_key_t index;
1871  index.tx_hash = tx_hash;
1872  index.relative_out_index = relative_out_index;
1873 
1874  MDB_val k = {sizeof(index), (void *)&index};
1875 
1876  auto result = mdb_cursor_get(m_cur_utxos, &k, NULL, MDB_SET);
1877  if (result == MDB_NOTFOUND)
1878  return false;
1879  if (result != 0)
1880  throw1(DB_ERROR(lmdb_error("Error finding utxo: ", result).c_str()));
1881 
1883  return true;
1884 }
1885 
1886 uint64_t BlockchainLMDB::get_utxo_unlock_time(const crypto::hash tx_hash, const uint32_t relative_out_index)
1887 {
1888  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1889  check_open();
1890 
1892  RCURSOR(utxos)
1893 
1894  chainstate_key_t index;
1895  index.tx_hash = tx_hash;
1896  index.relative_out_index = relative_out_index;
1897 
1898  MDB_val k = {sizeof(index), (void *)&index};
1899  MDB_val v;
1900 
1901  auto result = mdb_cursor_get(m_cur_utxos, &k, &v, MDB_SET_KEY);
1902  if (result == MDB_NOTFOUND)
1903  return false;
1904  if (result != 0)
1905  throw1(DB_ERROR(lmdb_error("Error finding utxo: ", result).c_str()));
1906 
1907  auto res = *(const chainstate_value_t *) v.mv_data;
1909  return res.unlock_time;
1910 }
1911 
1912 
1913 void BlockchainLMDB::remove_chainstate_utxo(const crypto::hash tx_hash, const uint32_t relative_out_index)
1914 {
1915  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1916  check_open();
1917 
1918  mdb_txn_cursors *m_cursors = &m_wcursors;
1919  CURSOR(utxos)
1920 
1921  chainstate_key_t index;
1922  index.tx_hash = tx_hash;
1923  index.relative_out_index = relative_out_index;
1924 
1925  MDB_val k = {sizeof(index), (void *)&index};
1926 
1927  auto result = mdb_cursor_get(m_cur_utxos, &k, NULL, MDB_SET);
1928  if (result != 0 && result != MDB_NOTFOUND)
1929  throw1(DB_ERROR(lmdb_error("Error finding utxo to remove", result).c_str()));
1930  if (!result)
1931  {
1932  result = mdb_cursor_del(m_cur_utxos, 0);
1933  if (result)
1934  throw1(DB_ERROR(lmdb_error("Error adding removal of utxo to db transaction", result).c_str()));
1935  }
1936 }
1937 
1938 /* todo: This database is currently populated but isn't used at all. It may be broken because of a bug where tx inputs added
1939  * to the db weren't properly rewinded in the case of a bc transaction addition aborting mid way through. To solve this before using this database,
1940  * the database should be emptied and repopulated, in Blockchain::init. */
1941 void BlockchainLMDB::add_tx_input(const crypto::hash tx_hash, const uint32_t relative_out_index, const crypto::hash parent_tx_hash, const uint64_t in_index)
1942 {
1943  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1944  check_open();
1945 
1946  mdb_txn_cursors *m_cursors = &m_wcursors;
1947  CURSOR(tx_inputs)
1948 
1949  int result = 0;
1950 
1951  std::string hex_hash;
1952  hex_hash = epee::string_tools::pod_to_hex(tx_hash);
1953 
1954  chainstate_key_t key;
1955  key.tx_hash = tx_hash;
1956  key.relative_out_index = relative_out_index;
1957 
1958  tx_input_t data;
1959  data.tx_hash = parent_tx_hash;
1960  data.in_index = in_index;
1961 
1962  MDB_val_set(k, key);
1963  MDB_val_set(v, data);
1964 
1965  if (auto result = mdb_cursor_put(m_cur_tx_inputs, &k, &v, MDB_NODUPDATA)) {
1966  if (result == MDB_KEYEXIST)
1967  throw1(UTXO_EXISTS("Attempting to add tx input that's already in the db"));
1968  else
1969  throw1(DB_ERROR(lmdb_error("Error adding tx input to db transaction: ", result).c_str()));
1970  }
1971 }
1972 
1973 tx_input_t BlockchainLMDB::get_tx_input(const crypto::hash tx_hash, const uint32_t relative_out_index)
1974 {
1975  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
1976  check_open();
1977 
1978  std::string hex_hash;
1979  hex_hash = epee::string_tools::pod_to_hex(tx_hash);
1981  RCURSOR(tx_inputs)
1982 
1984  key.tx_hash = tx_hash;
1985  key.relative_out_index = relative_out_index;
1986 
1987  MDB_val k = {sizeof(key), (void *)&key};
1988  MDB_val v;
1989  auto result = mdb_cursor_get(m_cur_tx_inputs, &k, &v, MDB_SET_KEY);
1990  if (result == MDB_NOTFOUND)
1991  return tx_input_t();
1992  if (result != 0)
1993  throw1(DB_ERROR(lmdb_error("Error finding tx input: ", result).c_str()));
1994 
1996  return *(const tx_input_t *) v.mv_data;
1997 }
1998 
1999 void BlockchainLMDB::remove_tx_input(const crypto::hash tx_hash, const uint32_t relative_out_index)
2000 {
2001  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2002  check_open();
2003 
2004  mdb_txn_cursors *m_cursors = &m_wcursors;
2005  CURSOR(tx_inputs)
2006 
2008  key.tx_hash = tx_hash;
2009  key.relative_out_index = relative_out_index;
2010 
2011  MDB_val k = {sizeof(key), (void *)&key};
2012 
2013  auto result = mdb_cursor_get(m_cur_tx_inputs, &k, NULL, MDB_SET);
2014  if (result != 0 && result != MDB_NOTFOUND)
2015  throw1(DB_ERROR(lmdb_error("Error finding tx input to remove", result).c_str()));
2016  if (!result)
2017  {
2018  result = mdb_cursor_del(m_cur_tx_inputs, 0);
2019  if (result)
2020  throw1(DB_ERROR(lmdb_error("Error adding removal of tx input to db transaction", result).c_str()));
2021  }
2022 }
2023 
2024 void BlockchainLMDB::add_addr_output(const crypto::hash tx_hash, const uint32_t relative_out_index,
2025  const crypto::public_key& combined_key,
2026  uint64_t amount, uint64_t unlock_time)
2027 {
2028  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2029  check_open();
2030  mdb_txn_cursors *m_cursors = &m_wcursors;
2031  CURSOR(addr_outputs)
2032 
2033  int result = 0;
2034 
2035  MDB_val k = {sizeof(combined_key), (void *)&combined_key};
2036  MDB_val v;
2037  result = mdb_cursor_get(m_cur_addr_outputs, &k, &v, MDB_SET);
2038  if (result != 0 && result != MDB_NOTFOUND)
2039  throw1(DB_ERROR(lmdb_error("Error finding addr output to add: ", result).c_str()));
2040 
2041  mdb_size_t num_elems = 0;
2042 
2043  if(result == 0)
2044  {
2045  result = mdb_cursor_get(m_cur_addr_outputs, &k, &v, MDB_LAST_DUP);
2046  if (result)
2047  throw0(DB_ERROR(std::string("Failed to get number outputs for address: ").append(mdb_strerror(result)).c_str()));
2048 
2049  const acc_outs_t res = *(const acc_outs_t *) v.mv_data;
2050  num_elems = res.db_index + 1;
2051  }
2052 
2053  acc_outs_t acc;
2054  acc.db_index = num_elems;
2055  acc.tx_hash = tx_hash;
2056  acc.relative_out_index = relative_out_index;
2057  acc.amount = amount;
2058  acc.unlock_time = unlock_time;
2059 
2060  k = {sizeof(combined_key), (void *)&combined_key};
2061  MDB_val acc_v = {sizeof(acc), (void *)&acc};
2062 
2063  result = mdb_cursor_put(m_cur_addr_outputs, &k, &acc_v, MDB_APPENDDUP);
2064  if (result == MDB_KEYEXIST)
2065  throw1(UTXO_EXISTS("Attempting to add addr output that's already in the db."));
2066  else if(result != 0)
2067  throw1(DB_ERROR(lmdb_error("Error adding addr output to db transaction: ", result).c_str()));
2068 
2069 }
2070 
2071 std::vector<address_outputs> BlockchainLMDB::get_addr_output_all(const crypto::public_key& combined_key)
2072 {
2073  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2074  check_open();
2075 
2077  RCURSOR(addr_outputs);
2078 
2079  int result = 0;
2080  std::vector<address_outputs> address_outputs;
2081 
2082  MDB_val k = {sizeof(combined_key), (void *)&combined_key};
2083 
2085  while (1) {
2086  MDB_val v;
2087  int ret = mdb_cursor_get(m_cur_addr_outputs, &k, &v, op);
2088  op = MDB_NEXT_DUP;
2089  if (ret == MDB_NOTFOUND)
2090  break;
2091  if (ret)
2092  throw0(DB_ERROR("Failed to enumerate address outputs"));
2093 
2094  const acc_outs_t res = *(const acc_outs_t *) v.mv_data;
2095 
2096  cryptonote::address_outputs addr_out;
2097  addr_out.out_id = res.db_index;
2098  addr_out.tx_hash = res.tx_hash;
2099  addr_out.relative_out_index = res.relative_out_index;
2100  addr_out.amount = res.amount;
2101  addr_out.spent = !check_chainstate_utxo(res.tx_hash, res.relative_out_index);
2102 
2103  address_outputs.push_back(addr_out);
2104 
2105  }
2106 
2108 
2109  return address_outputs;
2110 }
2111 
2112 std::vector<address_outputs> BlockchainLMDB::get_addr_output_batch(const crypto::public_key& combined_key, uint64_t start_db_index, uint64_t batch_size, bool desc)
2113 {
2114  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2115  check_open();
2116 
2118  RCURSOR(addr_outputs);
2119 
2120  std::vector<address_outputs> address_outputs;
2121 
2122  MDB_val k = {sizeof(combined_key), (void *)&combined_key};
2123  MDB_val v;
2124 
2125  MDB_cursor_op op;
2126  if (start_db_index)
2127  op = MDB_GET_BOTH;
2128  else
2129  {
2130  op = desc ? MDB_LAST_DUP : MDB_FIRST_DUP;
2131  int result = mdb_cursor_get(m_cur_addr_outputs, &k, &v, MDB_SET_KEY);
2132  if (result != 0 && result != MDB_NOTFOUND)
2133  throw1(DB_ERROR(lmdb_error("Failed to enumerate address outputs", result).c_str()));
2134  }
2135 
2136  std::set<std::string> tx_hashes;
2137  for(size_t i = 0; i < batch_size + 1; ++i) {
2138  if(op == MDB_GET_BOTH)
2139  v = MDB_val{sizeof(start_db_index), (void*)&start_db_index};
2140 
2141  int ret = mdb_cursor_get(m_cur_addr_outputs, &k, &v, op);
2142  op = desc ? MDB_PREV_DUP : MDB_NEXT_DUP;
2143  if (ret == MDB_NOTFOUND)
2144  break;
2145  if (ret)
2146  throw0(DB_ERROR("Failed to enumerate address outputs"));
2147 
2148  const acc_outs_t res = *(const acc_outs_t *) v.mv_data;
2149 
2150  std::string tx_hash_hex = epee::string_tools::pod_to_hex(res.tx_hash);
2151  if(tx_hashes.find(tx_hash_hex) != tx_hashes.end())
2152  {
2153  --i;
2154  continue;
2155  }
2156 
2157  cryptonote::address_outputs addr_out;
2158  addr_out.out_id = res.db_index;
2159  addr_out.tx_hash = res.tx_hash;
2160  addr_out.relative_out_index = res.relative_out_index;
2161  addr_out.amount = res.amount;
2162  addr_out.spent = !check_chainstate_utxo(res.tx_hash, res.relative_out_index);
2163 
2164  address_outputs.push_back(addr_out);
2165  tx_hashes.emplace(tx_hash_hex);
2166  }
2167 
2169  return address_outputs;
2170 }
2171 
2172 void BlockchainLMDB::add_addr_tx(const crypto::hash tx_hash, const crypto::public_key& combined_key)
2173 {
2174  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2175  check_open();
2176  mdb_txn_cursors *m_cursors = &m_wcursors;
2177  CURSOR(addr_txs)
2178 
2179  int result = 0;
2180 
2181  MDB_val k = {sizeof(combined_key), (void *)&combined_key};
2182  MDB_val v;
2183  result = mdb_cursor_get(m_cur_addr_txs, &k, &v, MDB_SET);
2184  if (result != 0 && result != MDB_NOTFOUND)
2185  throw1(DB_ERROR(lmdb_error("Error finding addr tx to add: ", result).c_str()));
2186 
2187  mdb_size_t num_elems = 0;
2188 
2189  if(result == 0)
2190  {
2191  result = mdb_cursor_get(m_cur_addr_txs, &k, &v, MDB_LAST_DUP);
2192  if (result)
2193  throw0(DB_ERROR(std::string("Failed to get number txs for address: ").append(mdb_strerror(result)).c_str()));
2194 
2195  const acc_addr_tx_t res = *(const acc_addr_tx_t *) v.mv_data;
2196  num_elems = res.db_index + 1;
2197  }
2198 
2199  acc_addr_tx_t acc;
2200  acc.db_index = num_elems;
2201  acc.tx_hash = tx_hash;
2202 
2203  k = {sizeof(combined_key), (void *)&combined_key};
2204  MDB_val acc_v = {sizeof(acc), (void *)&acc};
2205 
2206  result = mdb_cursor_put(m_cur_addr_txs, &k, &acc_v, MDB_APPENDDUP);
2207  if (result == MDB_KEYEXIST)
2208  throw1(UTXO_EXISTS("Attempting to add addr tx that's already in the db."));
2209  else if(result != 0)
2210  throw1(DB_ERROR(lmdb_error("Error adding addr tx to db transaction: ", result).c_str()));
2211 
2212 }
2213 
2214 std::vector<address_txs> BlockchainLMDB::get_addr_tx_all(const crypto::public_key& combined_key)
2215 {
2216  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2217  check_open();
2218 
2220  RCURSOR(addr_txs);
2221 
2222  int result = 0;
2223  std::vector<address_txs> address_txs;
2224 
2225  MDB_val k = {sizeof(combined_key), (void *)&combined_key};
2226 
2228  while (1) {
2229  MDB_val v;
2230  int ret = mdb_cursor_get(m_cur_addr_txs, &k, &v, op);
2231  op = MDB_NEXT_DUP;
2232  if (ret == MDB_NOTFOUND)
2233  break;
2234  if (ret)
2235  throw0(DB_ERROR("Failed to enumerate address txs"));
2236 
2237  const acc_addr_tx_t res = *(const acc_addr_tx_t *) v.mv_data;
2238 
2239  cryptonote::address_txs addr_tx;
2240  addr_tx.addr_tx_id = res.db_index;
2241  addr_tx.tx_hash = res.tx_hash;
2242  address_txs.push_back(addr_tx);
2243  }
2244 
2246  return address_txs;
2247 }
2248 
2249 std::vector<address_txs> BlockchainLMDB::get_addr_tx_batch(const crypto::public_key& combined_key, uint64_t start_db_index, uint64_t batch_size, bool desc)
2250 {
2251  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2252  check_open();
2253 
2255  RCURSOR(addr_txs);
2256 
2257  std::vector<address_txs> address_txs;
2258 
2259  MDB_val k = {sizeof(combined_key), (void *)&combined_key};
2260  MDB_val v;
2261 
2262  MDB_cursor_op op;
2263  if (start_db_index)
2264  op = MDB_GET_BOTH;
2265  else
2266  {
2267  op = desc ? MDB_LAST_DUP : MDB_FIRST_DUP;
2268  int result = mdb_cursor_get(m_cur_addr_txs, &k, &v, MDB_SET_KEY);
2269  if (result != 0 && result != MDB_NOTFOUND)
2270  throw1(DB_ERROR(lmdb_error("Failed to enumerate address txs", result).c_str()));
2271  }
2272 
2273  std::set<std::string> tx_hashes;
2274  for(size_t i = 0; i < batch_size + 1; ++i) {
2275  if(op == MDB_GET_BOTH)
2276  v = MDB_val{sizeof(start_db_index), (void*)&start_db_index};
2277 
2278  int ret = mdb_cursor_get(m_cur_addr_txs, &k, &v, op);
2279  op = desc ? MDB_PREV_DUP : MDB_NEXT_DUP;
2280  if (ret == MDB_NOTFOUND)
2281  break;
2282  if (ret)
2283  throw0(DB_ERROR("Failed to enumerate address txs"));
2284 
2285  const acc_addr_tx_t res = *(const acc_addr_tx_t *) v.mv_data;
2286 
2287  cryptonote::address_txs addr_tx;
2288  addr_tx.addr_tx_id = res.db_index;
2289  addr_tx.tx_hash = res.tx_hash;
2290 
2291  address_txs.push_back(addr_tx);
2292  }
2293 
2295  return address_txs;
2296 }
2297 
2299 {
2300  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2301  check_open();
2302 
2304  RCURSOR(addr_outputs);
2305 
2306  uint64_t balance = 0;
2307 
2308  MDB_val k = {sizeof(combined_key), (void *)&combined_key};
2309 
2311  while (1) {
2312  MDB_val v;
2313  int ret = mdb_cursor_get(m_cur_addr_outputs, &k, &v, op);
2314  op = MDB_NEXT_DUP;
2315  if (ret == MDB_NOTFOUND)
2316  break;
2317  if (ret)
2318  throw0(DB_ERROR("Failed to enumerate address outputs"));
2319 
2320  const acc_outs_t res = *(const acc_outs_t *) v.mv_data;
2321 
2322  if(check_chainstate_utxo(res.tx_hash, res.relative_out_index))
2323  balance += res.amount;
2324  }
2325 
2327 
2328  return balance;
2329 }
2330 
2331 void BlockchainLMDB::remove_addr_output(const crypto::hash tx_hash, const uint32_t relative_out_index,
2332  const crypto::public_key& combined_key,
2333  uint64_t amount, uint64_t unlock_time)
2334 {
2335  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2336  check_open();
2337  mdb_txn_cursors *m_cursors = &m_wcursors;
2338 
2339  CURSOR(addr_outputs)
2340 
2341  int result = 0;
2342 
2343  MDB_val k = {sizeof(combined_key), (void *)&combined_key};
2344  MDB_val v;
2345 
2346  result = mdb_cursor_get(m_cur_addr_outputs, &k, &v, MDB_SET);
2347  if (result != 0 && result != MDB_NOTFOUND)
2348  throw1(DB_ERROR(lmdb_error("Failed to enumerate address outputs", result).c_str()));
2349  if (result == MDB_NOTFOUND)
2350  return;
2351 
2353  while (1) {
2354  k = {sizeof(combined_key), (void *)&combined_key};
2355  int ret = mdb_cursor_get(m_cur_addr_outputs, &k, &v, op);
2356  op = MDB_PREV_DUP;
2357  if (ret == MDB_NOTFOUND)
2358  break;
2359  if (ret)
2360  throw0(DB_ERROR("Failed to enumerate outputs"));
2361 
2362  const acc_outs_t res = *(const acc_outs_t *) v.mv_data;
2363 
2364  if(res.tx_hash == tx_hash && res.relative_out_index == relative_out_index && res.amount == amount && res.unlock_time == unlock_time ) {
2365  result = mdb_cursor_del(m_cur_addr_outputs, 0);
2366  if (result)
2367  throw1(DB_ERROR(lmdb_error("Error removing of addr output from db: ", result).c_str()));
2368 
2369  break;
2370  }
2371  }
2372 }
2373 
2374 void BlockchainLMDB::remove_addr_tx(const crypto::hash tx_hash, const crypto::public_key& combined_key)
2375 {
2376  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2377  check_open();
2378  mdb_txn_cursors *m_cursors = &m_wcursors;
2379 
2380  CURSOR(addr_txs)
2381 
2382  int result = 0;
2383 
2384  MDB_val k = {sizeof(combined_key), (void *)&combined_key};
2385  MDB_val v;
2386 
2387  result = mdb_cursor_get(m_cur_addr_txs, &k, &v, MDB_SET);
2388  if (result != 0 && result != MDB_NOTFOUND)
2389  throw1(DB_ERROR(lmdb_error("Failed to enumerate address txs", result).c_str()));
2390  if (result == MDB_NOTFOUND)
2391  return;
2392 
2394  while (1) {
2395  k = {sizeof(combined_key), (void *)&combined_key};
2396  int ret = mdb_cursor_get(m_cur_addr_txs, &k, &v, op);
2397  op = MDB_PREV_DUP;
2398  if (ret == MDB_NOTFOUND)
2399  break;
2400  if (ret)
2401  throw0(DB_ERROR("Failed to enumerate outputs"));
2402 
2403  const acc_addr_tx_t res = *(const acc_addr_tx_t *) v.mv_data;
2404 
2405  if(res.tx_hash == tx_hash) {
2406  result = mdb_cursor_del(m_cur_addr_txs, 0);
2407  if (result)
2408  throw1(DB_ERROR(lmdb_error("Error removing of addr tx from db: ", result).c_str()));
2409 
2410  break;
2411  }
2412  }
2413 }
2414 
2416 {
2417  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2418  check_open();
2419  mdb_txn_cursors *m_cursors = &m_wcursors;
2420 
2421  CURSOR(txpool_meta)
2422  CURSOR(txpool_blob)
2423 
2424  MDB_val k = {sizeof(txid), (void *)&txid};
2425  MDB_val v = {sizeof(meta), (void *)&meta};
2426  if (auto result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) {
2427  if (result == MDB_KEYEXIST)
2428  throw1(DB_ERROR("Attempting to add txpool tx metadata that's already in the db"));
2429  else
2430  throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str()));
2431  }
2432  MDB_val_sized(blob_val, blob);
2433  if (auto result = mdb_cursor_put(m_cur_txpool_blob, &k, &blob_val, MDB_NODUPDATA)) {
2434  if (result == MDB_KEYEXIST)
2435  throw1(DB_ERROR("Attempting to add txpool tx blob that's already in the db"));
2436  else
2437  throw1(DB_ERROR(lmdb_error("Error adding txpool tx blob to db transaction: ", result).c_str()));
2438  }
2439 }
2440 
2442 {
2443  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2444  check_open();
2445  mdb_txn_cursors *m_cursors = &m_wcursors;
2446 
2447  CURSOR(txpool_meta)
2448  CURSOR(txpool_blob)
2449 
2450  MDB_val k = {sizeof(txid), (void *)&txid};
2451  MDB_val v;
2452  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET);
2453  if (result != 0)
2454  throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta to update: ", result).c_str()));
2455  result = mdb_cursor_del(m_cur_txpool_meta, 0);
2456  if (result)
2457  throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx metadata to db transaction: ", result).c_str()));
2458  v = MDB_val({sizeof(meta), (void *)&meta});
2459  if ((result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) != 0) {
2460  if (result == MDB_KEYEXIST)
2461  throw1(DB_ERROR("Attempting to add txpool tx metadata that's already in the db"));
2462  else
2463  throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str()));
2464  }
2465 }
2466 
2467 uint64_t BlockchainLMDB::get_txpool_tx_count(bool include_unrelayed_txes) const
2468 {
2469  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2470  check_open();
2471 
2472  int result;
2473  uint64_t num_entries = 0;
2474 
2476 
2477  if (include_unrelayed_txes)
2478  {
2479  // No filtering, we can get the number of tx the "fast" way
2480  MDB_stat db_stats;
2481  if ((result = mdb_stat(m_txn, m_txpool_meta, &db_stats)))
2482  throw0(DB_ERROR(lmdb_error("Failed to query m_txpool_meta: ", result).c_str()));
2483  num_entries = db_stats.ms_entries;
2484  }
2485  else
2486  {
2487  // Filter unrelayed tx out of the result, so we need to loop over transactions and check their meta data
2488  RCURSOR(txpool_meta);
2489  RCURSOR(txpool_blob);
2490 
2491  MDB_val k;
2492  MDB_val v;
2493  MDB_cursor_op op = MDB_FIRST;
2494  while (1)
2495  {
2496  result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, op);
2497  op = MDB_NEXT;
2498  if (result == MDB_NOTFOUND)
2499  break;
2500  if (result)
2501  throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str()));
2502  const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data;
2503  if (!meta.do_not_relay)
2504  ++num_entries;
2505  }
2506  }
2508 
2509  return num_entries;
2510 }
2511 
2513 {
2514  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2515  check_open();
2516 
2518  RCURSOR(txpool_meta)
2519 
2520  MDB_val k = {sizeof(txid), (void *)&txid};
2521  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, NULL, MDB_SET);
2522  if (result != 0 && result != MDB_NOTFOUND)
2523  throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str()));
2525  return result != MDB_NOTFOUND;
2526 }
2527 
2529 {
2530  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2531  check_open();
2532  mdb_txn_cursors *m_cursors = &m_wcursors;
2533 
2534  CURSOR(txpool_meta)
2535  CURSOR(txpool_blob)
2536 
2537  MDB_val k = {sizeof(txid), (void *)&txid};
2538  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, NULL, MDB_SET);
2539  if (result != 0 && result != MDB_NOTFOUND)
2540  throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta to remove: ", result).c_str()));
2541  if (!result)
2542  {
2543  result = mdb_cursor_del(m_cur_txpool_meta, 0);
2544  if (result)
2545  throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx metadata to db transaction: ", result).c_str()));
2546  }
2547  result = mdb_cursor_get(m_cur_txpool_blob, &k, NULL, MDB_SET);
2548  if (result != 0 && result != MDB_NOTFOUND)
2549  throw1(DB_ERROR(lmdb_error("Error finding txpool tx blob to remove: ", result).c_str()));
2550  if (!result)
2551  {
2552  result = mdb_cursor_del(m_cur_txpool_blob, 0);
2553  if (result)
2554  throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx blob to db transaction: ", result).c_str()));
2555  }
2556 }
2557 
2559 {
2560  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2561  check_open();
2562 
2564  RCURSOR(txpool_meta)
2565 
2566  MDB_val k = {sizeof(txid), (void *)&txid};
2567  MDB_val v;
2568  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET);
2569  if (result == MDB_NOTFOUND)
2570  return false;
2571  if (result != 0)
2572  throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str()));
2573 
2574  meta = *(const txpool_tx_meta_t*)v.mv_data;
2576  return true;
2577 }
2578 
2580 {
2581  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2582  check_open();
2583 
2585  RCURSOR(txpool_blob)
2586 
2587  MDB_val k = {sizeof(txid), (void *)&txid};
2588  MDB_val v;
2589  auto result = mdb_cursor_get(m_cur_txpool_blob, &k, &v, MDB_SET);
2590  if (result == MDB_NOTFOUND)
2591  return false;
2592  if (result != 0)
2593  throw1(DB_ERROR(lmdb_error("Error finding txpool tx blob: ", result).c_str()));
2594 
2595  bd.assign(reinterpret_cast<const char*>(v.mv_data), v.mv_size);
2597  return true;
2598 }
2599 
2601 {
2603  if (!get_txpool_tx_blob(txid, bd))
2604  throw1(DB_ERROR("Tx not found in txpool: "));
2605  return bd;
2606 }
2607 
2609 {
2610  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2611  check_open();
2612 
2614  RCURSOR(properties)
2615  MDB_val_str(k, "pruning_seed");
2616  MDB_val v;
2617  int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET);
2618  if (result == MDB_NOTFOUND)
2619  return 0;
2620  if (result)
2621  throw0(DB_ERROR(lmdb_error("Failed to retrieve pruning seed: ", result).c_str()));
2622  if (v.mv_size != sizeof(uint32_t))
2623  throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size"));
2624  uint32_t pruning_seed;
2625  memcpy(&pruning_seed, v.mv_data, sizeof(pruning_seed));
2627  return pruning_seed;
2628 }
2629 
2630 static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id)
2631 {
2632  MDB_val v;
2633  int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET);
2634  if (ret)
2635  throw0(DB_ERROR(lmdb_error("Failed to find transaction pruned data: ", ret).c_str()));
2636  if (v.mv_size == 0)
2637  throw0(DB_ERROR("Invalid transaction pruned data"));
2638  return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size});
2639 }
2640 
2642 
2643 bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
2644 {
2645  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2646  const uint32_t log_stripes = tools::get_pruning_log_stripes(pruning_seed);
2647  if (log_stripes && log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES)
2648  throw0(DB_ERROR("Pruning seed not in range"));
2649  pruning_seed = tools::get_pruning_stripe(pruning_seed);;
2650  if (pruning_seed > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES))
2651  throw0(DB_ERROR("Pruning seed not in range"));
2652  check_open();
2653 
2654  TIME_MEASURE_START(t);
2655 
2656  size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0, commit_counter = 0;
2657  uint64_t n_bytes = 0;
2658 
2659  mdb_txn_safe txn;
2660  auto result = mdb_txn_begin(m_env, NULL, 0, txn);
2661  if (result)
2662  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
2663 
2664  MDB_stat db_stats;
2665  if ((result = mdb_stat(txn, m_txs_prunable, &db_stats)))
2666  throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str()));
2667  const size_t pages0 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages;
2668 
2669  MDB_val_str(k, "pruning_seed");
2670  MDB_val v;
2671  result = mdb_get(txn, m_properties, &k, &v);
2672  bool prune_tip_table = false;
2673  if (result == MDB_NOTFOUND)
2674  {
2675  // not pruned yet
2676  if (mode != prune_mode_prune)
2677  {
2678  txn.abort();
2680  MDEBUG("Pruning not enabled, nothing to do");
2681  return true;
2682  }
2683  if (pruning_seed == 0)
2684  pruning_seed = tools::get_random_stripe();
2685  pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES);
2686  v.mv_data = &pruning_seed;
2687  v.mv_size = sizeof(pruning_seed);
2688  result = mdb_put(txn, m_properties, &k, &v, 0);
2689  if (result)
2690  throw0(DB_ERROR("Failed to save pruning seed"));
2691  prune_tip_table = false;
2692  }
2693  else if (result == 0)
2694  {
2695  // pruned already
2696  if (v.mv_size != sizeof(uint32_t))
2697  throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size"));
2698  const uint32_t data = *(const uint32_t*)v.mv_data;
2699  if (pruning_seed == 0)
2700  pruning_seed = tools::get_pruning_stripe(data);
2701  if (tools::get_pruning_stripe(data) != pruning_seed)
2702  throw0(DB_ERROR("Blockchain already pruned with different seed"));
2704  throw0(DB_ERROR("Blockchain already pruned with different base"));
2705  pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES);
2706  prune_tip_table = (mode == prune_mode_update);
2707  }
2708  else
2709  {
2710  throw0(DB_ERROR(lmdb_error("Failed to retrieve or create pruning seed: ", result).c_str()));
2711  }
2712 
2713  if (mode == prune_mode_check)
2714  MINFO("Checking blockchain pruning...");
2715  else
2716  MINFO("Pruning blockchain...");
2717 
2718  MDB_cursor *c_txs_pruned, *c_txs_prunable, *c_txs_prunable_tip;
2719  result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned);
2720  if (result)
2721  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str()));
2722  result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable);
2723  if (result)
2724  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str()));
2725  result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip);
2726  if (result)
2727  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str()));
2728  const uint64_t blockchain_height = height();
2729 
2730  if (prune_tip_table)
2731  {
2732  MDB_cursor_op op = MDB_FIRST;
2733  while (1)
2734  {
2735  int ret = mdb_cursor_get(c_txs_prunable_tip, &k, &v, op);
2736  op = MDB_NEXT;
2737  if (ret == MDB_NOTFOUND)
2738  break;
2739  if (ret)
2740  throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str()));
2741 
2742  uint64_t block_height;
2743  memcpy(&block_height, v.mv_data, sizeof(block_height));
2744  if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS < blockchain_height)
2745  {
2746  ++n_total_records;
2747  if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &k))
2748  {
2749  ++n_prunable_records;
2750  result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET);
2751  if (result == MDB_NOTFOUND)
2752  MWARNING("Already pruned at height " << block_height << "/" << blockchain_height);
2753  else if (result)
2754  throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str()));
2755  else
2756  {
2757  MDEBUG("Pruning at height " << block_height << "/" << blockchain_height);
2758  ++n_pruned_records;
2759  ++commit_counter;
2760  n_bytes += k.mv_size + v.mv_size;
2761  result = mdb_cursor_del(c_txs_prunable, 0);
2762  if (result)
2763  throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str()));
2764  }
2765  }
2766  result = mdb_cursor_del(c_txs_prunable_tip, 0);
2767  if (result)
2768  throw0(DB_ERROR(lmdb_error("Failed to delete transaction tip data: ", result).c_str()));
2769 
2770  if (mode != prune_mode_check && commit_counter >= 4096)
2771  {
2772  MDEBUG("Committing txn at checkpoint...");
2773  txn.commit();
2774  result = mdb_txn_begin(m_env, NULL, 0, txn);
2775  if (result)
2776  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
2777  result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned);
2778  if (result)
2779  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str()));
2780  result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable);
2781  if (result)
2782  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str()));
2783  result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip);
2784  if (result)
2785  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str()));
2786  commit_counter = 0;
2787  }
2788  }
2789  }
2790  }
2791  else
2792  {
2793  MDB_cursor *c_tx_indices;
2794  result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices);
2795  if (result)
2796  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str()));
2797  MDB_cursor_op op = MDB_FIRST;
2798  while (1)
2799  {
2800  int ret = mdb_cursor_get(c_tx_indices, &k, &v, op);
2801  op = MDB_NEXT;
2802  if (ret == MDB_NOTFOUND)
2803  break;
2804  if (ret)
2805  throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str()));
2806 
2807  ++n_total_records;
2808  //const txindex *ti = (const txindex *)v.mv_data;
2809  txindex ti;
2810  memcpy(&ti, v.mv_data, sizeof(ti));
2811  const uint64_t block_height = ti.data.block_id;
2812  if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height)
2813  {
2814  MDB_val_set(kp, ti.data.tx_id);
2815  MDB_val_set(vp, block_height);
2816  if (mode == prune_mode_check)
2817  {
2818  result = mdb_cursor_get(c_txs_prunable_tip, &kp, &vp, MDB_SET);
2819  if (result && result != MDB_NOTFOUND)
2820  throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str()));
2821  if (result == MDB_NOTFOUND)
2822  MERROR("Transaction not found in prunable tip table for height " << block_height << "/" << blockchain_height <<
2823  ", seed " << epee::string_tools::to_string_hex(pruning_seed));
2824  }
2825  else
2826  {
2827  result = mdb_cursor_put(c_txs_prunable_tip, &kp, &vp, 0);
2828  if (result && result != MDB_NOTFOUND)
2829  throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str()));
2830  }
2831  }
2832  MDB_val_set(kp, ti.data.tx_id);
2833  if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &kp))
2834  {
2835  result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET);
2836  if (result && result != MDB_NOTFOUND)
2837  throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str()));
2838  if (mode == prune_mode_check)
2839  {
2840  if (result != MDB_NOTFOUND)
2841  MERROR("Prunable data found for pruned height " << block_height << "/" << blockchain_height <<
2842  ", seed " << epee::string_tools::to_string_hex(pruning_seed));
2843  }
2844  else
2845  {
2846  ++n_prunable_records;
2847  if (result == MDB_NOTFOUND)
2848  MWARNING("Already pruned at height " << block_height << "/" << blockchain_height);
2849  else
2850  {
2851  MDEBUG("Pruning at height " << block_height << "/" << blockchain_height);
2852  ++n_pruned_records;
2853  n_bytes += kp.mv_size + v.mv_size;
2854  result = mdb_cursor_del(c_txs_prunable, 0);
2855  if (result)
2856  throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str()));
2857  ++commit_counter;
2858  }
2859  }
2860  }
2861  else
2862  {
2863  if (mode == prune_mode_check)
2864  {
2865  MDB_val_set(kp, ti.data.tx_id);
2866  result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET);
2867  if (result && result != MDB_NOTFOUND)
2868  throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str()));
2869  if (result == MDB_NOTFOUND)
2870  MERROR("Prunable data not found for unpruned height " << block_height << "/" << blockchain_height <<
2871  ", seed " << epee::string_tools::to_string_hex(pruning_seed));
2872  }
2873  }
2874 
2875  if (mode != prune_mode_check && commit_counter >= 4096)
2876  {
2877  MDEBUG("Committing txn at checkpoint...");
2878  txn.commit();
2879  result = mdb_txn_begin(m_env, NULL, 0, txn);
2880  if (result)
2881  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
2882  result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned);
2883  if (result)
2884  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str()));
2885  result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable);
2886  if (result)
2887  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str()));
2888  result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip);
2889  if (result)
2890  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str()));
2891  result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices);
2892  if (result)
2893  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str()));
2894  MDB_val val;
2895  val.mv_size = sizeof(ti);
2896  val.mv_data = (void *)&ti;
2897  result = mdb_cursor_get(c_tx_indices, (MDB_val*)&zerokval, &val, MDB_GET_BOTH);
2898  if (result)
2899  throw0(DB_ERROR(lmdb_error("Failed to restore cursor for tx_indices: ", result).c_str()));
2900  commit_counter = 0;
2901  }
2902  }
2903  mdb_cursor_close(c_tx_indices);
2904  }
2905 
2906  if ((result = mdb_stat(txn, m_txs_prunable, &db_stats)))
2907  throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str()));
2908  const size_t pages1 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages;
2909  const size_t db_bytes = (pages0 - pages1) * db_stats.ms_psize;
2910 
2911  mdb_cursor_close(c_txs_prunable_tip);
2912  mdb_cursor_close(c_txs_prunable);
2913  mdb_cursor_close(c_txs_pruned);
2914 
2915  txn.commit();
2916 
2918 
2919  MINFO((mode == prune_mode_check ? "Checked" : "Pruned") << " blockchain in " <<
2920  t << " ms: " << (n_bytes/1024.0f/1024.0f) << " MB (" << db_bytes/1024.0f/1024.0f << " MB) pruned in " <<
2921  n_pruned_records << " records (" << pages0 - pages1 << "/" << pages0 << " " << db_stats.ms_psize << " byte pages), " <<
2922  n_prunable_records << "/" << n_total_records << " pruned records");
2923  return true;
2924 }
2925 
2927 {
2928  return prune_worker(prune_mode_prune, pruning_seed);
2929 }
2930 
2932 {
2933  return prune_worker(prune_mode_update, 0);
2934 }
2935 
2937 {
2938  return prune_worker(prune_mode_check, 0);
2939 }
2940 
2941 bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, bool include_unrelayed_txes) const
2942 {
2943  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2944  check_open();
2945 
2947  RCURSOR(txpool_meta);
2948  RCURSOR(txpool_blob);
2949 
2950  MDB_val k;
2951  MDB_val v;
2952  bool ret = true;
2953 
2954  MDB_cursor_op op = MDB_FIRST;
2955  while (1)
2956  {
2957  int result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, op);
2958  op = MDB_NEXT;
2959  if (result == MDB_NOTFOUND)
2960  break;
2961  if (result)
2962  throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str()));
2963  const crypto::hash txid = *(const crypto::hash*)k.mv_data;
2964  const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data;
2965  if (!include_unrelayed_txes && meta.do_not_relay)
2966  // Skipping that tx
2967  continue;
2968  const cryptonote::blobdata *passed_bd = NULL;
2970  if (include_blob)
2971  {
2972  MDB_val b;
2973  result = mdb_cursor_get(m_cur_txpool_blob, &k, &b, MDB_SET);
2974  if (result == MDB_NOTFOUND)
2975  throw0(DB_ERROR("Failed to find txpool tx blob to match metadata"));
2976  if (result)
2977  throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx blob: ", result).c_str()));
2978  bd.assign(reinterpret_cast<const char*>(b.mv_data), b.mv_size);
2979  passed_bd = &bd;
2980  }
2981 
2982  if (!f(txid, meta, passed_bd)) {
2983  ret = false;
2984  break;
2985  }
2986  }
2987 
2989 
2990  return ret;
2991 }
2992 
2994 {
2995  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
2996  check_open();
2997 
2999  RCURSOR(block_heights);
3000 
3001  bool ret = false;
3002  MDB_val_set(key, h);
3003  auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH);
3004  if (get_result == MDB_NOTFOUND)
3005  {
3006  LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
3007  }
3008  else if (get_result)
3009  throw0(DB_ERROR(lmdb_error("DB error attempting to fetch block index from hash", get_result).c_str()));
3010  else
3011  {
3012  if (height)
3013  {
3014  const blk_height *bhp = (const blk_height *)key.mv_data;
3015  *height = bhp->bh_height;
3016  }
3017  ret = true;
3018  }
3019 
3021  return ret;
3022 }
3023 
3025 {
3026  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3027  check_open();
3028 
3030 }
3031 
3033 {
3034  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3035  check_open();
3036 
3038  RCURSOR(block_heights);
3039 
3040  MDB_val_set(key, h);
3041  auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH);
3042  if (get_result == MDB_NOTFOUND)
3043  throw1(BLOCK_DNE("Attempted to retrieve non-existent block height"));
3044  else if (get_result)
3045  throw0(DB_ERROR("Error attempting to retrieve a block height from the db"));
3046 
3047  blk_height *bhp = (blk_height *)key.mv_data;
3048  uint64_t ret = bhp->bh_height;
3050  return ret;
3051 }
3052 
3054 {
3055  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3056  check_open();
3057 
3058  // block_header object is automatically cast from block object
3059  return get_block(h);
3060 }
3061 
3063 {
3064  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3065  check_open();
3066 
3068  RCURSOR(blocks);
3069 
3070  MDB_val_copy<uint64_t> key(height);
3071  MDB_val result;
3072  auto get_result = mdb_cursor_get(m_cur_blocks, &key, &result, MDB_SET);
3073  if (get_result == MDB_NOTFOUND)
3074  {
3075  throw0(BLOCK_DNE(std::string("Attempt to get block from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block not in db").c_str()));
3076  }
3077  else if (get_result)
3078  throw0(DB_ERROR("Error attempting to retrieve a block from the db"));
3079 
3080  blobdata bd;
3081  bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size);
3082 
3084 
3085  return bd;
3086 }
3087 
3089 {
3090  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3091  check_open();
3092 
3094  RCURSOR(block_info);
3095 
3096  MDB_val_set(result, height);
3097  auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH);
3098  if (get_result == MDB_NOTFOUND)
3099  {
3100  throw0(BLOCK_DNE(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str()));
3101  }
3102  else if (get_result)
3103  throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db"));
3104 
3105  mdb_block_info *bi = (mdb_block_info *)result.mv_data;
3106  uint64_t ret = bi->bi_timestamp;
3108  return ret;
3109 }
3110 
3111 std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const
3112 {
3113  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3114  check_open();
3115  std::vector<uint64_t> res;
3116  int result;
3117 
3118  if (heights.empty())
3119  return {};
3120  res.reserve(heights.size());
3121 
3123  RCURSOR(block_info);
3124 
3125  MDB_stat db_stats;
3126  if ((result = mdb_stat(m_txn, m_blocks, &db_stats)))
3127  throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
3128  for (size_t i = 0; i < heights.size(); ++i)
3129  if (heights[i] >= db_stats.ms_entries)
3130  throw0(BLOCK_DNE(std::string("Attempt to get rct distribution from height " + std::to_string(heights[i]) + " failed -- block size not in db").c_str()));
3131 
3132  MDB_val v;
3133 
3134  uint64_t prev_height = heights[0];
3135  uint64_t range_begin = 0, range_end = 0;
3136  for (uint64_t height: heights)
3137  {
3138  if (height >= range_begin && height < range_end)
3139  {
3140  // nohting to do
3141  }
3142  else
3143  {
3144  if (height == prev_height + 1)
3145  {
3146  MDB_val k2;
3148  range_begin = ((const mdb_block_info*)v.mv_data)->bi_height;
3149  range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please
3150  if (height < range_begin || height >= range_end)
3151  throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multuple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str()));
3152  }
3153  else
3154  {
3155  v.mv_size = sizeof(uint64_t);
3156  v.mv_data = (void*)&height;
3157  result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
3158  range_begin = height;
3159  range_end = range_begin + 1;
3160  }
3161  if (result)
3162  throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str()));
3163  }
3164  const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin);
3165  res.push_back(bi->bi_cum_rct);
3166  prev_height = height;
3167  }
3168 
3170  return res;
3171 }
3172 
3174 {
3175  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3176  check_open();
3177  uint64_t m_height = height();
3178 
3179  // if no blocks, return 0
3180  if (m_height == 0)
3181  {
3182  return 0;
3183  }
3184 
3185  return get_block_timestamp(m_height - 1);
3186 }
3187 
3189 {
3190  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3191  check_open();
3192 
3194  RCURSOR(block_info);
3195 
3196  MDB_val_set(result, height);
3197  auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH);
3198  if (get_result == MDB_NOTFOUND)
3199  {
3200  throw0(BLOCK_DNE(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str()));
3201  }
3202  else if (get_result)
3203  throw0(DB_ERROR("Error attempting to retrieve a block size from the db"));
3204 
3205  mdb_block_info *bi = (mdb_block_info *)result.mv_data;
3206  size_t ret = bi->bi_weight;
3208  return ret;
3209 }
3210 
3211 std::vector<uint64_t> BlockchainLMDB::get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const
3212 {
3213  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3214  check_open();
3215 
3217  RCURSOR(block_info);
3218 
3219  const uint64_t h = height();
3220  if (start_height >= h)
3221  throw0(DB_ERROR(("Height " + std::to_string(start_height) + " not in blockchain").c_str()));
3222 
3223  std::vector<uint64_t> ret;
3224  ret.reserve(count);
3225 
3226  MDB_val v;
3227  uint64_t range_begin = 0, range_end = 0;
3228  for (uint64_t height = start_height; height < h && count--; ++height)
3229  {
3230  if (height >= range_begin && height < range_end)
3231  {
3232  // nothing to do
3233  }
3234  else
3235  {
3236  int result = 0;
3237  if (range_end > 0)
3238  {
3239  MDB_val k2;
3241  range_begin = ((const mdb_block_info*)v.mv_data)->bi_height;
3242  range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please
3243  if (height < range_begin || height >= range_end)
3244  throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multiple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str()));
3245  }
3246  else
3247  {
3248  v.mv_size = sizeof(uint64_t);
3249  v.mv_data = (void*)&height;
3250  result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
3251  range_begin = height;
3252  range_end = range_begin + 1;
3253  }
3254  if (result)
3255  throw0(DB_ERROR(lmdb_error("Error attempting to retrieve block_info from the db: ", result).c_str()));
3256  }
3257  const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin);
3258  ret.push_back(*(const uint64_t*)(((const char*)bi) + offset));
3259  }
3260 
3262  return ret;
3263 }
3264 
3265 uint64_t BlockchainLMDB::get_max_block_size()
3266 {
3267  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3268  check_open();
3269 
3271  RCURSOR(properties)
3272  MDB_val_str(k, "max_block_size");
3273  MDB_val v;
3274  int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET);
3275  if (result == MDB_NOTFOUND)
3276  return std::numeric_limits<uint64_t>::max();
3277  if (result)
3278  throw0(DB_ERROR(lmdb_error("Failed to retrieve max block size: ", result).c_str()));
3279  if (v.mv_size != sizeof(uint64_t))
3280  throw0(DB_ERROR("Failed to retrieve or create max block size: unexpected value size"));
3281  uint64_t max_block_size;
3282  memcpy(&max_block_size, v.mv_data, sizeof(max_block_size));
3284  return max_block_size;
3285 }
3286 
3287 void BlockchainLMDB::add_max_block_size(uint64_t sz)
3288 {
3289  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3290  check_open();
3291  mdb_txn_cursors *m_cursors = &m_wcursors;
3292 
3293  CURSOR(properties)
3294 
3295  MDB_val_str(k, "max_block_size");
3296  MDB_val v;
3297  int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET);
3298  if (result && result != MDB_NOTFOUND)
3299  throw0(DB_ERROR(lmdb_error("Failed to retrieve max block size: ", result).c_str()));
3300  uint64_t max_block_size = 0;
3301  if (result == 0)
3302  {
3303  if (v.mv_size != sizeof(uint64_t))
3304  throw0(DB_ERROR("Failed to retrieve or create max block size: unexpected value size"));
3305  memcpy(&max_block_size, v.mv_data, sizeof(max_block_size));
3306  }
3307  if (sz > max_block_size)
3308  max_block_size = sz;
3309  v.mv_data = (void*)&max_block_size;
3310  v.mv_size = sizeof(max_block_size);
3311  result = mdb_cursor_put(m_cur_properties, &k, &v, 0);
3312  if (result)
3313  throw0(DB_ERROR(lmdb_error("Failed to set max_block_size: ", result).c_str()));
3314 }
3315 
3316 
3317 std::vector<uint64_t> BlockchainLMDB::get_block_weights(uint64_t start_height, size_t count) const
3318 {
3319  return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_weight));
3320 }
3321 
3322 std::vector<uint64_t> BlockchainLMDB::get_long_term_block_weights(uint64_t start_height, size_t count) const
3323 {
3324  return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_long_term_block_weight));
3325 }
3326 
3328 {
3329  LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height);
3330  check_open();
3331  mdb_txn_cursors *m_cursors = &m_wcursors;
3332 
3333  int result;
3334 
3335  CURSOR(block_info)
3336 
3337  MDB_val_set(val_bi, height);
3338  result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &val_bi, MDB_GET_BOTH);
3339  if (result == MDB_NOTFOUND)
3340  {
3341  throw0(BLOCK_DNE(std::string("Attempt to set cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str()));
3342  }
3343  else if (result)
3344  throw0(DB_ERROR("Error attempting to set a cumulative difficulty"));
3345 
3346  mdb_block_info *result_bi = (mdb_block_info *)val_bi.mv_data;
3347 
3348  mdb_block_info bi;
3349  bi.bi_height = result_bi->bi_height;
3350  bi.bi_timestamp = result_bi->bi_timestamp;
3351  bi.bi_coins = result_bi->bi_coins;
3352  bi.bi_weight = result_bi->bi_weight;
3353  //bi.bi_diff_lo = diff; // TODO
3354  bi.bi_hash = result_bi->bi_hash;
3355 
3356  MDB_val_set(val, bi);
3357  result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&val_bi, &val, MDB_CURRENT);
3358  if (result)
3359  throw0(DB_ERROR(lmdb_error("Failed to set cumulative difficulty to db transaction: ", result).c_str()));
3360 
3361 }
3362 
3364 {
3365  LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height);
3366  check_open();
3367 
3369  RCURSOR(block_info);
3370 
3371  MDB_val_set(result, height);
3372  auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH);
3373  if (get_result == MDB_NOTFOUND)
3374  {
3375  throw0(BLOCK_DNE(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str()));
3376  }
3377  else if (get_result)
3378  throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db"));
3379 
3380  mdb_block_info *bi = (mdb_block_info *)result.mv_data;
3381  difficulty_type ret = bi->bi_diff_hi;
3382  ret <<= 64;
3383  ret |= bi->bi_diff_lo;
3385  return ret;
3386 }
3387 
3389 {
3390  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3391  check_open();
3392 
3393  difficulty_type diff1 = 0;
3394  difficulty_type diff2 = 0;
3395 
3397  if (height != 0)
3398  {
3400  }
3401 
3402  return diff1 - diff2;
3403 }
3404 
3406 {
3407  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3408  check_open();
3409 
3411  RCURSOR(block_info);
3412 
3413  MDB_val_set(result, height);
3414  auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH);
3415  if (get_result == MDB_NOTFOUND)
3416  {
3417  throw0(BLOCK_DNE(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str()));
3418  }
3419  else if (get_result)
3420  throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db"));
3421 
3422  mdb_block_info *bi = (mdb_block_info *)result.mv_data;
3423  uint64_t ret = bi->bi_coins;
3425  return ret;
3426 }
3427 
3429 {
3430  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3431  check_open();
3432 
3434  RCURSOR(block_info);
3435 
3436  MDB_val_set(result, height);
3437  auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH);
3438  if (get_result == MDB_NOTFOUND)
3439  {
3440  throw0(BLOCK_DNE(std::string("Attempt to get block long term weight from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block info not in db").c_str()));
3441  }
3442  else if (get_result)
3443  throw0(DB_ERROR("Error attempting to retrieve a long term block weight from the db"));
3444 
3445  mdb_block_info *bi = (mdb_block_info *)result.mv_data;
3448  return ret;
3449 }
3450 
3452 {
3453  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3454  check_open();
3455 
3457  RCURSOR(block_info);
3458 
3459  MDB_val_set(result, height);
3460  auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH);
3461  if (get_result == MDB_NOTFOUND)
3462  {
3463  throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str()));
3464  }
3465  else if (get_result)
3466  throw0(DB_ERROR(lmdb_error("Error attempting to retrieve a block hash from the db: ", get_result).c_str()));
3467 
3468  mdb_block_info *bi = (mdb_block_info *)result.mv_data;
3469  crypto::hash ret = bi->bi_hash;
3471  return ret;
3472 }
3473 
3474 std::vector<block> BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const
3475 {
3476  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3477  check_open();
3478  std::vector<block> v;
3479 
3480  for (uint64_t height = h1; height <= h2; ++height)
3481  {
3482  v.push_back(get_block_from_height(height));
3483  }
3484 
3485  return v;
3486 }
3487 
3488 std::vector<crypto::hash> BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const
3489 {
3490  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3491  check_open();
3492  std::vector<crypto::hash> v;
3493 
3494  for (uint64_t height = h1; height <= h2; ++height)
3495  {
3496  v.push_back(get_block_hash_from_height(height));
3497  }
3498 
3499  return v;
3500 }
3501 
3503 {
3504  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3505  check_open();
3506  uint64_t m_height = height();
3507  if (block_height)
3508  *block_height = m_height - 1;
3509  if (m_height != 0)
3510  {
3511  return get_block_hash_from_height(m_height - 1);
3512  }
3513 
3514  return null_hash;
3515 }
3516 
3518 {
3519  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3520  check_open();
3521  uint64_t m_height = height();
3522 
3523  if (m_height != 0)
3524  {
3525  return get_block_from_height(m_height - 1);
3526  }
3527 
3528  block b;
3529  return b;
3530 }
3531 
3533 {
3534  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3535  check_open();
3537  int result;
3538 
3539  // get current height
3540  MDB_stat db_stats;
3541  if ((result = mdb_stat(m_txn, m_blocks, &db_stats)))
3542  throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
3543  return db_stats.ms_entries;
3544 }
3545 
3546 uint64_t BlockchainLMDB::num_outputs() const
3547 {
3548  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3549  check_open();
3551  int result;
3552 
3553  RCURSOR(output_txs)
3554 
3555  uint64_t num = 0;
3556  MDB_val k, v;
3557  result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_LAST);
3558  if (result == MDB_NOTFOUND)
3559  num = 0;
3560  else if (result == 0)
3561  num = 1 + ((const outtx*)v.mv_data)->output_id;
3562  else
3563  throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str()));
3564 
3565  return num;
3566 }
3567 
3569 {
3570  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3571  check_open();
3572 
3574  RCURSOR(tx_indices);
3575 
3576  MDB_val_set(key, h);
3577  bool tx_found = false;
3578 
3579  TIME_MEASURE_START(time1);
3580  auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &key, MDB_GET_BOTH);
3581  if (get_result == 0)
3582  tx_found = true;
3583  else if (get_result != MDB_NOTFOUND)
3584  throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", get_result).c_str()));
3585 
3586  TIME_MEASURE_FINISH(time1);
3587  time_tx_exists += time1;
3588 
3590 
3591  if (! tx_found)
3592  {
3593  LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
3594  return false;
3595  }
3596 
3597  return true;
3598 }
3599 
3601 {
3602  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3603  check_open();
3604 
3606  RCURSOR(tx_indices);
3607 
3608  MDB_val_set(v, h);
3609 
3610  TIME_MEASURE_START(time1);
3611  auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
3612  TIME_MEASURE_FINISH(time1);
3613  time_tx_exists += time1;
3614  if (!get_result) {
3615  txindex *tip = (txindex *)v.mv_data;
3616  tx_id = tip->data.tx_id;
3617  }
3618 
3620 
3621  bool ret = false;
3622  if (get_result == MDB_NOTFOUND)
3623  {
3624  LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
3625  }
3626  else if (get_result)
3627  throw0(DB_ERROR(lmdb_error("DB error attempting to fetch transaction from hash", get_result).c_str()));
3628  else
3629  ret = true;
3630 
3631  return ret;
3632 }
3633 
3635 {
3636  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3637  check_open();
3638 
3640  RCURSOR(tx_indices);
3641 
3642  MDB_val_set(v, h);
3643  auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
3644  if (get_result == MDB_NOTFOUND)
3645  throw1(TX_DNE(lmdb_error(std::string("tx data with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str()));
3646  else if (get_result)
3647  throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx data from hash: ", get_result).c_str()));
3648 
3649  txindex *tip = (txindex *)v.mv_data;
3650  uint64_t ret = tip->data.unlock_time;
3652  return ret;
3653 }
3654 
3656 {
3657  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3658  check_open();
3659 
3661  RCURSOR(tx_indices);
3662  RCURSOR(txs_pruned);
3663  RCURSOR(txs_prunable);
3664 
3665  MDB_val_set(v, h);
3666  MDB_val result0, result1;
3667  auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
3668  if (get_result == 0)
3669  {
3670  txindex *tip = (txindex *)v.mv_data;
3671  MDB_val_set(val_tx_id, tip->data.tx_id);
3672  get_result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &result0, MDB_SET);
3673  if (get_result == 0)
3674  {
3675  get_result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &result1, MDB_SET);
3676  }
3677  }
3678  if (get_result == MDB_NOTFOUND)
3679  return false;
3680  else if (get_result)
3681  throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str()));
3682 
3683  bd.assign(reinterpret_cast<char*>(result0.mv_data), result0.mv_size);
3684  bd.append(reinterpret_cast<char*>(result1.mv_data), result1.mv_size);
3685 
3687 
3688  return true;
3689 }
3690 
3692 {
3693  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3694  check_open();
3695 
3697  RCURSOR(tx_indices);
3698  RCURSOR(txs_pruned);
3699 
3700  MDB_val_set(v, h);
3701  MDB_val result;
3702  auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
3703  if (get_result == 0)
3704  {
3705  txindex *tip = (txindex *)v.mv_data;
3706  MDB_val_set(val_tx_id, tip->data.tx_id);
3707  get_result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &result, MDB_SET);
3708  }
3709  if (get_result == MDB_NOTFOUND)
3710  return false;
3711  else if (get_result)
3712  throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str()));
3713 
3714  bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size);
3715 
3717 
3718  return true;
3719 }
3720 
3722 {
3723  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3724  check_open();
3725 
3727  RCURSOR(tx_indices);
3728  RCURSOR(txs_prunable);
3729 
3730  MDB_val_set(v, h);
3731  MDB_val result;
3732  auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
3733  if (get_result == 0)
3734  {
3735  const txindex *tip = (const txindex *)v.mv_data;
3736  MDB_val_set(val_tx_id, tip->data.tx_id);
3737  get_result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &result, MDB_SET);
3738  }
3739  if (get_result == MDB_NOTFOUND)
3740  return false;
3741  else if (get_result)
3742  throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str()));
3743 
3744  bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size);
3745 
3747 
3748  return true;
3749 }
3750 
3751 bool BlockchainLMDB::get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const
3752 {
3753  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3754  check_open();
3755 
3757  RCURSOR(tx_indices);
3758  RCURSOR(txs_prunable_hash);
3759 
3760  MDB_val_set(v, tx_hash);
3761  MDB_val result, val_tx_prunable_hash;
3762  auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
3763  if (get_result == 0)
3764  {
3765  txindex *tip = (txindex *)v.mv_data;
3766  MDB_val_set(val_tx_id, tip->data.tx_id);
3767  get_result = mdb_cursor_get(m_cur_txs_prunable_hash, &val_tx_id, &result, MDB_SET);
3768  }
3769  if (get_result == MDB_NOTFOUND)
3770  return false;
3771  else if (get_result)
3772  throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx prunable hash from tx hash", get_result).c_str()));
3773 
3774  prunable_hash = *(const crypto::hash*)result.mv_data;
3775 
3777 
3778  return true;
3779 }
3780 
3782 {
3783  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3784  check_open();
3785 
3787  int result;
3788 
3789  MDB_stat db_stats;
3790  if ((result = mdb_stat(m_txn, m_txs_pruned, &db_stats)))
3791  throw0(DB_ERROR(lmdb_error("Failed to query m_txs_pruned: ", result).c_str()));
3792 
3794 
3795  return db_stats.ms_entries;
3796 }
3797 
3798 std::vector<transaction> BlockchainLMDB::get_tx_list(const std::vector<crypto::hash>& hlist) const
3799 {
3800  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3801  check_open();
3802  std::vector<transaction> v;
3803 
3804  for (auto& h : hlist)
3805  {
3806  v.push_back(get_tx(h));
3807  }
3808 
3809  return v;
3810 }
3811 
3813 {
3814  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3815  check_open();
3816 
3818  RCURSOR(tx_indices);
3819 
3820  MDB_val_set(v, h);
3821  auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
3822  if (get_result == MDB_NOTFOUND)
3823  {
3824  throw1(TX_DNE(std::string("tx_data_t with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
3825  }
3826  else if (get_result)
3827  throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx height from hash", get_result).c_str()));
3828 
3829  txindex *tip = (txindex *)v.mv_data;
3830  uint64_t ret = tip->data.block_id;
3832  return ret;
3833 }
3834 
3836 {
3837  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3838  check_open();
3839 
3841  RCURSOR(output_amounts);
3842 
3843  MDB_val_copy<uint64_t> k(amount);
3844  MDB_val v;
3845  mdb_size_t num_elems = 0;
3846  auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET);
3847  if (result == MDB_SUCCESS)
3848  {
3850  }
3851  else if (result != MDB_NOTFOUND)
3852  throw0(DB_ERROR("DB error attempting to get number of outputs of an amount"));
3853 
3855 
3856  return num_elems;
3857 }
3858 
3859 output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const
3860 {
3861  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3862  check_open();
3863 
3865  RCURSOR(output_amounts);
3866 
3867  MDB_val_set(k, amount);
3868  MDB_val_set(v, index);
3869  auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH);
3870  if (get_result == MDB_NOTFOUND)
3871  throw1(OUTPUT_DNE(std::string("Attempting to get output pubkey by index, but key does not exist: amount " +
3872  std::to_string(amount) + ", index " + std::to_string(index)).c_str()));
3873  else if (get_result)
3874  throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db"));
3875 
3876  output_data_t ret;
3877  if (amount == 0)
3878  {
3879  const outkey *okp = (const outkey *)v.mv_data;
3880  ret = okp->data;
3881  }
3882  else
3883  {
3884  const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data;
3885  memcpy(&ret, &okp->data, sizeof(pre_rct_output_data_t));;
3886  if (include_commitmemt)
3887  ret.commitment = rct::zeroCommit(amount);
3888  }
3890  return ret;
3891 }
3892 
3894 {
3895  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3896  check_open();
3897 
3899  RCURSOR(output_txs);
3900 
3901  MDB_val_set(v, output_id);
3902 
3903  auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
3904  if (get_result == MDB_NOTFOUND)
3905  throw1(OUTPUT_DNE("output with given index not in db"));
3906  else if (get_result)
3907  throw0(DB_ERROR("DB error attempting to fetch output tx hash"));
3908 
3909  outtx *ot = (outtx *)v.mv_data;
3910  tx_out_index ret = tx_out_index(ot->tx_hash, ot->local_index);
3911 
3913  return ret;
3914 }
3915 
3917 {
3918  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3919  std::vector < uint64_t > offsets;
3920  std::vector<tx_out_index> indices;
3921  offsets.push_back(index);
3922  get_output_tx_and_index(amount, offsets, indices);
3923  if (!indices.size())
3924  throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
3925 
3926  return indices[0];
3927 }
3928 
3929 std::vector<std::vector<uint64_t>> BlockchainLMDB::get_tx_amount_output_indices(uint64_t tx_id, size_t n_txes) const
3930 {
3931  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3932 
3933  check_open();
3934 
3936  RCURSOR(tx_outputs);
3937 
3938  MDB_val_set(k_tx_id, tx_id);
3939  MDB_val v;
3940  std::vector<std::vector<uint64_t>> amount_output_indices_set;
3941  amount_output_indices_set.reserve(n_txes);
3942 
3943  MDB_cursor_op op = MDB_SET;
3944  while (n_txes-- > 0)
3945  {
3946  int result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, op);
3947  if (result == MDB_NOTFOUND)
3948  LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in "
3949  "tx_outputs, but it should have an empty entry even if it's a tx without "
3950  "outputs");
3951  else if (result)
3952  throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str()));
3953 
3954  op = MDB_NEXT;
3955 
3956  const uint64_t* indices = (const uint64_t*)v.mv_data;
3957  size_t num_outputs = v.mv_size / sizeof(uint64_t);
3958 
3959  amount_output_indices_set.resize(amount_output_indices_set.size() + 1);
3960  std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.back();
3961  amount_output_indices.reserve(num_outputs);
3962  for (size_t i = 0; i < num_outputs; ++i)
3963  {
3964  amount_output_indices.push_back(indices[i]);
3965  }
3966  }
3967 
3969  return amount_output_indices_set;
3970 }
3971 
3973 {
3974  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3975  check_open();
3976 
3977  bool ret;
3978 
3980  RCURSOR(spent_keys);
3981 
3982  MDB_val k = {sizeof(img), (void *)&img};
3983  ret = (mdb_cursor_get(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_GET_BOTH) == 0);
3984 
3986  return ret;
3987 }
3988 
3989 bool BlockchainLMDB::for_all_key_images(std::function<bool(const crypto::key_image&)> f) const
3990 {
3991  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
3992  check_open();
3993 
3995  RCURSOR(spent_keys);
3996 
3997  MDB_val k, v;
3998  bool fret = true;
3999 
4000  k = zerokval;
4001  MDB_cursor_op op = MDB_FIRST;
4002  while (1)
4003  {
4004  int ret = mdb_cursor_get(m_cur_spent_keys, &k, &v, op);
4005  op = MDB_NEXT;
4006  if (ret == MDB_NOTFOUND)
4007  break;
4008  if (ret < 0)
4009  throw0(DB_ERROR("Failed to enumerate key images"));
4010  const crypto::key_image k_image = *(const crypto::key_image*)v.mv_data;
4011  if (!f(k_image)) {
4012  fret = false;
4013  break;
4014  }
4015  }
4016 
4018 
4019  return fret;
4020 }
4021 
4022 bool BlockchainLMDB::for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)> f) const
4023 {
4024  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4025  check_open();
4026 
4028  RCURSOR(blocks);
4029 
4030  MDB_val k;
4031  MDB_val v;
4032  bool fret = true;
4033 
4034  MDB_cursor_op op;
4035  if (h1)
4036  {
4037  k = MDB_val{sizeof(h1), (void*)&h1};
4038  op = MDB_SET;
4039  } else
4040  {
4041  op = MDB_FIRST;
4042  }
4043  while (1)
4044  {
4045  int ret = mdb_cursor_get(m_cur_blocks, &k, &v, op);
4046  op = MDB_NEXT;
4047  if (ret == MDB_NOTFOUND)
4048  break;
4049  if (ret)
4050  throw0(DB_ERROR("Failed to enumerate blocks"));
4051  uint64_t height = *(const uint64_t*)k.mv_data;
4052  blobdata bd;
4053  bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
4054  block b;
4056  throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
4058  if (!get_block_hash(b, hash))
4059  throw0(DB_ERROR("Failed to get block hash from blob retrieved from the db"));
4060  if (!f(height, hash, b)) {
4061  fret = false;
4062  break;
4063  }
4064  if (height >= h2)
4065  break;
4066  }
4067 
4069 
4070  return fret;
4071 }
4072 
4073 bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f, bool pruned) const
4074 {
4075  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4076  check_open();
4077 
4079  RCURSOR(txs_pruned);
4080  RCURSOR(txs_prunable);
4081  RCURSOR(tx_indices);
4082 
4083  MDB_val k;
4084  MDB_val v;
4085  bool fret = true;
4086 
4087  MDB_cursor_op op = MDB_FIRST;
4088  while (1)
4089  {
4090  int ret = mdb_cursor_get(m_cur_tx_indices, &k, &v, op);
4091  op = MDB_NEXT;
4092  if (ret == MDB_NOTFOUND)
4093  break;
4094  if (ret)
4095  throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str()));
4096 
4097  txindex *ti = (txindex *)v.mv_data;
4098  const crypto::hash hash = ti->key;
4099  k.mv_data = (void *)&ti->data.tx_id;
4100  k.mv_size = sizeof(ti->data.tx_id);
4101 
4102  ret = mdb_cursor_get(m_cur_txs_pruned, &k, &v, MDB_SET);
4103  if (ret == MDB_NOTFOUND)
4104  break;
4105  if (ret)
4106  throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str()));
4107  transaction tx;
4108  blobdata bd;
4109  bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
4110  if (pruned)
4111  {
4113  throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
4114  }
4115  else
4116  {
4117  ret = mdb_cursor_get(m_cur_txs_prunable, &k, &v, MDB_SET);
4118  if (ret)
4119  throw0(DB_ERROR(lmdb_error("Failed to get prunable tx data the db: ", ret).c_str()));
4120  bd.append(reinterpret_cast<char*>(v.mv_data), v.mv_size);
4121  if (!parse_and_validate_tx_from_blob(bd, tx))
4122  throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
4123  }
4124  if (!f(hash, tx)) {
4125  fret = false;
4126  break;
4127  }
4128  }
4129 
4131 
4132  return fret;
4133 }
4134 
4135 bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const
4136 {
4137  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4138  check_open();
4139 
4141  RCURSOR(output_amounts);
4142 
4143  MDB_val k;
4144  MDB_val v;
4145  bool fret = true;
4146 
4147  MDB_cursor_op op = MDB_FIRST;
4148  while (1)
4149  {
4150  int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op);
4151  op = MDB_NEXT;
4152  if (ret == MDB_NOTFOUND)
4153  break;
4154  if (ret)
4155  throw0(DB_ERROR("Failed to enumerate outputs"));
4156  uint64_t amount = *(const uint64_t*)k.mv_data;
4157  outkey *ok = (outkey *)v.mv_data;
4159  if (!f(amount, toi.first, ok->data.height, toi.second)) {
4160  fret = false;
4161  break;
4162  }
4163  }
4164 
4166 
4167  return fret;
4168 }
4169 
4170 bool BlockchainLMDB::for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const
4171 {
4172  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4173  check_open();
4174 
4176  RCURSOR(output_amounts);
4177 
4178  MDB_val_set(k, amount);
4179  MDB_val v;
4180  bool fret = true;
4181 
4182  MDB_cursor_op op = MDB_SET;
4183  while (1)
4184  {
4185  int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op);
4186  op = MDB_NEXT_DUP;
4187  if (ret == MDB_NOTFOUND)
4188  break;
4189  if (ret)
4190  throw0(DB_ERROR("Failed to enumerate outputs"));
4191  uint64_t out_amount = *(const uint64_t*)k.mv_data;
4192  if (amount != out_amount)
4193  {
4194  MERROR("Amount is not the expected amount");
4195  fret = false;
4196  break;
4197  }
4198  const outkey *ok = (const outkey *)v.mv_data;
4199  if (!f(ok->data.height)) {
4200  fret = false;
4201  break;
4202  }
4203  }
4204 
4206 
4207  return fret;
4208 }
4209 
4210 // batch_num_blocks: (optional) Used to check if resize needed before batch transaction starts.
4211 bool BlockchainLMDB::batch_start(uint64_t batch_num_blocks, uint64_t batch_bytes)
4212 {
4213  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4214  if (! m_batch_transactions)
4215  throw0(DB_ERROR("batch transactions not enabled"));
4216  if (m_batch_active)
4217  return false;
4218  if (m_write_batch_txn != nullptr)
4219  return false;
4220  if (m_write_txn)
4221  throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use"));
4222  check_open();
4223 
4224  m_writer = boost::this_thread::get_id();
4225  check_and_resize_for_batch(batch_num_blocks, batch_bytes);
4226 
4227  m_write_batch_txn = new mdb_txn_safe();
4228 
4229  // NOTE: need to make sure it's destroyed properly when done
4230  if (auto mdb_res = lmdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn))
4231  {
4232  delete m_write_batch_txn;
4233  m_write_batch_txn = nullptr;
4234  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str()));
4235  }
4236  // indicates this transaction is for batch transactions, but not whether it's
4237  // active
4238  m_write_batch_txn->m_batch_txn = true;
4239  m_write_txn = m_write_batch_txn;
4240 
4241  m_batch_active = true;
4242  memset(&m_wcursors, 0, sizeof(m_wcursors));
4243  if (m_tinfo.get())
4244  {
4245  if (m_tinfo->m_ti_rflags.m_rf_txn)
4246  mdb_txn_reset(m_tinfo->m_ti_rtxn);
4247  memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
4248  }
4249 
4250  LOG_PRINT_L3("batch transaction: begin");
4251  return true;
4252 }
4253 
4255 {
4256  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4257  if (! m_batch_transactions)
4258  throw0(DB_ERROR("batch transactions not enabled"));
4259  if (! m_batch_active)
4260  throw1(DB_ERROR("batch transaction not in progress"));
4261  if (m_write_batch_txn == nullptr)
4262  throw1(DB_ERROR("batch transaction not in progress"));
4263  if (m_writer != boost::this_thread::get_id())
4264  throw1(DB_ERROR("batch transaction owned by other thread"));
4265 
4266  check_open();
4267 
4268  LOG_PRINT_L3("batch transaction: committing...");
4269  TIME_MEASURE_START(time1);
4270  m_write_txn->commit();
4271  TIME_MEASURE_FINISH(time1);
4272  time_commit1 += time1;
4273  LOG_PRINT_L3("batch transaction: committed");
4274 
4275  m_write_txn = nullptr;
4276  delete m_write_batch_txn;
4277  m_write_batch_txn = nullptr;
4278  memset(&m_wcursors, 0, sizeof(m_wcursors));
4279 }
4280 
4281 void BlockchainLMDB::cleanup_batch()
4282 {
4283  // for destruction of batch transaction
4284  m_write_txn = nullptr;
4285  delete m_write_batch_txn;
4286  m_write_batch_txn = nullptr;
4287  m_batch_active = false;
4288  memset(&m_wcursors, 0, sizeof(m_wcursors));
4289 }
4290 
4292 {
4293  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4294  if (! m_batch_transactions)
4295  throw0(DB_ERROR("batch transactions not enabled"));
4296  if (! m_batch_active)
4297  throw1(DB_ERROR("batch transaction not in progress"));
4298  if (m_write_batch_txn == nullptr)
4299  throw1(DB_ERROR("batch transaction not in progress"));
4300  if (m_writer != boost::this_thread::get_id())
4301  throw1(DB_ERROR("batch transaction owned by other thread"));
4302  check_open();
4303  LOG_PRINT_L3("batch transaction: committing...");
4304  TIME_MEASURE_START(time1);
4305  try
4306  {
4307  m_write_txn->commit();
4308  TIME_MEASURE_FINISH(time1);
4309  time_commit1 += time1;
4310  cleanup_batch();
4311  }
4312  catch (const std::exception &e)
4313  {
4314  cleanup_batch();
4315  throw;
4316  }
4317  LOG_PRINT_L3("batch transaction: end");
4318 }
4319 
4321 {
4322  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4323  if (! m_batch_transactions)
4324  throw0(DB_ERROR("batch transactions not enabled"));
4325  if (! m_batch_active)
4326  throw1(DB_ERROR("batch transaction not in progress"));
4327  if (m_write_batch_txn == nullptr)
4328  throw1(DB_ERROR("batch transaction not in progress"));
4329  if (m_writer != boost::this_thread::get_id())
4330  throw1(DB_ERROR("batch transaction owned by other thread"));
4331  check_open();
4332  // for destruction of batch transaction
4333  m_write_txn = nullptr;
4334  // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called.
4335  m_write_batch_txn->abort();
4336  delete m_write_batch_txn;
4337  m_write_batch_txn = nullptr;
4338  m_batch_active = false;
4339  memset(&m_wcursors, 0, sizeof(m_wcursors));
4340  LOG_PRINT_L3("batch transaction: aborted");
4341 }
4342 
4343 void BlockchainLMDB::set_batch_transactions(bool batch_transactions)
4344 {
4345  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4346  if ((batch_transactions) && (m_batch_transactions))
4347  {
4348  MINFO("batch transaction mode already enabled, but asked to enable batch mode");
4349  }
4350  m_batch_transactions = batch_transactions;
4351  MINFO("batch transactions " << (m_batch_transactions ? "enabled" : "disabled"));
4352 }
4353 
4354 // return true if we started the txn, false if already started
4356 {
4357  bool ret = false;
4358  mdb_threadinfo *tinfo;
4359  if (m_write_txn && m_writer == boost::this_thread::get_id()) {
4360  *mtxn = m_write_txn->m_txn;
4361  *mcur = (mdb_txn_cursors *)&m_wcursors;
4362  return ret;
4363  }
4364  /* Check for existing info and force reset if env doesn't match -
4365  * only happens if env was opened/closed multiple times in same process
4366  */
4367  if (!(tinfo = m_tinfo.get()) || mdb_txn_env(tinfo->m_ti_rtxn) != m_env)
4368  {
4369  tinfo = new mdb_threadinfo;
4370  m_tinfo.reset(tinfo);
4371  memset(&tinfo->m_ti_rcursors, 0, sizeof(tinfo->m_ti_rcursors));
4372  memset(&tinfo->m_ti_rflags, 0, sizeof(tinfo->m_ti_rflags));
4373  if (auto mdb_res = lmdb_txn_begin(m_env, NULL, MDB_RDONLY, &tinfo->m_ti_rtxn))
4374  throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str()));
4375  ret = true;
4376  } else if (!tinfo->m_ti_rflags.m_rf_txn)
4377  {
4378  if (auto mdb_res = lmdb_txn_renew(tinfo->m_ti_rtxn))
4379  throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str()));
4380  ret = true;
4381  }
4382  if (ret)
4383  tinfo->m_ti_rflags.m_rf_txn = true;
4384  *mtxn = tinfo->m_ti_rtxn;
4385  *mcur = &tinfo->m_ti_rcursors;
4386 
4387  if (ret)
4388  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4389  return ret;
4390 }
4391 
4393 {
4394  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4395  mdb_txn_reset(m_tinfo->m_ti_rtxn);
4396  memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
4397 }
4398 
4400 {
4401  MDB_txn *mtxn;
4402  mdb_txn_cursors *mcur;
4403  return block_rtxn_start(&mtxn, &mcur);
4404 }
4405 
4407 {
4408  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4409  // Distinguish the exceptions here from exceptions that would be thrown while
4410  // using the txn and committing it.
4411  //
4412  // If an exception is thrown in this setup, we don't want the caller to catch
4413  // it and proceed as if there were an existing write txn, such as trying to
4414  // call block_txn_abort(). It also indicates a serious issue which will
4415  // probably be thrown up another layer.
4416  if (! m_batch_active && m_write_txn)
4417  throw0(DB_ERROR_TXN_START((std::string("Attempted to start new write txn when write txn already exists in ")+__FUNCTION__).c_str()));
4418  if (! m_batch_active)
4419  {
4420  m_writer = boost::this_thread::get_id();
4421  m_write_txn = new mdb_txn_safe();
4422  if (auto mdb_res = lmdb_txn_begin(m_env, NULL, 0, *m_write_txn))
4423  {
4424  delete m_write_txn;
4425  m_write_txn = nullptr;
4426  throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str()));
4427  }
4428  memset(&m_wcursors, 0, sizeof(m_wcursors));
4429  if (m_tinfo.get())
4430  {
4431  if (m_tinfo->m_ti_rflags.m_rf_txn)
4432  mdb_txn_reset(m_tinfo->m_ti_rtxn);
4433  memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
4434  }
4435  } else if (m_writer != boost::this_thread::get_id())
4436  throw0(DB_ERROR_TXN_START((std::string("Attempted to start new write txn when batch txn already exists in ")+__FUNCTION__).c_str()));
4437 }
4438 
4440 {
4441  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4442  if (!m_write_txn)
4443  throw0(DB_ERROR_TXN_START((std::string("Attempted to stop write txn when no such txn exists in ")+__FUNCTION__).c_str()));
4444  if (m_writer != boost::this_thread::get_id())
4445  throw0(DB_ERROR_TXN_START((std::string("Attempted to stop write txn from the wrong thread in ")+__FUNCTION__).c_str()));
4446  {
4447  if (! m_batch_active)
4448  {
4449  TIME_MEASURE_START(time1);
4450  m_write_txn->commit();
4451  TIME_MEASURE_FINISH(time1);
4452  time_commit1 += time1;
4453 
4454  delete m_write_txn;
4455  m_write_txn = nullptr;
4456  memset(&m_wcursors, 0, sizeof(m_wcursors));
4457  }
4458  }
4459 }
4460 
4462 {
4463  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4464  if (!m_write_txn)
4465  throw0(DB_ERROR_TXN_START((std::string("Attempted to abort write txn when no such txn exists in ")+__FUNCTION__).c_str()));
4466  if (m_writer != boost::this_thread::get_id())
4467  throw0(DB_ERROR_TXN_START((std::string("Attempted to abort write txn from the wrong thread in ")+__FUNCTION__).c_str()));
4468 
4469  if (! m_batch_active)
4470  {
4471  delete m_write_txn;
4472  m_write_txn = nullptr;
4473  memset(&m_wcursors, 0, sizeof(m_wcursors));
4474  }
4475 }
4476 
4478 {
4479  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4480  mdb_txn_reset(m_tinfo->m_ti_rtxn);
4481  memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
4482 }
4483 
4484 uint64_t BlockchainLMDB::add_block(const std::pair<block, blobdata>& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
4485  const std::vector<std::pair<transaction, blobdata>>& txs)
4486 {
4487  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4488  check_open();
4489  uint64_t m_height = height();
4490 
4491  if (m_height % 1024 == 0)
4492  {
4493  // for batch mode, DB resize check is done at start of batch transaction
4494  if (! m_batch_active && need_resize())
4495  {
4496  LOG_PRINT_L0("LMDB memory map needs to be resized, doing that now.");
4497  do_resize();
4498  }
4499  }
4500 
4501  try
4502  {
4503  BlockchainDB::add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs);
4504  }
4505  catch (const DB_ERROR_TXN_START& e)
4506  {
4507  throw;
4508  }
4509 
4510  return ++m_height;
4511 }
4512 
4513 void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
4514 {
4515  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4516  check_open();
4517 
4518  block_wtxn_start();
4519 
4520  try
4521  {
4522  BlockchainDB::pop_block(blk, txs);
4523  block_wtxn_stop();
4524  }
4525  catch (...)
4526  {
4527  block_wtxn_abort();
4528  throw;
4529  }
4530 }
4531 
4532 void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,
4533  std::vector<tx_out_index> &tx_out_indices) const
4534 {
4535  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4536  check_open();
4537  tx_out_indices.clear();
4538  tx_out_indices.reserve(global_indices.size());
4539 
4541  RCURSOR(output_txs);
4542 
4543  for (const uint64_t &output_id : global_indices)
4544  {
4545  MDB_val_set(v, output_id);
4546 
4547  auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
4548  if (get_result == MDB_NOTFOUND)
4549  throw1(OUTPUT_DNE("output with given index not in db"));
4550  else if (get_result)
4551  throw0(DB_ERROR("DB error attempting to fetch output tx hash"));
4552 
4553  const outtx *ot = (const outtx *)v.mv_data;
4554  tx_out_indices.push_back(tx_out_index(ot->tx_hash, ot->local_index));
4555  }
4556 
4558 }
4559 
4560 void BlockchainLMDB::get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial) const
4561 {
4562  if (amounts.size() != 1 && amounts.size() != offsets.size())
4563  throw0(DB_ERROR("Invalid sizes of amounts and offets"));
4564 
4565  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4566  TIME_MEASURE_START(db3);
4567  check_open();
4568  outputs.clear();
4569  outputs.reserve(offsets.size());
4570 
4572 
4573  RCURSOR(output_amounts);
4574 
4575  for (size_t i = 0; i < offsets.size(); ++i)
4576  {
4577  const uint64_t amount = amounts.size() == 1 ? amounts[0] : amounts[i];
4578  MDB_val_set(k, amount);
4579  MDB_val_set(v, offsets[i]);
4580 
4581  auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH);
4582  if (get_result == MDB_NOTFOUND)
4583  {
4584  if (allow_partial)
4585  {
4586  MDEBUG("Partial result: " << outputs.size() << "/" << offsets.size());
4587  break;
4588  }
4589  throw1(OUTPUT_DNE((std::string("Attempting to get output pubkey by global index (amount ") + boost::lexical_cast<std::string>(amount) + ", index " + boost::lexical_cast<std::string>(offsets[i]) + ", count " + boost::lexical_cast<std::string>(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast<std::string>(height()) + ")").c_str()));
4590  }
4591  else if (get_result)
4592  throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str()));
4593 
4594  if (amount == 0)
4595  {
4596  const outkey *okp = (const outkey *)v.mv_data;
4597  outputs.push_back(okp->data);
4598  }
4599  else
4600  {
4601  const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data;
4602  outputs.resize(outputs.size() + 1);
4603  output_data_t &data = outputs.back();
4604  memcpy(&data, &okp->data, sizeof(pre_rct_output_data_t));
4605  data.commitment = rct::zeroCommit(amount);
4606  }
4607  }
4608 
4610 
4611  TIME_MEASURE_FINISH(db3);
4612  LOG_PRINT_L3("db3: " << db3);
4613 }
4614 
4615 void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const
4616 {
4617  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4618  check_open();
4619  indices.clear();
4620 
4621  std::vector <uint64_t> tx_indices;
4622  tx_indices.reserve(offsets.size());
4624 
4625  RCURSOR(output_amounts);
4626 
4627  MDB_val_set(k, amount);
4628  for (const uint64_t &index : offsets)
4629  {
4630  MDB_val_set(v, index);
4631 
4632  auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH);
4633  if (get_result == MDB_NOTFOUND)
4634  throw1(OUTPUT_DNE("Attempting to get output by index, but key does not exist"));
4635  else if (get_result)
4636  throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output from the db", get_result).c_str()));
4637 
4638  const outkey *okp = (const outkey *)v.mv_data;
4639  tx_indices.push_back(okp->output_id);
4640  }
4641 
4642  TIME_MEASURE_START(db3);
4643  if(tx_indices.size() > 0)
4644  {
4645  get_output_tx_and_index_from_global(tx_indices, indices);
4646  }
4647  TIME_MEASURE_FINISH(db3);
4648  LOG_PRINT_L3("db3: " << db3);
4649 }
4650 
4651 std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> BlockchainLMDB::get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const
4652 {
4653  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4654  check_open();
4655 
4657  RCURSOR(output_amounts);
4658 
4659  std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> histogram;
4660  MDB_val k;
4661  MDB_val v;
4662 
4663  if (amounts.empty())
4664  {
4665  MDB_cursor_op op = MDB_FIRST;
4666  while (1)
4667  {
4668  int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op);
4669  op = MDB_NEXT_NODUP;
4670  if (ret == MDB_NOTFOUND)
4671  break;
4672  if (ret)
4673  throw0(DB_ERROR(lmdb_error("Failed to enumerate outputs: ", ret).c_str()));
4674  mdb_size_t num_elems = 0;
4676  uint64_t amount = *(const uint64_t*)k.mv_data;
4677  if (num_elems >= min_count)
4678  histogram[amount] = std::make_tuple(num_elems, 0, 0);
4679  }
4680  }
4681  else
4682  {
4683  for (const auto &amount: amounts)
4684  {
4685  MDB_val_copy<uint64_t> k(amount);
4686  int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET);
4687  if (ret == MDB_NOTFOUND)
4688  {
4689  if (0 >= min_count)
4690  histogram[amount] = std::make_tuple(0, 0, 0);
4691  }
4692  else if (ret == MDB_SUCCESS)
4693  {
4694  mdb_size_t num_elems = 0;
4696  if (num_elems >= min_count)
4697  histogram[amount] = std::make_tuple(num_elems, 0, 0);
4698  }
4699  else
4700  {
4701  throw0(DB_ERROR(lmdb_error("Failed to enumerate outputs: ", ret).c_str()));
4702  }
4703  }
4704  }
4705 
4706  if (unlocked || recent_cutoff > 0) {
4707  const uint64_t blockchain_height = height();
4708  for (std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>::iterator i = histogram.begin(); i != histogram.end(); ++i) {
4709  uint64_t amount = i->first;
4710  uint64_t num_elems = std::get<0>(i->second);
4711  while (num_elems > 0) {
4712  const tx_out_index toi = get_output_tx_and_index(amount, num_elems - 1);
4713  const uint64_t height = get_tx_block_height(toi.first);
4714  if (height + (get_hard_fork_version(height) > 7 ? ETN_DEFAULT_TX_SPENDABLE_AGE_V8 : CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE) <= blockchain_height)
4715  break;
4716  --num_elems;
4717  }
4718  // modifying second does not invalidate the iterator
4719  std::get<1>(i->second) = num_elems;
4720 
4721  if (recent_cutoff > 0)
4722  {
4723  uint64_t recent = 0;
4724  while (num_elems > 0) {
4725  const tx_out_index toi = get_output_tx_and_index(amount, num_elems - 1);
4726  const uint64_t height = get_tx_block_height(toi.first);
4727  const uint64_t ts = get_block_timestamp(height);
4728  if (ts < recent_cutoff)
4729  break;
4730  --num_elems;
4731  ++recent;
4732  }
4733  // modifying second does not invalidate the iterator
4734  std::get<2>(i->second) = recent;
4735  }
4736  }
4737  }
4738 
4740 
4741  return histogram;
4742 }
4743 
4744 bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const
4745 {
4746  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4747  check_open();
4748 
4750  RCURSOR(output_amounts);
4751 
4752  distribution.clear();
4753  const uint64_t db_height = height();
4754  if (from_height >= db_height)
4755  return false;
4756  distribution.resize(db_height - from_height, 0);
4757 
4758  bool fret = true;
4759  MDB_val_set(k, amount);
4760  MDB_val v;
4761  MDB_cursor_op op = MDB_SET;
4762  base = 0;
4763  while (1)
4764  {
4765  int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op);
4766  op = MDB_NEXT_DUP;
4767  if (ret == MDB_NOTFOUND)
4768  break;
4769  if (ret)
4770  throw0(DB_ERROR("Failed to enumerate outputs"));
4771  const outkey *ok = (const outkey *)v.mv_data;
4772  const uint64_t height = ok->data.height;
4773  if (height >= from_height)
4774  distribution[height - from_height]++;
4775  else
4776  base++;
4777  if (to_height > 0 && height > to_height)
4778  break;
4779  }
4780 
4781  distribution[0] += base;
4782  for (size_t n = 1; n < distribution.size(); ++n)
4783  distribution[n] += distribution[n - 1];
4784  base = 0;
4785 
4787 
4788  return true;
4789 }
4790 
4791 void BlockchainLMDB::check_hard_fork_info()
4792 {
4793 }
4794 
4795 void BlockchainLMDB::drop_hard_fork_info()
4796 {
4797  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4798  check_open();
4799 
4800  TXN_PREFIX(0);
4801 
4802  auto result = mdb_drop(*txn_ptr, m_hf_starting_heights, 1);
4803  if (result)
4804  throw1(DB_ERROR(lmdb_error("Error dropping hard fork starting heights db: ", result).c_str()));
4805  result = mdb_drop(*txn_ptr, m_hf_versions, 1);
4806  if (result)
4807  throw1(DB_ERROR(lmdb_error("Error dropping hard fork versions db: ", result).c_str()));
4808 
4810 }
4811 
4812 void BlockchainLMDB::set_validator_list(std::string validator_list, uint32_t expiration_date) {
4813  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4814  check_open();
4815 
4816  TXN_BLOCK_PREFIX(0);
4817 
4818  validator_db v;
4819  v.validators = std::vector<uint8_t>(validator_list.begin(), validator_list.end());
4820  v.expiration_date = expiration_date;
4821 
4822  MDB_val_copy<uint64_t> val_key(0);
4823  MDB_val_copy<blobdata> val_value(validator_to_blob(v));
4824 
4825  int result = mdb_put(*txn_ptr, m_validators, &val_key, &val_value, MDB_APPEND);
4826  if (result == MDB_KEYEXIST)
4827  result = mdb_put(*txn_ptr, m_validators, &val_key, &val_value, 0);
4828  if (result)
4829  throw1(DB_ERROR(lmdb_error("Error adding validator list to db transaction: ", result).c_str()));
4830 
4832 }
4833 
4834 std::string BlockchainLMDB::get_validator_list() const {
4835  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4836  check_open();
4837 
4839  RCURSOR(validators);
4840 
4841  MDB_val_copy<uint64_t> val_key(0);
4842  MDB_val val_ret;
4843  auto result = mdb_cursor_get(m_cur_validators, &val_key, &val_ret, MDB_SET);
4844  if (result == MDB_NOTFOUND || result) {
4845  LOG_PRINT_L1("Error attempting to retrieve the list of validators from the db.");
4847  return std::string("");
4848  }
4849 
4850  blobdata ret;
4851  ret.assign(reinterpret_cast<const char*>(val_ret.mv_data), val_ret.mv_size);
4852 
4853  validator_db v = validator_from_blob(ret);
4854 
4855  if((v.expiration_date) - time(nullptr) <= 0) {
4857  return std::string("");
4858  }
4859 
4861  return std::string(v.validators.begin(), v.validators.end());
4862 }
4863 
4864 void BlockchainLMDB::set_hard_fork_version(uint64_t height, uint8_t version)
4865 {
4866  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4867  check_open();
4868 
4869  TXN_BLOCK_PREFIX(0);
4870 
4871  MDB_val_copy<uint64_t> val_key(height);
4872  MDB_val_copy<uint8_t> val_value(version);
4873  int result;
4874  result = mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, MDB_APPEND);
4875  if (result == MDB_KEYEXIST)
4876  result = mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, 0);
4877  if (result)
4878  throw1(DB_ERROR(lmdb_error("Error adding hard fork version to db transaction: ", result).c_str()));
4879 
4881 }
4882 
4883 uint8_t BlockchainLMDB::get_hard_fork_version(uint64_t height) const
4884 {
4885  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4886  check_open();
4887 
4889  RCURSOR(hf_versions);
4890 
4891  MDB_val_copy<uint64_t> val_key(height);
4892  MDB_val val_ret;
4893  auto result = mdb_cursor_get(m_cur_hf_versions, &val_key, &val_ret, MDB_SET);
4894  if (result == MDB_NOTFOUND || result)
4895  throw0(DB_ERROR(lmdb_error("Error attempting to retrieve a hard fork version at height " + boost::lexical_cast<std::string>(height) + " from the db: ", result).c_str()));
4896 
4897  uint8_t ret = *(const uint8_t*)val_ret.mv_data;
4899  return ret;
4900 }
4901 
4902 bool BlockchainLMDB::is_read_only() const
4903 {
4904  unsigned int flags;
4905  auto result = mdb_env_get_flags(m_env, &flags);
4906  if (result)
4907  throw0(DB_ERROR(lmdb_error("Error getting database environment info: ", result).c_str()));
4908 
4909  if (flags & MDB_RDONLY)
4910  return true;
4911 
4912  return false;
4913 }
4914 
4915 uint64_t BlockchainLMDB::get_database_size() const
4916 {
4917  uint64_t size = 0;
4918  boost::filesystem::path datafile(m_folder);
4920  if (!epee::file_io_utils::get_file_size(datafile.string(), size))
4921  size = 0;
4922  return size;
4923 }
4924 
4925 void BlockchainLMDB::fixup()
4926 {
4927  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4928  // Always call parent as well
4930 }
4931 
4932 #define RENAME_DB(name) do { \
4933  char n2[] = name; \
4934  MDB_dbi tdbi; \
4935  n2[sizeof(n2)-2]--; \
4936  /* play some games to put (name) on a writable page */ \
4937  result = mdb_dbi_open(txn, n2, MDB_CREATE, &tdbi); \
4938  if (result) \
4939  throw0(DB_ERROR(lmdb_error("Failed to create " + std::string(n2) + ": ", result).c_str())); \
4940  result = mdb_drop(txn, tdbi, 1); \
4941  if (result) \
4942  throw0(DB_ERROR(lmdb_error("Failed to delete " + std::string(n2) + ": ", result).c_str())); \
4943  k.mv_data = (void *)name; \
4944  k.mv_size = sizeof(name)-1; \
4945  result = mdb_cursor_open(txn, 1, &c_cur); \
4946  if (result) \
4947  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for " name ": ", result).c_str())); \
4948  result = mdb_cursor_get(c_cur, &k, NULL, MDB_SET_KEY); \
4949  if (result) \
4950  throw0(DB_ERROR(lmdb_error("Failed to get DB record for " name ": ", result).c_str())); \
4951  ptr = (char *)k.mv_data; \
4952  ptr[sizeof(name)-2]++; } while(0)
4953 
4954 #define LOGIF(y) if (ELPP->vRegistry()->allowed(y, "global"))
4955 
4956 void BlockchainLMDB::migrate_0_1()
4957 {
4958  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
4959  uint64_t i, z, m_height;
4960  int result;
4961  mdb_txn_safe txn(false);
4962  MDB_val k, v;
4963  char *ptr;
4964 
4965  MGINFO_YELLOW("Migrating blockchain from DB version 0 to 1 - this may take a while:");
4966  MINFO("updating blocks, hf_versions, outputs, txs, and spent_keys tables...");
4967 
4968  do {
4969  result = mdb_txn_begin(m_env, NULL, 0, txn);
4970  if (result)
4971  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
4972 
4973  MDB_stat db_stats;
4974  if ((result = mdb_stat(txn, m_blocks, &db_stats)))
4975  throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
4976  m_height = db_stats.ms_entries;
4977  MINFO("Total number of blocks: " << m_height);
4978  MINFO("block migration will update block_heights, block_info, and hf_versions...");
4979 
4980  MINFO("migrating block_heights:");
4981  MDB_dbi o_heights;
4982 
4983  unsigned int flags;
4984  result = mdb_dbi_flags(txn, m_block_heights, &flags);
4985  if (result)
4986  throw0(DB_ERROR(lmdb_error("Failed to retrieve block_heights flags: ", result).c_str()));
4987  /* if the flags are what we expect, this table has already been migrated */
4989  txn.abort();
4990  LOG_PRINT_L1(" block_heights already migrated");
4991  break;
4992  }
4993 
4994  /* the block_heights table name is the same but the old version and new version
4995  * have incompatible DB flags. Create a new table with the right flags. We want
4996  * the name to be similar to the old name so that it will occupy the same location
4997  * in the DB.
4998  */
4999  o_heights = m_block_heights;
5000  lmdb_db_open(txn, "block_heightr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for block_heightr");
5001  mdb_set_dupsort(txn, m_block_heights, compare_hash32);
5002 
5003  MDB_cursor *c_old, *c_cur;
5004  blk_height bh;
5005  MDB_val_set(nv, bh);
5006 
5007  /* old table was k(hash), v(height).
5008  * new table is DUPFIXED, k(zeroval), v{hash, height}.
5009  */
5010  i = 0;
5011  z = m_height;
5012  while(1) {
5013  if (!(i % 2000)) {
5014  if (i) {
5016  std::cout << i << " / " << z << " \r" << std::flush;
5017  }
5018  txn.commit();
5019  result = mdb_txn_begin(m_env, NULL, 0, txn);
5020  if (result)
5021  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5022  }
5023  result = mdb_cursor_open(txn, m_block_heights, &c_cur);
5024  if (result)
5025  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_heightr: ", result).c_str()));
5026  result = mdb_cursor_open(txn, o_heights, &c_old);
5027  if (result)
5028  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_heights: ", result).c_str()));
5029  if (!i) {
5030  MDB_stat ms;
5031  result = mdb_stat(txn, m_block_heights, &ms);
5032  if (result)
5033  throw0(DB_ERROR(lmdb_error("Failed to query block_heights table: ", result).c_str()));
5034  i = ms.ms_entries;
5035  }
5036  }
5037  result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT);
5038  if (result == MDB_NOTFOUND) {
5039  txn.commit();
5040  break;
5041  }
5042  else if (result)
5043  throw0(DB_ERROR(lmdb_error("Failed to get a record from block_heights: ", result).c_str()));
5044  bh.bh_hash = *(crypto::hash *)k.mv_data;
5045  bh.bh_height = *(uint64_t *)v.mv_data;
5046  result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP);
5047  if (result)
5048  throw0(DB_ERROR(lmdb_error("Failed to put a record into block_heightr: ", result).c_str()));
5049  /* we delete the old records immediately, so the overall DB and mapsize should not grow.
5050  * This is a little slower than just letting mdb_drop() delete it all at the end, but
5051  * it saves a significant amount of disk space.
5052  */
5053  result = mdb_cursor_del(c_old, 0);
5054  if (result)
5055  throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_heights: ", result).c_str()));
5056  i++;
5057  }
5058 
5059  result = mdb_txn_begin(m_env, NULL, 0, txn);
5060  if (result)
5061  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5062  /* Delete the old table */
5063  result = mdb_drop(txn, o_heights, 1);
5064  if (result)
5065  throw0(DB_ERROR(lmdb_error("Failed to delete old block_heights table: ", result).c_str()));
5066 
5067  RENAME_DB("block_heightr");
5068 
5069  /* close and reopen to get old dbi slot back */
5070  mdb_dbi_close(m_env, m_block_heights);
5071  lmdb_db_open(txn, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for block_heights");
5072  mdb_set_dupsort(txn, m_block_heights, compare_hash32);
5073  txn.commit();
5074 
5075  } while(0);
5076 
5077  /* old tables are k(height), v(value).
5078  * new table is DUPFIXED, k(zeroval), v{height, values...}.
5079  */
5080  do {
5081  LOG_PRINT_L1("migrating block info:");
5082 
5083  MDB_dbi coins;
5084  result = mdb_txn_begin(m_env, NULL, 0, txn);
5085  if (result)
5086  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5087  result = mdb_dbi_open(txn, "block_coins", 0, &coins);
5088  if (result == MDB_NOTFOUND) {
5089  txn.abort();
5090  LOG_PRINT_L1(" block_info already migrated");
5091  break;
5092  }
5093  MDB_dbi diffs, hashes, sizes, timestamps;
5094  mdb_block_info_1 bi;
5095  MDB_val_set(nv, bi);
5096 
5097  lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs");
5098  lmdb_db_open(txn, "block_hashes", 0, hashes, "Failed to open db handle for block_hashes");
5099  lmdb_db_open(txn, "block_sizes", 0, sizes, "Failed to open db handle for block_sizes");
5100  lmdb_db_open(txn, "block_timestamps", 0, timestamps, "Failed to open db handle for block_timestamps");
5101  MDB_cursor *c_cur, *c_coins, *c_diffs, *c_hashes, *c_sizes, *c_timestamps;
5102  i = 0;
5103  z = m_height;
5104  while(1) {
5105  MDB_val k, v;
5106  if (!(i % 2000)) {
5107  if (i) {
5109  std::cout << i << " / " << z << " \r" << std::flush;
5110  }
5111  txn.commit();
5112  result = mdb_txn_begin(m_env, NULL, 0, txn);
5113  if (result)
5114  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5115  }
5116  result = mdb_cursor_open(txn, m_block_info, &c_cur);
5117  if (result)
5118  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str()));
5119  result = mdb_cursor_open(txn, coins, &c_coins);
5120  if (result)
5121  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_coins: ", result).c_str()));
5122  result = mdb_cursor_open(txn, diffs, &c_diffs);
5123  if (result)
5124  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_diffs: ", result).c_str()));
5125  result = mdb_cursor_open(txn, hashes, &c_hashes);
5126  if (result)
5127  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_hashes: ", result).c_str()));
5128  result = mdb_cursor_open(txn, sizes, &c_sizes);
5129  if (result)
5130  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_coins: ", result).c_str()));
5131  result = mdb_cursor_open(txn, timestamps, &c_timestamps);
5132  if (result)
5133  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_timestamps: ", result).c_str()));
5134  if (!i) {
5135  MDB_stat ms;
5136  result = mdb_stat(txn, m_block_info, &ms);
5137  if (result)
5138  throw0(DB_ERROR(lmdb_error("Failed to query block_info table: ", result).c_str()));
5139  i = ms.ms_entries;
5140  }
5141  }
5142  result = mdb_cursor_get(c_coins, &k, &v, MDB_NEXT);
5143  if (result == MDB_NOTFOUND) {
5144  break;
5145  } else if (result)
5146  throw0(DB_ERROR(lmdb_error("Failed to get a record from block_coins: ", result).c_str()));
5147  bi.bi_height = *(uint64_t *)k.mv_data;
5148  bi.bi_coins = *(uint64_t *)v.mv_data;
5149  result = mdb_cursor_get(c_diffs, &k, &v, MDB_NEXT);
5150  if (result)
5151  throw0(DB_ERROR(lmdb_error("Failed to get a record from block_diffs: ", result).c_str()));
5152  bi.bi_diff = *(uint64_t *)v.mv_data;
5153  result = mdb_cursor_get(c_hashes, &k, &v, MDB_NEXT);
5154  if (result)
5155  throw0(DB_ERROR(lmdb_error("Failed to get a record from block_hashes: ", result).c_str()));
5156  bi.bi_hash = *(crypto::hash *)v.mv_data;
5157  result = mdb_cursor_get(c_sizes, &k, &v, MDB_NEXT);
5158  if (result)
5159  throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str()));
5160  if (v.mv_size == sizeof(uint32_t))
5161  bi.bi_weight = *(uint32_t *)v.mv_data;
5162  else
5163  bi.bi_weight = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0
5164  result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT);
5165  if (result)
5166  throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str()));
5167  bi.bi_timestamp = *(uint64_t *)v.mv_data;
5168  result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP);
5169  if (result)
5170  throw0(DB_ERROR(lmdb_error("Failed to put a record into block_info: ", result).c_str()));
5171  result = mdb_cursor_del(c_coins, 0);
5172  if (result)
5173  throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_coins: ", result).c_str()));
5174  result = mdb_cursor_del(c_diffs, 0);
5175  if (result)
5176  throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_diffs: ", result).c_str()));
5177  result = mdb_cursor_del(c_hashes, 0);
5178  if (result)
5179  throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_hashes: ", result).c_str()));
5180  result = mdb_cursor_del(c_sizes, 0);
5181  if (result)
5182  throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_sizes: ", result).c_str()));
5183  result = mdb_cursor_del(c_timestamps, 0);
5184  if (result)
5185  throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_timestamps: ", result).c_str()));
5186  i++;
5187  }
5188  mdb_cursor_close(c_timestamps);
5189  mdb_cursor_close(c_sizes);
5190  mdb_cursor_close(c_hashes);
5191  mdb_cursor_close(c_diffs);
5192  mdb_cursor_close(c_coins);
5193  result = mdb_drop(txn, timestamps, 1);
5194  if (result)
5195  throw0(DB_ERROR(lmdb_error("Failed to delete block_timestamps from the db: ", result).c_str()));
5196  result = mdb_drop(txn, sizes, 1);
5197  if (result)
5198  throw0(DB_ERROR(lmdb_error("Failed to delete block_sizes from the db: ", result).c_str()));
5199  result = mdb_drop(txn, hashes, 1);
5200  if (result)
5201  throw0(DB_ERROR(lmdb_error("Failed to delete block_hashes from the db: ", result).c_str()));
5202  result = mdb_drop(txn, diffs, 1);
5203  if (result)
5204  throw0(DB_ERROR(lmdb_error("Failed to delete block_diffs from the db: ", result).c_str()));
5205  result = mdb_drop(txn, coins, 1);
5206  if (result)
5207  throw0(DB_ERROR(lmdb_error("Failed to delete block_coins from the db: ", result).c_str()));
5208  txn.commit();
5209  } while(0);
5210 
5211  do {
5212  LOG_PRINT_L1("migrating hf_versions:");
5213  MDB_dbi o_hfv;
5214 
5215  unsigned int flags;
5216  result = mdb_txn_begin(m_env, NULL, 0, txn);
5217  if (result)
5218  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5219  result = mdb_dbi_flags(txn, m_hf_versions, &flags);
5220  if (result)
5221  throw0(DB_ERROR(lmdb_error("Failed to retrieve hf_versions flags: ", result).c_str()));
5222  /* if the flags are what we expect, this table has already been migrated */
5223  if (flags & MDB_INTEGERKEY) {
5224  txn.abort();
5225  LOG_PRINT_L1(" hf_versions already migrated");
5226  break;
5227  }
5228 
5229  /* the hf_versions table name is the same but the old version and new version
5230  * have incompatible DB flags. Create a new table with the right flags.
5231  */
5232  o_hfv = m_hf_versions;
5233  lmdb_db_open(txn, "hf_versionr", MDB_INTEGERKEY | MDB_CREATE, m_hf_versions, "Failed to open db handle for hf_versionr");
5234 
5235  MDB_cursor *c_old, *c_cur;
5236  i = 0;
5237  z = m_height;
5238 
5239  while(1) {
5240  if (!(i % 2000)) {
5241  if (i) {
5243  std::cout << i << " / " << z << " \r" << std::flush;
5244  }
5245  txn.commit();
5246  result = mdb_txn_begin(m_env, NULL, 0, txn);
5247  if (result)
5248  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5249  }
5250  result = mdb_cursor_open(txn, m_hf_versions, &c_cur);
5251  if (result)
5252  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keyr: ", result).c_str()));
5253  result = mdb_cursor_open(txn, o_hfv, &c_old);
5254  if (result)
5255  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keys: ", result).c_str()));
5256  if (!i) {
5257  MDB_stat ms;
5258  result = mdb_stat(txn, m_hf_versions, &ms);
5259  if (result)
5260  throw0(DB_ERROR(lmdb_error("Failed to query hf_versions table: ", result).c_str()));
5261  i = ms.ms_entries;
5262  }
5263  }
5264  result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT);
5265  if (result == MDB_NOTFOUND) {
5266  txn.commit();
5267  break;
5268  }
5269  else if (result)
5270  throw0(DB_ERROR(lmdb_error("Failed to get a record from hf_versions: ", result).c_str()));
5271  result = mdb_cursor_put(c_cur, &k, &v, MDB_APPEND);
5272  if (result)
5273  throw0(DB_ERROR(lmdb_error("Failed to put a record into hf_versionr: ", result).c_str()));
5274  result = mdb_cursor_del(c_old, 0);
5275  if (result)
5276  throw0(DB_ERROR(lmdb_error("Failed to delete a record from hf_versions: ", result).c_str()));
5277  i++;
5278  }
5279 
5280  result = mdb_txn_begin(m_env, NULL, 0, txn);
5281  if (result)
5282  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5283  /* Delete the old table */
5284  result = mdb_drop(txn, o_hfv, 1);
5285  if (result)
5286  throw0(DB_ERROR(lmdb_error("Failed to delete old hf_versions table: ", result).c_str()));
5287  RENAME_DB("hf_versionr");
5288  mdb_dbi_close(m_env, m_hf_versions);
5289  lmdb_db_open(txn, "hf_versions", MDB_INTEGERKEY, m_hf_versions, "Failed to open db handle for hf_versions");
5290 
5291  txn.commit();
5292  } while(0);
5293 
5294  do {
5295  LOG_PRINT_L1("deleting old indices:");
5296 
5297  /* Delete all other tables, we're just going to recreate them */
5298  MDB_dbi dbi;
5299  result = mdb_txn_begin(m_env, NULL, 0, txn);
5300  if (result)
5301  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5302 
5303  result = mdb_dbi_open(txn, "tx_unlocks", 0, &dbi);
5304  if (result == MDB_NOTFOUND) {
5305  txn.abort();
5306  LOG_PRINT_L1(" old indices already deleted");
5307  break;
5308  }
5309  txn.abort();
5310 
5311 #define DELETE_DB(x) do { \
5312  LOG_PRINT_L1(" " x ":"); \
5313  result = mdb_txn_begin(m_env, NULL, 0, txn); \
5314  if (result) \
5315  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); \
5316  result = mdb_dbi_open(txn, x, 0, &dbi); \
5317  if (!result) { \
5318  result = mdb_drop(txn, dbi, 1); \
5319  if (result) \
5320  throw0(DB_ERROR(lmdb_error("Failed to delete " x ": ", result).c_str())); \
5321  txn.commit(); \
5322  } } while(0)
5323 
5324  DELETE_DB("tx_heights");
5325  DELETE_DB("output_txs");
5326  DELETE_DB("output_indices");
5327  DELETE_DB("output_keys");
5328  DELETE_DB("spent_keys");
5329  DELETE_DB("output_amounts");
5330  DELETE_DB("tx_outputs");
5331  DELETE_DB("tx_unlocks");
5332 
5333  /* reopen new DBs with correct flags */
5334  result = mdb_txn_begin(m_env, NULL, 0, txn);
5335  if (result)
5336  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5337  lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_txs, "Failed to open db handle for m_output_txs");
5338  mdb_set_dupsort(txn, m_output_txs, compare_uint64);
5339  lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs");
5340  lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys");
5341  mdb_set_dupsort(txn, m_spent_keys, compare_hash32);
5342  lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
5343  mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
5344  txn.commit();
5345  } while(0);
5346 
5347  do {
5348  LOG_PRINT_L1("migrating txs and outputs:");
5349 
5350  unsigned int flags;
5351  result = mdb_txn_begin(m_env, NULL, 0, txn);
5352  if (result)
5353  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5354  result = mdb_dbi_flags(txn, m_txs, &flags);
5355  if (result)
5356  throw0(DB_ERROR(lmdb_error("Failed to retrieve txs flags: ", result).c_str()));
5357  /* if the flags are what we expect, this table has already been migrated */
5358  if (flags & MDB_INTEGERKEY) {
5359  txn.abort();
5360  LOG_PRINT_L1(" txs already migrated");
5361  break;
5362  }
5363 
5364  MDB_dbi o_txs;
5365  blobdata bd;
5366  block b;
5367  MDB_val hk;
5368 
5369  o_txs = m_txs;
5370  mdb_set_compare(txn, o_txs, compare_hash32);
5371  lmdb_db_open(txn, "txr", MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for txr");
5372 
5373  txn.commit();
5374 
5375  MDB_cursor *c_blocks, *c_txs, *c_props, *c_cur;
5376  i = 0;
5377  z = m_height;
5378 
5379  hk.mv_size = sizeof(crypto::hash);
5380  set_batch_transactions(true);
5381  batch_start(1000);
5382  txn.m_txn = m_write_txn->m_txn;
5383  m_height = 0;
5384 
5385  while(1) {
5386  if (!(i % 1000)) {
5387  if (i) {
5389  std::cout << i << " / " << z << " \r" << std::flush;
5390  }
5391  MDB_val_set(pk, "txblk");
5392  MDB_val_set(pv, m_height);
5393  result = mdb_cursor_put(c_props, &pk, &pv, 0);
5394  if (result)
5395  throw0(DB_ERROR(lmdb_error("Failed to update txblk property: ", result).c_str()));
5396  txn.commit();
5397  result = mdb_txn_begin(m_env, NULL, 0, txn);
5398  if (result)
5399  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5400  m_write_txn->m_txn = txn.m_txn;
5401  m_write_batch_txn->m_txn = txn.m_txn;
5402  memset(&m_wcursors, 0, sizeof(m_wcursors));
5403  }
5404  result = mdb_cursor_open(txn, m_blocks, &c_blocks);
5405  if (result)
5406  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str()));
5407  result = mdb_cursor_open(txn, m_properties, &c_props);
5408  if (result)
5409  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for properties: ", result).c_str()));
5410  result = mdb_cursor_open(txn, o_txs, &c_txs);
5411  if (result)
5412  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str()));
5413  if (!i) {
5414  MDB_stat ms;
5415  result = mdb_stat(txn, m_txs, &ms);
5416  if (result)
5417  throw0(DB_ERROR(lmdb_error("Failed to query txs table: ", result).c_str()));
5418  i = ms.ms_entries;
5419  if (i) {
5420  MDB_val_set(pk, "txblk");
5421  result = mdb_cursor_get(c_props, &pk, &k, MDB_SET);
5422  if (result)
5423  throw0(DB_ERROR(lmdb_error("Failed to get a record from properties: ", result).c_str()));
5424  m_height = *(uint64_t *)k.mv_data;
5425  }
5426  }
5427  if (i) {
5428  result = mdb_cursor_get(c_blocks, &k, &v, MDB_SET);
5429  if (result)
5430  throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str()));
5431  }
5432  }
5433  result = mdb_cursor_get(c_blocks, &k, &v, MDB_NEXT);
5434  if (result == MDB_NOTFOUND) {
5435  MDB_val_set(pk, "txblk");
5436  result = mdb_cursor_get(c_props, &pk, &v, MDB_SET);
5437  if (result)
5438  throw0(DB_ERROR(lmdb_error("Failed to get a record from props: ", result).c_str()));
5439  result = mdb_cursor_del(c_props, 0);
5440  if (result)
5441  throw0(DB_ERROR(lmdb_error("Failed to delete a record from props: ", result).c_str()));
5442  batch_stop();
5443  break;
5444  } else if (result)
5445  throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str()));
5446 
5447  bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
5449  throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
5450 
5451  add_transaction(null_hash, std::make_pair(b.miner_tx, tx_to_blob(b.miner_tx)));
5452  for (unsigned int j = 0; j<b.tx_hashes.size(); j++) {
5453  transaction tx;
5454  hk.mv_data = &b.tx_hashes[j];
5455  result = mdb_cursor_get(c_txs, &hk, &v, MDB_SET);
5456  if (result)
5457  throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str()));
5458  bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
5459  if (!parse_and_validate_tx_from_blob(bd, tx))
5460  throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
5461  add_transaction(null_hash, std::make_pair(std::move(tx), bd), &b.tx_hashes[j]);
5462  result = mdb_cursor_del(c_txs, 0);
5463  if (result)
5464  throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str()));
5465  }
5466  i++;
5467  m_height = i;
5468  }
5469  result = mdb_txn_begin(m_env, NULL, 0, txn);
5470  if (result)
5471  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5472  result = mdb_drop(txn, o_txs, 1);
5473  if (result)
5474  throw0(DB_ERROR(lmdb_error("Failed to delete txs from the db: ", result).c_str()));
5475 
5476  RENAME_DB("txr");
5477 
5478  mdb_dbi_close(m_env, m_txs);
5479 
5480  lmdb_db_open(txn, "txs", MDB_INTEGERKEY, m_txs, "Failed to open db handle for txs");
5481 
5482  txn.commit();
5483  } while(0);
5484 
5485  uint32_t version = 1;
5486  v.mv_data = (void *)&version;
5487  v.mv_size = sizeof(version);
5488  MDB_val_str(vk, "version");
5489  result = mdb_txn_begin(m_env, NULL, 0, txn);
5490  if (result)
5491  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5492  result = mdb_put(txn, m_properties, &vk, &v, 0);
5493  if (result)
5494  throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str()));
5495  txn.commit();
5496 }
5497 
5498 void BlockchainLMDB::migrate_1_2()
5499 {
5500  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
5501  uint64_t i, z;
5502  int result;
5503  mdb_txn_safe txn(false);
5504  MDB_val k, v;
5505  char *ptr;
5506 
5507  MGINFO_YELLOW("Migrating blockchain from DB version 1 to 2 - this may take a while:");
5508  MINFO("updating txs_pruned and txs_prunable tables...");
5509 
5510  do {
5511  result = mdb_txn_begin(m_env, NULL, 0, txn);
5512  if (result)
5513  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5514 
5515  MDB_stat db_stats_txs;
5516  MDB_stat db_stats_txs_pruned;
5517  MDB_stat db_stats_txs_prunable;
5518  MDB_stat db_stats_txs_prunable_hash;
5519  if ((result = mdb_stat(txn, m_txs, &db_stats_txs)))
5520  throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str()));
5521  if ((result = mdb_stat(txn, m_txs_pruned, &db_stats_txs_pruned)))
5522  throw0(DB_ERROR(lmdb_error("Failed to query m_txs_pruned: ", result).c_str()));
5523  if ((result = mdb_stat(txn, m_txs_prunable, &db_stats_txs_prunable)))
5524  throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str()));
5525  if ((result = mdb_stat(txn, m_txs_prunable_hash, &db_stats_txs_prunable_hash)))
5526  throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable_hash: ", result).c_str()));
5527  if (db_stats_txs_pruned.ms_entries != db_stats_txs_prunable.ms_entries)
5528  throw0(DB_ERROR("Mismatched sizes for txs_pruned and txs_prunable"));
5529  if (db_stats_txs_pruned.ms_entries == db_stats_txs.ms_entries)
5530  {
5531  txn.commit();
5532  MINFO("txs already migrated");
5533  break;
5534  }
5535 
5536  MINFO("updating txs tables:");
5537 
5538  MDB_cursor *c_old, *c_cur0, *c_cur1, *c_cur2;
5539  i = 0;
5540 
5541  while(1) {
5542  if (!(i % 1000)) {
5543  if (i) {
5544  result = mdb_stat(txn, m_txs, &db_stats_txs);
5545  if (result)
5546  throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str()));
5548  std::cout << i << " / " << (i + db_stats_txs.ms_entries) << " \r" << std::flush;
5549  }
5550  txn.commit();
5551  result = mdb_txn_begin(m_env, NULL, 0, txn);
5552  if (result)
5553  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5554  }
5555  result = mdb_cursor_open(txn, m_txs_pruned, &c_cur0);
5556  if (result)
5557  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str()));
5558  result = mdb_cursor_open(txn, m_txs_prunable, &c_cur1);
5559  if (result)
5560  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str()));
5561  result = mdb_cursor_open(txn, m_txs_prunable_hash, &c_cur2);
5562  if (result)
5563  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_hash: ", result).c_str()));
5564  result = mdb_cursor_open(txn, m_txs, &c_old);
5565  if (result)
5566  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str()));
5567  if (!i) {
5568  i = db_stats_txs_pruned.ms_entries;
5569  }
5570  }
5571  MDB_val_set(k, i);
5572  result = mdb_cursor_get(c_old, &k, &v, MDB_SET);
5573  if (result == MDB_NOTFOUND) {
5574  txn.commit();
5575  break;
5576  }
5577  else if (result)
5578  throw0(DB_ERROR(lmdb_error("Failed to get a record from txs: ", result).c_str()));
5579 
5581  bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
5582  transaction tx;
5583  if (!parse_and_validate_tx_from_blob(bd, tx))
5584  throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
5585  std::stringstream ss;
5586  binary_archive<true> ba(ss);
5587  bool r = tx.serialize_base(ba);
5588  if (!r)
5589  throw0(DB_ERROR("Failed to serialize pruned tx"));
5590  std::string pruned = ss.str();
5591 
5592  if (pruned.size() > bd.size())
5593  throw0(DB_ERROR("Pruned tx is larger than raw tx"));
5594  if (memcmp(pruned.data(), bd.data(), pruned.size()))
5595  throw0(DB_ERROR("Pruned tx is not a prefix of the raw tx"));
5596 
5597  MDB_val nv;
5598  nv.mv_data = (void*)pruned.data();
5599  nv.mv_size = pruned.size();
5600  result = mdb_cursor_put(c_cur0, (MDB_val *)&k, &nv, 0);
5601  if (result)
5602  throw0(DB_ERROR(lmdb_error("Failed to put a record into txs_pruned: ", result).c_str()));
5603 
5604  nv.mv_data = (void*)(bd.data() + pruned.size());
5605  nv.mv_size = bd.size() - pruned.size();
5606  result = mdb_cursor_put(c_cur1, (MDB_val *)&k, &nv, 0);
5607  if (result)
5608  throw0(DB_ERROR(lmdb_error("Failed to put a record into txs_prunable: ", result).c_str()));
5609 
5610  result = mdb_cursor_del(c_old, 0);
5611  if (result)
5612  throw0(DB_ERROR(lmdb_error("Failed to delete a record from txs: ", result).c_str()));
5613 
5614  i++;
5615  }
5616  } while(0);
5617 
5618  uint32_t version = 2;
5619  v.mv_data = (void *)&version;
5620  v.mv_size = sizeof(version);
5621  MDB_val_str(vk, "version");
5622  result = mdb_txn_begin(m_env, NULL, 0, txn);
5623  if (result)
5624  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5625  result = mdb_put(txn, m_properties, &vk, &v, 0);
5626  if (result)
5627  throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str()));
5628  txn.commit();
5629 }
5630 
5631 void BlockchainLMDB::migrate_2_3()
5632 {
5633  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
5634  uint64_t i;
5635  int result;
5636  mdb_txn_safe txn(false);
5637  MDB_val k, v;
5638  char *ptr;
5639 
5640  MGINFO_YELLOW("Migrating blockchain from DB version 2 to 3 - this may take a while:");
5641 
5642  do {
5643  LOG_PRINT_L1("migrating block info:");
5644 
5645  result = mdb_txn_begin(m_env, NULL, 0, txn);
5646  if (result)
5647  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5648 
5649  MDB_stat db_stats;
5650  if ((result = mdb_stat(txn, m_blocks, &db_stats)))
5651  throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
5652  const uint64_t blockchain_height = db_stats.ms_entries;
5653 
5654  MDEBUG("enumerating rct outputs...");
5655  std::vector<uint64_t> distribution(blockchain_height, 0);
5656  bool r = for_all_outputs(0, [&](uint64_t height) {
5657  if (height >= blockchain_height)
5658  {
5659  MERROR("Output found claiming height >= blockchain height");
5660  return false;
5661  }
5662  distribution[height]++;
5663  return true;
5664  });
5665  if (!r)
5666  throw0(DB_ERROR("Failed to build rct output distribution"));
5667  for (size_t i = 1; i < distribution.size(); ++i)
5668  distribution[i] += distribution[i - 1];
5669 
5670  /* the block_info table name is the same but the old version and new version
5671  * have incompatible data. Create a new table. We want the name to be similar
5672  * to the old name so that it will occupy the same location in the DB.
5673  */
5674  MDB_dbi o_block_info = m_block_info;
5675  lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
5676  mdb_set_dupsort(txn, m_block_info, compare_uint64);
5677 
5678  MDB_cursor *c_old, *c_cur;
5679  i = 0;
5680  while(1) {
5681  if (!(i % 1000)) {
5682  if (i) {
5684  std::cout << i << " / " << blockchain_height << " \r" << std::flush;
5685  }
5686  txn.commit();
5687  result = mdb_txn_begin(m_env, NULL, 0, txn);
5688  if (result)
5689  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5690  }
5691  result = mdb_cursor_open(txn, m_block_info, &c_cur);
5692  if (result)
5693  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str()));
5694  result = mdb_cursor_open(txn, o_block_info, &c_old);
5695  if (result)
5696  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str()));
5697  if (!i) {
5698  MDB_stat db_stat;
5699  result = mdb_stat(txn, m_block_info, &db_stats);
5700  if (result)
5701  throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str()));
5702  i = db_stats.ms_entries;
5703  }
5704  }
5705  result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT);
5706  if (result == MDB_NOTFOUND) {
5707  txn.commit();
5708  break;
5709  }
5710  else if (result)
5711  throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str()));
5712  const mdb_block_info_1 *bi_old = (const mdb_block_info_1*)v.mv_data;
5713  mdb_block_info_2 bi;
5714  bi.bi_height = bi_old->bi_height;
5715  bi.bi_timestamp = bi_old->bi_timestamp;
5716  bi.bi_coins = bi_old->bi_coins;
5717  bi.bi_weight = bi_old->bi_weight;
5718  bi.bi_diff = bi_old->bi_diff;
5719  bi.bi_hash = bi_old->bi_hash;
5720  if (bi_old->bi_height >= distribution.size())
5721  throw0(DB_ERROR("Bad height in block_info record"));
5722  bi.bi_cum_rct = distribution[bi_old->bi_height];
5723  MDB_val_set(nv, bi);
5724  result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP);
5725  if (result)
5726  throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str()));
5727  /* we delete the old records immediately, so the overall DB and mapsize should not grow.
5728  * This is a little slower than just letting mdb_drop() delete it all at the end, but
5729  * it saves a significant amount of disk space.
5730  */
5731  result = mdb_cursor_del(c_old, 0);
5732  if (result)
5733  throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str()));
5734  i++;
5735  }
5736 
5737  result = mdb_txn_begin(m_env, NULL, 0, txn);
5738  if (result)
5739  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5740  /* Delete the old table */
5741  result = mdb_drop(txn, o_block_info, 1);
5742  if (result)
5743  throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str()));
5744 
5745  RENAME_DB("block_infn");
5746  mdb_dbi_close(m_env, m_block_info);
5747 
5748  lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
5749  mdb_set_dupsort(txn, m_block_info, compare_uint64);
5750 
5751  txn.commit();
5752  } while(0);
5753 
5754  uint32_t version = 3;
5755  v.mv_data = (void *)&version;
5756  v.mv_size = sizeof(version);
5757  MDB_val_str(vk, "version");
5758  result = mdb_txn_begin(m_env, NULL, 0, txn);
5759  if (result)
5760  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5761  result = mdb_put(txn, m_properties, &vk, &v, 0);
5762  if (result)
5763  throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str()));
5764  txn.commit();
5765 }
5766 
5767 void BlockchainLMDB::migrate_3_4()
5768 {
5769  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
5770  uint64_t i;
5771  int result;
5772  mdb_txn_safe txn(false);
5773  MDB_val k, v;
5774  char *ptr;
5775  bool past_long_term_weight = false;
5776 
5777  MGINFO_YELLOW("Migrating blockchain from DB version 3 to 4 - this may take a while:");
5778 
5779  do {
5780  LOG_PRINT_L1("migrating block info:");
5781 
5782  result = mdb_txn_begin(m_env, NULL, 0, txn);
5783  if (result)
5784  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5785 
5786  MDB_stat db_stats;
5787  if ((result = mdb_stat(txn, m_blocks, &db_stats)))
5788  throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
5789  const uint64_t blockchain_height = db_stats.ms_entries;
5790 
5791  boost::circular_buffer<uint64_t> long_term_block_weights(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE);
5792 
5793  /* the block_info table name is the same but the old version and new version
5794  * have incompatible data. Create a new table. We want the name to be similar
5795  * to the old name so that it will occupy the same location in the DB.
5796  */
5797  MDB_dbi o_block_info = m_block_info;
5798  lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
5799  mdb_set_dupsort(txn, m_block_info, compare_uint64);
5800 
5801 
5802  MDB_cursor *c_blocks;
5803  result = mdb_cursor_open(txn, m_blocks, &c_blocks);
5804  if (result)
5805  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str()));
5806 
5807  MDB_cursor *c_old, *c_cur;
5808  i = 0;
5809  while(1) {
5810  if (!(i % 1000)) {
5811  if (i) {
5813  std::cout << i << " / " << blockchain_height << " \r" << std::flush;
5814  }
5815  txn.commit();
5816  result = mdb_txn_begin(m_env, NULL, 0, txn);
5817  if (result)
5818  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5819  }
5820  result = mdb_cursor_open(txn, m_block_info, &c_cur);
5821  if (result)
5822  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str()));
5823  result = mdb_cursor_open(txn, o_block_info, &c_old);
5824  if (result)
5825  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str()));
5826  result = mdb_cursor_open(txn, m_blocks, &c_blocks);
5827  if (result)
5828  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str()));
5829  if (!i) {
5830  MDB_stat db_stat;
5831  result = mdb_stat(txn, m_block_info, &db_stats);
5832  if (result)
5833  throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str()));
5834  i = db_stats.ms_entries;
5835  }
5836  }
5837  result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT);
5838  if (result == MDB_NOTFOUND) {
5839  txn.commit();
5840  break;
5841  }
5842  else if (result)
5843  throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str()));
5844  const mdb_block_info_2 *bi_old = (const mdb_block_info_2*)v.mv_data;
5845  mdb_block_info_3 bi;
5846  bi.bi_height = bi_old->bi_height;
5847  bi.bi_timestamp = bi_old->bi_timestamp;
5848  bi.bi_coins = bi_old->bi_coins;
5849  bi.bi_weight = bi_old->bi_weight;
5850  bi.bi_diff = bi_old->bi_diff;
5851  bi.bi_hash = bi_old->bi_hash;
5852  bi.bi_cum_rct = bi_old->bi_cum_rct;
5853 
5854  // get block major version to determine which rule is in place
5855  if (!past_long_term_weight)
5856  {
5857  MDB_val_copy<uint64_t> kb(bi.bi_height);
5858  MDB_val vb;
5859  result = mdb_cursor_get(c_blocks, &kb, &vb, MDB_SET);
5860  if (result)
5861  throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
5862  if (vb.mv_size == 0)
5863  throw0(DB_ERROR("Invalid data from m_blocks"));
5864  const uint8_t block_major_version = *((const uint8_t*)vb.mv_data);
5865  if (block_major_version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT)
5866  past_long_term_weight = true;
5867  }
5868 
5869  uint64_t long_term_block_weight;
5870  if (past_long_term_weight)
5871  {
5872  std::vector<uint64_t> weights(long_term_block_weights.begin(), long_term_block_weights.end());
5873  uint64_t long_term_effective_block_median_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, epee::misc_utils::median(weights));
5874  long_term_block_weight = std::min<uint64_t>(bi.bi_weight, long_term_effective_block_median_weight + long_term_effective_block_median_weight * 2 / 5);
5875  }
5876  else
5877  {
5878  long_term_block_weight = bi.bi_weight;
5879  }
5880  long_term_block_weights.push_back(long_term_block_weight);
5881  bi.bi_long_term_block_weight = long_term_block_weight;
5882 
5883  MDB_val_set(nv, bi);
5884  result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP);
5885  if (result)
5886  throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str()));
5887  /* we delete the old records immediately, so the overall DB and mapsize should not grow.
5888  * This is a little slower than just letting mdb_drop() delete it all at the end, but
5889  * it saves a significant amount of disk space.
5890  */
5891  result = mdb_cursor_del(c_old, 0);
5892  if (result)
5893  throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str()));
5894  i++;
5895  }
5896 
5897  result = mdb_txn_begin(m_env, NULL, 0, txn);
5898  if (result)
5899  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5900  /* Delete the old table */
5901  result = mdb_drop(txn, o_block_info, 1);
5902  if (result)
5903  throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str()));
5904 
5905  RENAME_DB("block_infn");
5906  mdb_dbi_close(m_env, m_block_info);
5907 
5908  lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
5909  mdb_set_dupsort(txn, m_block_info, compare_uint64);
5910 
5911  txn.commit();
5912  } while(0);
5913 
5914  uint32_t version = 4;
5915  v.mv_data = (void *)&version;
5916  v.mv_size = sizeof(version);
5917  MDB_val_str(vk, "version");
5918  result = mdb_txn_begin(m_env, NULL, 0, txn);
5919  if (result)
5920  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5921  result = mdb_put(txn, m_properties, &vk, &v, 0);
5922  if (result)
5923  throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str()));
5924  txn.commit();
5925 }
5926 
5927 void BlockchainLMDB::migrate_4_5()
5928 {
5929  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
5930  uint64_t i;
5931  int result;
5932  mdb_txn_safe txn(false);
5933  MDB_val k, v;
5934  char *ptr;
5935 
5936  MGINFO_YELLOW("Migrating blockchain from DB version 4 to 5 - this may take a while:");
5937 
5938  do {
5939  LOG_PRINT_L1("migrating block info:");
5940 
5941  result = mdb_txn_begin(m_env, NULL, 0, txn);
5942  if (result)
5943  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5944 
5945  MDB_stat db_stats;
5946  if ((result = mdb_stat(txn, m_blocks, &db_stats)))
5947  throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
5948  const uint64_t blockchain_height = db_stats.ms_entries;
5949 
5950  /* the block_info table name is the same but the old version and new version
5951  * have incompatible data. Create a new table. We want the name to be similar
5952  * to the old name so that it will occupy the same location in the DB.
5953  */
5954  MDB_dbi o_block_info = m_block_info;
5955  lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
5956  mdb_set_dupsort(txn, m_block_info, compare_uint64);
5957 
5958 
5959  MDB_cursor *c_blocks;
5960  result = mdb_cursor_open(txn, m_blocks, &c_blocks);
5961  if (result)
5962  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str()));
5963 
5964  MDB_cursor *c_old, *c_cur;
5965  i = 0;
5966  while(1) {
5967  if (!(i % 1000)) {
5968  if (i) {
5970  std::cout << i << " / " << blockchain_height << " \r" << std::flush;
5971  }
5972  txn.commit();
5973  result = mdb_txn_begin(m_env, NULL, 0, txn);
5974  if (result)
5975  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
5976  }
5977  result = mdb_cursor_open(txn, m_block_info, &c_cur);
5978  if (result)
5979  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str()));
5980  result = mdb_cursor_open(txn, o_block_info, &c_old);
5981  if (result)
5982  throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str()));
5983  if (!i) {
5984  MDB_stat db_stat;
5985  result = mdb_stat(txn, m_block_info, &db_stats);
5986  if (result)
5987  throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str()));
5988  i = db_stats.ms_entries;
5989  }
5990  }
5991  result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT);
5992  if (result == MDB_NOTFOUND) {
5993  txn.commit();
5994  break;
5995  }
5996  else if (result)
5997  throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str()));
5998  const mdb_block_info_3 *bi_old = (const mdb_block_info_3*)v.mv_data;
5999  mdb_block_info_4 bi;
6000  bi.bi_height = bi_old->bi_height;
6001  bi.bi_timestamp = bi_old->bi_timestamp;
6002  bi.bi_coins = bi_old->bi_coins;
6003  bi.bi_weight = bi_old->bi_weight;
6004  bi.bi_diff_lo = bi_old->bi_diff;
6005  bi.bi_diff_hi = 0;
6006  bi.bi_hash = bi_old->bi_hash;
6007  bi.bi_cum_rct = bi_old->bi_cum_rct;
6008  bi.bi_long_term_block_weight = bi_old->bi_long_term_block_weight;
6009 
6010  MDB_val_set(nv, bi);
6011  result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP);
6012  if (result)
6013  throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str()));
6014  /* we delete the old records immediately, so the overall DB and mapsize should not grow.
6015  * This is a little slower than just letting mdb_drop() delete it all at the end, but
6016  * it saves a significant amount of disk space.
6017  */
6018  result = mdb_cursor_del(c_old, 0);
6019  if (result)
6020  throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str()));
6021  i++;
6022  }
6023 
6024  result = mdb_txn_begin(m_env, NULL, 0, txn);
6025  if (result)
6026  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
6027  /* Delete the old table */
6028  result = mdb_drop(txn, o_block_info, 1);
6029  if (result)
6030  throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str()));
6031 
6032  RENAME_DB("block_infn");
6033  mdb_dbi_close(m_env, m_block_info);
6034 
6035  lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
6036  mdb_set_dupsort(txn, m_block_info, compare_uint64);
6037 
6038  txn.commit();
6039  } while(0);
6040 
6041  uint32_t version = 5;
6042  v.mv_data = (void *)&version;
6043  v.mv_size = sizeof(version);
6044  MDB_val_str(vk, "version");
6045  result = mdb_txn_begin(m_env, NULL, 0, txn);
6046  if (result)
6047  throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
6048  result = mdb_put(txn, m_properties, &vk, &v, 0);
6049  if (result)
6050  throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str()));
6051  txn.commit();
6052 }
6053 
6054 void BlockchainLMDB::migrate(const uint32_t oldversion)
6055 {
6056  if (oldversion < 1)
6057  migrate_0_1();
6058  if (oldversion < 2)
6059  migrate_1_2();
6060  if (oldversion < 3)
6061  migrate_2_3();
6062  if (oldversion < 4)
6063  migrate_3_4();
6064  if (oldversion < 5)
6065  migrate_4_5();
6066 }
6067 
6068 } // namespace cryptonote
#define TXN_POSTFIX_SUCCESS()
Definition: db_lmdb.cpp:1797
const char * res
Definition: hmac_keccak.cpp:41
virtual void block_wtxn_start()
Definition: db_lmdb.cpp:4406
virtual void safesyncmode(const bool onoff)
toggle safe syncs for the DB
Definition: db_lmdb.cpp:1656
#define MDB_NOTFOUND
Definition: lmdb.h:439
#define MERROR(x)
Definition: misc_log_ex.h:73
#define m_cur_block_heights
Definition: db_lmdb.h:82
#define m_cur_validators
Definition: db_lmdb.h:98
virtual bool get_tx_blob(const crypto::hash &h, cryptonote::blobdata &tx) const
fetches the transaction blob with the given hash
Definition: db_lmdb.cpp:3655
virtual void set_batch_transactions(bool batch_transactions)
sets whether or not to batch transactions
Definition: db_lmdb.cpp:4343
#define m_cur_tx_inputs
Definition: db_lmdb.h:102
bool is_coinbase(const transaction &tx)
#define CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE
#define MDB_PREVSNAPSHOT
Definition: lmdb.h:336
int lmdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
Definition: db_lmdb.cpp:527
#define m_cur_tx_outputs
Definition: db_lmdb.h:92
mdb_threadinfo * m_tinfo
Definition: db_lmdb.h:170
#define LOG_PRINT_L2(x)
Definition: misc_log_ex.h:101
const uint32_t T[512]
int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
Set the size of the memory map to use for this environment.
void mdb_cursor_close(MDB_cursor *cursor)
Close a cursor handle.
virtual std::vector< std::string > get_filenames() const
get all files used by the BlockchainDB (if any)
Definition: db_lmdb.cpp:1724
Definition: lmdb.h:411
virtual std::vector< address_outputs > get_addr_output_all(const crypto::public_key &combined_key)
Definition: db_lmdb.cpp:2071
struct cryptonote::mdb_block_info_4 mdb_block_info_4
#define MDB_RDONLY
Definition: lmdb.h:320
struct cryptonote::acc_addr_tx_t acc_addr_tx_t
int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp)
Return count of duplicates for current key.
virtual uint64_t get_tx_unlock_time(const crypto::hash &h) const
fetch a transaction&#39;s unlock time/height
Definition: db_lmdb.cpp:3634
#define DBF_RDONLY
#define MTRACE(x)
Definition: misc_log_ex.h:77
#define MINFO(x)
Definition: misc_log_ex.h:75
int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi)
Open a database in the environment.
virtual std::vector< transaction > get_tx_list(const std::vector< crypto::hash > &hlist) const
fetches a list of transactions based on their hashes
Definition: db_lmdb.cpp:3798
virtual void block_rtxn_abort() const
Definition: db_lmdb.cpp:4477
#define MDB_NODUPDATA
Definition: lmdb.h:369
#define CRYPTONOTE_BLOCKCHAINDATA_LOCK_FILENAME
#define m_cur_spent_keys
Definition: db_lmdb.h:93
virtual uint64_t get_block_long_term_weight(const uint64_t &height) const
fetch a block&#39;s long term weight
Definition: db_lmdb.cpp:3428
#define LOG_PRINT_L1(x)
Definition: misc_log_ex.h:100
#define DBF_SALVAGE
struct cryptonote::mdb_block_info_2 mdb_block_info_2
static int compare_hash32(const MDB_val *a, const MDB_val *b)
Definition: db_lmdb.cpp:151
tx_data_t data
Definition: db_lmdb.h:46
#define MFATAL(x)
Definition: misc_log_ex.h:72
virtual void set_block_cumulative_difficulty(uint64_t height, difficulty_type diff)
sets a block&#39;s cumulative difficulty
Definition: db_lmdb.cpp:3327
::std::string string
Definition: gtest-port.h:1097
#define m_cur_output_txs
Definition: db_lmdb.h:84
struct cryptonote::txindex txindex
uint32_t get_random_stripe()
Definition: pruning.cpp:110
int mdb_txn_commit(MDB_txn *txn)
Commit all the operations of a transaction into the database.
int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, unsigned int flags)
Store by cursor.
int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat)
Retrieve statistics for a database.
virtual std::string get_db_name() const
gets the name of the folder the BlockchainDB&#39;s file(s) should be in
Definition: db_lmdb.cpp:1755
#define DBF_FASTEST
#define TXN_POSTFIX_RDONLY()
Definition: db_lmdb.cpp:1795
void mdb_dbi_close(MDB_env *env, MDB_dbi dbi)
Close a database handle. Normally unnecessary. Use with care:
#define m_cur_hf_versions
Definition: db_lmdb.h:96
#define LOGIF(y)
Definition: db_lmdb.cpp:4954
virtual uint64_t get_balance(const crypto::public_key &combined_key)
Definition: db_lmdb.cpp:2298
uint64_t height
Definition: blockchain.cpp:91
virtual block_header get_block_header(const crypto::hash &h) const
fetch a block header
Definition: db_lmdb.cpp:3053
crypto::hash key
Definition: db_lmdb.h:45
virtual bool for_all_outputs(std::function< bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const
runs a function over all outputs stored
Definition: db_lmdb.cpp:4135
#define CRYPTONOTE_PRUNING_LOG_STRIPES
pre_rct_output_data_t data
Definition: db_lmdb.cpp:365
virtual std::vector< address_txs > get_addr_tx_all(const crypto::public_key &combined_key)
Definition: db_lmdb.cpp:2214
virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0)
tells the BlockchainDB to start a new "batch" of blocks
Definition: db_lmdb.cpp:4211
#define ETN_DEFAULT_TX_SPENDABLE_AGE_V8
virtual bool update_pruning()
prunes recent blockchain changes as needed, iff pruning is enabled
Definition: db_lmdb.cpp:2931
output_data_t data
Definition: db_lmdb.cpp:371
epee::critical_section m_synchronization_lock
A lock, currently for when BlockchainLMDB needs to resize the backing db file.
bool is_v1_tx(const blobdata &tx_blob)
const char * key
Definition: hmac_keccak.cpp:39
int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
Set a custom data comparison function for a MDB_DUPSORT database.
#define MCLOG_RED(level, cat, x)
Definition: misc_log_ex.h:58
#define LOG_PRINT_L0(x)
Definition: misc_log_ex.h:99
#define RENAME_DB(name)
Definition: db_lmdb.cpp:4932
crypto namespace.
Definition: crypto.cpp:58
int mdb_env_info(MDB_env *env, MDB_envinfo *stat)
Return information about the LMDB environment.
thrown when there is an error starting a DB transaction
#define m_cur_txpool_blob
Definition: db_lmdb.h:95
int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs)
Set the maximum number of named databases for the environment.
Non-owning sequence of data. Does not deep copy.
Definition: span.h:56
#define CRYPTONOTE_BLOCKCHAINDATA_FILENAME
struct MDB_env MDB_env
Opaque structure for a database environment.
Definition: lmdb.h:260
mdb_size_t ms_branch_pages
Definition: lmdb.h:494
virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta)
add a txpool transaction
Definition: db_lmdb.cpp:2415
unsigned char uint8_t
Definition: stdint.h:124
virtual std::vector< std::vector< uint64_t > > get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes) const
gets output indices (amount-specific) for a transaction&#39;s outputs
Definition: db_lmdb.cpp:3929
crypto::hash tx_hash
Definition: db_lmdb.cpp:382
MDB_env * mdb_txn_env(MDB_txn *txn)
Returns the transaction&#39;s MDB_env.
static int compare_data(const MDB_val *a, const MDB_val *b)
Definition: db_lmdb.cpp:172
struct MDB_val MDB_val
Generic structure used for passing keys and data in and out of the database.
#define m_cur_txs_prunable_hash
Definition: db_lmdb.h:89
virtual uint64_t get_block_height(const crypto::hash &h) const
gets the height of the block with a given hash
Definition: db_lmdb.cpp:3032
const char * name
struct hash_func hashes[]
#define DBF_FAST
crypto::hash tx_hash
Definition: db_lmdb.cpp:376
#define MGINFO(x)
Definition: misc_log_ex.h:80
tuple make_tuple()
Definition: gtest-tuple.h:675
virtual bool block_rtxn_start() const
Definition: db_lmdb.cpp:4399
virtual bool tx_exists(const crypto::hash &h) const
check if a transaction with a given hash exists
Definition: db_lmdb.cpp:3568
virtual bool get_txpool_tx_meta(const crypto::hash &txid, txpool_tx_meta_t &meta) const
get a txpool transaction&#39;s metadata
Definition: db_lmdb.cpp:2558
#define m_cur_utxos
Definition: db_lmdb.h:99
crypto::hash bh_hash
Definition: db_lmdb.cpp:358
constexpr uint32_t get_pruning_log_stripes(uint32_t pruning_seed)
Definition: pruning.h:40
struct cryptonote::blk_height blk_height
#define MDB_MAPASYNC
Definition: lmdb.h:326
struct cryptonote::mdb_block_info_3 mdb_block_info_3
#define CRYPTONOTE_PRUNING_TIP_BLOCKS
void add_transaction(const crypto::hash &blk_hash, const std::pair< transaction, blobdata > &tx, const crypto::hash *tx_hash_ptr=NULL, const crypto::hash *tx_prunable_hash_ptr=NULL)
helper function for add_transactions, to add each individual transaction
#define TXN_BLOCK_POSTFIX_SUCCESS()
Definition: db_lmdb.cpp:1823
#define MDEBUG(x)
Definition: misc_log_ex.h:76
struct MDB_txn MDB_txn
Opaque structure for a transaction handle.
Definition: lmdb.h:267
#define MDB_CREATE
Definition: lmdb.h:357
unsigned get_max_concurrency()
Definition: util.cpp:868
virtual tx_input_t get_tx_input(const crypto::hash tx_hash, const uint32_t relative_out_index)
Definition: db_lmdb.cpp:1973
#define MDB_CURRENT
Definition: lmdb.h:371
thrown when a requested transaction does not exist
virtual bool txpool_has_tx(const crypto::hash &txid) const
check whether a txid is in the txpool
Definition: db_lmdb.cpp:2512
constexpr std::size_t size() const noexcept
Definition: span.h:111
virtual difficulty_type get_block_difficulty(const uint64_t &height) const
fetch a block&#39;s difficulty
Definition: db_lmdb.cpp:3388
Holds cryptonote related classes and helpers.
Definition: ban.cpp:40
uint64_t local_index
Definition: db_lmdb.cpp:377
int mdb_env_get_flags(MDB_env *env, unsigned int *flags)
Get environment flags.
virtual bool get_prunable_tx_hash(const crypto::hash &tx_hash, crypto::hash &prunable_hash) const
fetches the prunable transaction hash
Definition: db_lmdb.cpp:3751
virtual std::vector< block > get_blocks_range(const uint64_t &h1, const uint64_t &h2) const
fetch a list of blocks
Definition: db_lmdb.cpp:3474
int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
Empty or delete+close a database.
#define HF_VERSION_LONG_TERM_BLOCK_WEIGHT
void lmdb_resized(MDB_env *env)
Definition: db_lmdb.cpp:502
virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t &index) const
gets an output&#39;s tx hash and index
Definition: db_lmdb.cpp:3893
blobdata tx_to_blob(const transaction &tx)
#define MDB_val_set(var, val)
Definition: db_lmdb.cpp:89
#define MDB_MAP_RESIZED
Definition: lmdb.h:465
int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode)
Open an environment handle.
mdb_size_t count(MDB_cursor *cur)
Statistics for a database in the environment.
Definition: lmdb.h:490
time_t time
Definition: blockchain.cpp:93
#define TXN_BLOCK_PREFIX(flags)
Definition: db_lmdb.cpp:1812
Information about the environment.
Definition: lmdb.h:501
virtual uint32_t get_blockchain_pruning_seed() const
get the blockchain pruning seed
Definition: db_lmdb.cpp:2608
static std::atomic_flag creation_gate
Definition: db_lmdb.h:177
std::string pod_to_hex(const t_pod_type &s)
Definition: string_tools.h:317
virtual uint64_t get_block_already_generated_coins(const uint64_t &height) const
fetch a block&#39;s already generated coins
Definition: db_lmdb.cpp:3405
void * mv_data
Definition: lmdb.h:288
bool serialize(Archive &ar, T &v)
#define TXN_PREFIX(flags)
Definition: db_lmdb.cpp:1777
#define MDB_SUCCESS
Definition: lmdb.h:435
virtual void block_rtxn_stop() const
Definition: db_lmdb.cpp:4392
void commit(std::string message="")
Definition: db_lmdb.cpp:453
virtual block get_block_from_height(const uint64_t &height) const
fetch a block by height
virtual bool block_exists(const crypto::hash &h, uint64_t *height=NULL) const
checks if a block exists
Definition: db_lmdb.cpp:2993
void mdb_txn_reset(MDB_txn *txn)
Reset a read-only transaction.
unsigned int uint32_t
Definition: stdint.h:126
virtual bool for_all_key_images(std::function< bool(const crypto::key_image &)>) const
runs a function over all key images stored
Definition: db_lmdb.cpp:3989
#define MDB_WRITEMAP
Definition: lmdb.h:324
virtual bool get_prunable_tx_blob(const crypto::hash &h, cryptonote::blobdata &tx) const
fetches the prunable transaction blob with the given hash
Definition: db_lmdb.cpp:3721
int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags)
Retrieve the DB flags for a database handle.
bool get_block_hash(const block &b, crypto::hash &res)
size_t mv_size
Definition: lmdb.h:287
uint64_t time_tx_exists
a performance metric
Definition: lmdb.h:422
static int compare_uint64(const MDB_val *a, const MDB_val *b)
Definition: db_lmdb.cpp:143
virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta)
update a txpool transaction&#39;s metadata
Definition: db_lmdb.cpp:2441
virtual size_t get_block_weight(const uint64_t &height) const
fetch a block&#39;s weight
Definition: db_lmdb.cpp:3188
virtual void close()
close the BlockchainDB
Definition: db_lmdb.cpp:1624
#define m_cur_txpool_meta
Definition: db_lmdb.h:94
#define RCURSOR(name)
Definition: db_lmdb.cpp:292
#define m_cur_output_amounts
Definition: db_lmdb.h:85
int mdb_txn_renew(MDB_txn *txn)
Renew a read-only transaction.
virtual transaction get_tx(const crypto::hash &h) const
fetches the transaction with the given hash
struct cryptonote::acc_outs_t acc_outs_t
virtual bool get_txpool_tx_blob(const crypto::hash &txid, cryptonote::blobdata &bd) const
get a txpool transaction&#39;s blob
Definition: db_lmdb.cpp:2579
mdb_size_t me_last_pgno
Definition: lmdb.h:504
unsigned __int64 uint64_t
Definition: stdint.h:136
#define MGINFO_YELLOW(x)
Definition: misc_log_ex.h:83
virtual uint64_t add_block(const std::pair< block, blobdata > &blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type &cumulative_difficulty, const uint64_t &coins_generated, const std::vector< std::pair< transaction, blobdata >> &txs)
handles the addition of a new block to BlockchainDB
Definition: db_lmdb.cpp:4484
virtual std::vector< address_txs > get_addr_tx_batch(const crypto::public_key &combined_key, uint64_t start_db_index=0, uint64_t batch_size=100, bool desc=false)
Definition: db_lmdb.cpp:2249
static void wait_no_active_txns()
Definition: db_lmdb.cpp:492
#define CRITICAL_REGION_LOCAL(x)
Definition: syncobj.h:228
bool t_serializable_object_to_blob(const t_object &to, blobdata &b_blob)
virtual void batch_stop()
ends a batch transaction
Definition: db_lmdb.cpp:4291
virtual bool remove_data_file(const std::string &folder) const
remove file(s) storing the database
Definition: db_lmdb.cpp:1740
uint64_t height
the height of the block which created the output
#define MDB_INTEGERKEY
Definition: lmdb.h:349
#define MDB_DUPFIXED
Definition: lmdb.h:351
int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
Set a custom key comparison function for a database.
mdb_size_t ms_overflow_pages
Definition: lmdb.h:496
#define m_cur_txs_pruned
Definition: db_lmdb.h:87
virtual uint64_t get_block_timestamp(const uint64_t &height) const
fetch a block&#39;s timestamp
Definition: db_lmdb.cpp:3088
struct cryptonote::outtx outtx
thrown when a requested block does not exist
#define TIME_MEASURE_START(var_name)
Definition: profile_tools.h:61
unsigned int MDB_dbi
A handle for an individual database in the DB environment.
Definition: lmdb.h:270
int mdb_env_sync(MDB_env *env, int force)
Flush the data buffers to disk.
std::map< uint64_t, std::tuple< uint64_t, uint64_t, uint64_t > > get_output_histogram(const std::vector< uint64_t > &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const
return a histogram of outputs on the blockchain
Definition: db_lmdb.cpp:4651
virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes=true) const
get the number of transactions in the txpool
Definition: db_lmdb.cpp:2467
bool parse_and_validate_tx_base_from_blob(const blobdata &tx_blob, transaction &tx)
bool m_open
Whether or not the BlockchainDB is open/ready for use.
virtual bool lock()
acquires the BlockchainDB lock
Definition: db_lmdb.cpp:1763
bool parse_and_validate_tx_from_blob(const blobdata &tx_blob, transaction &tx)
virtual void unlock()
This function releases the BlockchainDB lock.
Definition: db_lmdb.cpp:1771
#define m_cur_addr_txs
Definition: db_lmdb.h:101
uint64_t amount_index
Definition: db_lmdb.cpp:369
#define LOG_PRINT_L3(x)
Definition: misc_log_ex.h:102
POD_CLASS public_key
Definition: crypto.h:76
int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
Create a transaction for use with the environment.
MDB_cursor_op
Cursor Get operations.
Definition: lmdb.h:398
Useful when application has potentially harmful situtaions.
version
Supported socks variants.
Definition: socks.h:57
rct::key commitment
the output&#39;s amount commitment (for spend verification)
#define MWARNING(x)
Definition: misc_log_ex.h:74
#define TIME_MEASURE_FINISH(var_name)
Definition: profile_tools.h:64
int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, MDB_cursor_op op)
Retrieve by cursor.
virtual difficulty_type get_block_cumulative_difficulty(const uint64_t &height) const
fetch a block&#39;s cumulative difficulty
Definition: db_lmdb.cpp:3363
std::string message("Message requiring signing")
virtual void open(const std::string &filename, const int mdb_flags=0)
open a db, or create it if necessary.
Definition: db_lmdb.cpp:1357
int lmdb_txn_renew(MDB_txn *txn)
Definition: db_lmdb.cpp:537
mdb_txn_cursors m_ti_rcursors
Definition: db_lmdb.h:134
int mdb_env_create(MDB_env **env)
Create an LMDB environment handle.
void mdb_txn_abort(MDB_txn *txn)
Abandon all the operations of the transaction instead of saving them.
const GenericPointer< typename T::ValueType > T2 T::AllocatorType & a
Definition: pointer.h:1124
std::string blobdata
Definition: blobdatatype.h:39
mdb_size_t ms_leaf_pages
Definition: lmdb.h:495
type_vec_type median(std::vector< type_vec_type > &v)
mdb_block_info_4 mdb_block_info
Definition: db_lmdb.cpp:355
std::string to_string_hex(uint32_t val)
Definition: string_tools.h:211
#define MDB_APPEND
Definition: lmdb.h:377
virtual bool for_all_txpool_txes(std::function< bool(const crypto::hash &, const txpool_tx_meta_t &, const cryptonote::blobdata *)> f, bool include_blob=false, bool include_unrelayed_txes=true) const
runs a function over all txpool transactions
Definition: db_lmdb.cpp:2941
A generic BlockchainDB exception.
#define m_cur_block_info
Definition: db_lmdb.h:83
Mainly useful to represent current progress of application.
struct MDB_cursor MDB_cursor
Opaque structure for navigating through a database.
Definition: lmdb.h:273
virtual void remove_txpool_tx(const crypto::hash &txid)
remove a txpool transaction
Definition: db_lmdb.cpp:2528
The BlockchainDB backing store interface declaration/contract.
#define m_cur_txs_prunable_tip
Definition: db_lmdb.h:90
Generic structure used for passing keys and data in and out of the database.
Definition: lmdb.h:286
#define TXN_PREFIX_RDONLY()
Definition: db_lmdb.cpp:1788
static int compare_publickey(const MDB_val *a, const MDB_val *b)
Definition: db_lmdb.cpp:188
const T & move(const T &t)
Definition: gtest-port.h:1317
struct cryptonote::outkey outkey
#define VERSION
Definition: db_lmdb.cpp:60
boost::multiprecision::uint128_t difficulty_type
Definition: difficulty.h:43
unsigned int ms_psize
Definition: lmdb.h:491
POD_CLASS key_image
Definition: crypto.h:102
uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes)
Definition: pruning.cpp:54
a struct containing txpool per transaction metadata
void * memcpy(void *a, const void *b, size_t c)
#define CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE
#define m_cur_addr_outputs
Definition: db_lmdb.h:100
virtual bool prune_blockchain(uint32_t pruning_seed=0)
prunes the blockchain
Definition: db_lmdb.cpp:2926
virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t &height) const
fetch a block blob by height
Definition: db_lmdb.cpp:3062
bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed)
Definition: pruning.cpp:44
#define MDB_val_sized(var, val)
Definition: db_lmdb.cpp:91
#define MDB_KEYEXIST
Definition: lmdb.h:437
virtual void batch_abort()
aborts a batch transaction
Definition: db_lmdb.cpp:4320
virtual bool for_all_transactions(std::function< bool(const crypto::hash &, const cryptonote::transaction &)>, bool pruned) const
runs a function over all transactions stored
Definition: db_lmdb.cpp:4073
virtual void fixup()
fix up anything that may be wrong due to past bugs
Definition: lmdb.h:408
virtual std::vector< crypto::hash > get_hashes_range(const uint64_t &h1, const uint64_t &h2) const
fetch a list of block hashes
Definition: db_lmdb.cpp:3488
#define AUTO_VAL_INIT(v)
Definition: misc_language.h:53
virtual block get_top_block() const
fetch the top block
Definition: db_lmdb.cpp:3517
int mdb_env_stat(MDB_env *env, MDB_stat *stat)
Return statistics about the LMDB environment.
char * mdb_strerror(int err)
Return a string describing a given error code.
int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers)
Set the maximum number of threads/reader slots for the environment.
virtual uint64_t get_top_block_timestamp() const
fetch the top block&#39;s timestamp
Definition: db_lmdb.cpp:3173
virtual bool for_blocks_range(const uint64_t &h1, const uint64_t &h2, std::function< bool(uint64_t, const crypto::hash &, const cryptonote::block &)>) const
runs a function over a range of blocks
Definition: db_lmdb.cpp:4022
mdb_size_t me_mapsize
Definition: lmdb.h:503
key zeroCommit(etn_amount amount)
Definition: rctOps.cpp:322
uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes)
Definition: pruning.cpp:37
uint64_t relative_out_index
Definition: db_lmdb.cpp:383
int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff)
Set environment flags.
thrown when a requested output does not exist
int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags)
Delete current key/data pair.
POD_CLASS hash
Definition: hash.h:50
virtual crypto::hash get_block_hash_from_height(const uint64_t &height) const
fetch a block&#39;s hash
Definition: db_lmdb.cpp:3451
virtual void batch_commit()
Definition: db_lmdb.cpp:4254
#define MDB_val_str(var, val)
Definition: db_lmdb.cpp:93
std::pair< crypto::hash, uint64_t > tx_out_index
std::string to_string(t_connection_type type)
uint64_t num_active_tx() const
Definition: db_lmdb.cpp:482
uint64_t time_commit1
a performance metric
bool get_file_size(const std::string &path_to_file, uint64_t &size)
struct cryptonote::mdb_block_info_1 mdb_block_info_1
#define m_cur_properties
Definition: db_lmdb.h:97
else if(0==res)
#define MDB_DUPSORT
Definition: lmdb.h:345
thrown when opening the BlockchainDB fails
uint64_t output_id
Definition: db_lmdb.cpp:370
struct cryptonote::mdb_txn_cursors mdb_txn_cursors
#define CURSOR(name)
Definition: db_lmdb.cpp:285
blobdata block_to_blob(const block &b)
size_t mdb_size_t
Definition: lmdb.h:196
static void prevent_new_txns()
Definition: db_lmdb.cpp:487
int compare_uint64(const MDB_val *a, const MDB_val *b)
#define DELETE_DB(x)
mdb_size_t ms_entries
Definition: lmdb.h:497
#define m_cur_tx_indices
Definition: db_lmdb.h:91
virtual void sync()
sync the BlockchainDB with disk
Definition: db_lmdb.cpp:1640
#define m_cur_txs_prunable
Definition: db_lmdb.h:88
#define m_cur_blocks
Definition: db_lmdb.h:81
virtual cryptonote::blobdata get_block_blob(const crypto::hash &h) const
fetches the block with the given hash
Definition: db_lmdb.cpp:3024
virtual block get_block(const crypto::hash &h) const
fetches the block with the given hash
#define DBF_ADDR_TX_SALVAGE
virtual std::vector< uint64_t > get_long_term_block_weights(uint64_t start_height, size_t count) const
fetch the last N blocks&#39; long term weights
Definition: db_lmdb.cpp:3322
virtual uint64_t get_tx_block_height(const crypto::hash &h) const
fetches the height of a transaction&#39;s block
Definition: db_lmdb.cpp:3812
uint64_t output_id
Definition: db_lmdb.cpp:375
virtual uint64_t get_num_outputs(const uint64_t &amount) const
fetches the number of outputs of a given amount
Definition: db_lmdb.cpp:3835
int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data)
Get items from a database.
virtual std::vector< address_outputs > get_addr_output_batch(const crypto::public_key &combined_key, uint64_t start_db_index=0, uint64_t batch_size=100, bool desc=false)
Definition: db_lmdb.cpp:2112
#define MDB_NOSYNC
Definition: lmdb.h:318
virtual bool has_key_image(const crypto::key_image &img) const
check if a key image is stored as spent
Definition: db_lmdb.cpp:3972
#define MDB_NORDAHEAD
Definition: lmdb.h:332
a struct containing output metadata
struct cryptonote::mdb_threadinfo mdb_threadinfo
virtual output_data_t get_output_key(const uint64_t &amount, const uint64_t &index, bool include_commitmemt) const
get some of an output&#39;s data
Definition: db_lmdb.cpp:3859
bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector< uint64_t > &distribution, uint64_t &base) const
Definition: db_lmdb.cpp:4744
virtual void reset()
Remove everything from the BlockchainDB.
Definition: db_lmdb.cpp:1662
virtual void block_wtxn_stop()
Definition: db_lmdb.cpp:4439
int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned int flags)
Store items into a database.
#define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5
void mdb_env_close(MDB_env *env)
Close the environment and release the memory map.
std::atomic< unsigned int > unprunable_size
int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor)
Create a cursor handle.
#define MDB_APPENDDUP
Definition: lmdb.h:379
static int compare_string(const MDB_val *a, const MDB_val *b)
Definition: db_lmdb.cpp:165
virtual uint64_t get_tx_count() const
fetches the total number of transactions ever
Definition: db_lmdb.cpp:3781
bool parse_and_validate_block_from_blob(const blobdata &b_blob, block &b, crypto::hash *block_hash)
virtual uint64_t height() const
fetch the current blockchain height
Definition: db_lmdb.cpp:3532
virtual std::vector< uint64_t > get_block_weights(uint64_t start_height, size_t count) const
fetch the last N blocks&#39; weights
Definition: db_lmdb.cpp:3317
virtual std::vector< uint64_t > get_block_cumulative_rct_outputs(const std::vector< uint64_t > &heights) const
fetch a block&#39;s cumulative number of rct outputs
Definition: db_lmdb.cpp:3111
static std::atomic< uint64_t > num_active_txns
Definition: db_lmdb.h:174
static void allow_new_txns()
Definition: db_lmdb.cpp:497
virtual bool get_pruned_tx_blob(const crypto::hash &h, cryptonote::blobdata &tx) const
fetches the pruned transaction blob with the given hash
Definition: db_lmdb.cpp:3691
virtual bool check_pruning()
checks pruning was done correctly, iff enabled
Definition: db_lmdb.cpp:2936
virtual void block_wtxn_abort()
Definition: db_lmdb.cpp:4461
struct cryptonote::pre_rct_outkey pre_rct_outkey
virtual tx_out_index get_output_tx_and_index(const uint64_t &amount, const uint64_t &index) const
gets an output&#39;s tx hash and index
Definition: db_lmdb.cpp:3916
boost::optional< bool > is_hdd(const char *file_path)
Definition: util.cpp:813
virtual crypto::hash top_block_hash(uint64_t *block_height=NULL) const
fetch the top block&#39;s hash
Definition: db_lmdb.cpp:3502