Electroneum
blockchain_prune.cpp
Go to the documentation of this file.
1 // Copyright (c) 2018, The Monero Project
2 //
3 // All rights reserved.
4 //
5 // Redistribution and use in source and binary forms, with or without modification, are
6 // permitted provided that the following conditions are met:
7 //
8 // 1. Redistributions of source code must retain the above copyright notice, this list of
9 // conditions and the following disclaimer.
10 //
11 // 2. Redistributions in binary form must reproduce the above copyright notice, this list
12 // of conditions and the following disclaimer in the documentation and/or other
13 // materials provided with the distribution.
14 //
15 // 3. Neither the name of the copyright holder nor the names of its contributors may be
16 // used to endorse or promote products derived from this software without specific
17 // prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
20 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
21 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
22 // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
27 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 
29 #include <array>
30 #include <lmdb.h>
31 #include <boost/algorithm/string.hpp>
32 #include "common/command_line.h"
33 #include "common/pruning.h"
38 #include "blockchain_db/db_types.h"
39 #include "version.h"
40 
41 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY
42 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "bcutil"
43 
44 #define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val}
45 
46 namespace po = boost::program_options;
47 using namespace epee;
48 using namespace cryptonote;
49 
50 static std::string db_path;
51 
52 // default to fast:1
53 static uint64_t records_per_sync = 128;
54 static const size_t slack = 512 * 1024 * 1024;
55 
56 static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name)
57 {
58  std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string());
59  if (ec)
60  MERROR("Error renaming " << replacement_name << " to " << replaced_name << ": " << ec.message());
61  return ec;
62 }
63 
64 static void open(MDB_env *&env, const boost::filesystem::path &path, uint64_t db_flags, bool readonly)
65 {
66  int dbr;
67  int flags = 0;
68 
69  if (db_flags & DBF_FAST)
70  flags |= MDB_NOSYNC;
71  if (db_flags & DBF_FASTEST)
73  if (readonly)
74  flags |= MDB_RDONLY;
75 
76  dbr = mdb_env_create(&env);
77  if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
78  dbr = mdb_env_set_maxdbs(env, 32);
79  if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
80  dbr = mdb_env_open(env, path.string().c_str(), flags, 0664);
81  if (dbr) throw std::runtime_error("Failed to open database file '"
82  + path.string() + "': " + std::string(mdb_strerror(dbr)));
83 }
84 
85 static void close(MDB_env *env)
86 {
87  mdb_env_close(env);
88 }
89 
90 static void add_size(MDB_env *env, uint64_t bytes)
91 {
92  try
93  {
94  boost::filesystem::path path(db_path);
95  boost::filesystem::space_info si = boost::filesystem::space(path);
96  if(si.available < bytes)
97  {
98  MERROR("!! WARNING: Insufficient free space to extend database !!: " <<
99  (si.available >> 20L) << " MB available, " << (bytes >> 20L) << " MB needed");
100  return;
101  }
102  }
103  catch(...)
104  {
105  // print something but proceed.
106  MWARNING("Unable to query free disk space.");
107  }
108 
109  MDB_envinfo mei;
110  mdb_env_info(env, &mei);
111  MDB_stat mst;
112  mdb_env_stat(env, &mst);
113 
114  uint64_t new_mapsize = (uint64_t)mei.me_mapsize + bytes;
115  new_mapsize += (new_mapsize % mst.ms_psize);
116 
117  int result = mdb_env_set_mapsize(env, new_mapsize);
118  if (result)
119  throw std::runtime_error("Failed to set new mapsize to " + std::to_string(new_mapsize) + ": " + std::string(mdb_strerror(result)));
120 
121  MGINFO("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
122 }
123 
124 static void check_resize(MDB_env *env, size_t bytes)
125 {
126  MDB_envinfo mei;
127  MDB_stat mst;
128 
129  mdb_env_info(env, &mei);
130  mdb_env_stat(env, &mst);
131 
132  uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
133  if (size_used + bytes + slack >= mei.me_mapsize)
134  add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize);
135 }
136 
137 static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes)
138 {
139  if (nrecords % records_per_sync && bytes <= slack / 2)
140  return false;
141  int dbr = mdb_txn_commit(*txn);
142  if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr)));
143  check_resize(env, bytes);
144  dbr = mdb_txn_begin(env, NULL, 0, txn);
145  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
146  bytes = 0;
147  return true;
148 }
149 
150 static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0)
151 {
152  MDB_dbi dbi0, dbi1;
153  MDB_txn *txn0, *txn1;
154  MDB_cursor *cur0, *cur1;
155  bool tx_active0 = false, tx_active1 = false;
156  int dbr;
157 
158  MINFO("Copying " << table);
159 
161  if (tx_active1) mdb_txn_abort(txn1);
162  if (tx_active0) mdb_txn_abort(txn0);
163  });
164 
165  dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0);
166  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
167  tx_active0 = true;
168  dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
169  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
170  tx_active1 = true;
171 
172  dbr = mdb_dbi_open(txn0, table, flags, &dbi0);
173  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
174  if (cmp)
175  ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn0, dbi0, cmp);
176 
177  dbr = mdb_dbi_open(txn1, table, flags, &dbi1);
178  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
179  if (cmp)
180  ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn1, dbi1, cmp);
181 
182  dbr = mdb_txn_commit(txn1);
183  if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr)));
184  tx_active1 = false;
185  MDB_stat stats;
186  dbr = mdb_env_stat(env0, &stats);
187  if (dbr) throw std::runtime_error("Failed to stat " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr)));
188  check_resize(env1, (stats.ms_branch_pages + stats.ms_overflow_pages + stats.ms_leaf_pages) * stats.ms_psize);
189  dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
190  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
191  tx_active1 = true;
192 
193  dbr = mdb_drop(txn1, dbi1, 0);
194  if (dbr) throw std::runtime_error("Failed to empty " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr)));
195 
196  dbr = mdb_cursor_open(txn0, dbi0, &cur0);
197  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
198  dbr = mdb_cursor_open(txn1, dbi1, &cur1);
199  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
200 
201  MDB_val k;
202  MDB_val v;
204  size_t nrecords = 0, bytes = 0;
205  while (1)
206  {
207  int ret = mdb_cursor_get(cur0, &k, &v, op);
208  op = MDB_NEXT;
209  if (ret == MDB_NOTFOUND)
210  break;
211  if (ret)
212  throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret)));
213 
214  bytes += k.mv_size + v.mv_size;
215  if (resize_point(++nrecords, env1, &txn1, bytes))
216  {
217  dbr = mdb_cursor_open(txn1, dbi1, &cur1);
218  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
219  }
220 
221  ret = mdb_cursor_put(cur1, &k, &v, putflags);
222  if (ret)
223  throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret)));
224  }
225 
226  mdb_cursor_close(cur1);
227  mdb_cursor_close(cur0);
228  mdb_txn_commit(txn1);
229  tx_active1 = false;
230  mdb_txn_commit(txn0);
231  tx_active0 = false;
232  mdb_dbi_close(env1, dbi1);
233  mdb_dbi_close(env0, dbi0);
234 }
235 
236 static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id)
237 {
238  MDB_val v;
239  int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET);
240  if (ret)
241  throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret)));
242  if (v.mv_size == 0)
243  throw std::runtime_error("Invalid transaction pruned data");
245 }
246 
247 static void prune(MDB_env *env0, MDB_env *env1)
248 {
249  MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties;
250  MDB_txn *txn0, *txn1;
251  MDB_cursor *cur0_txs_pruned, *cur0_txs_prunable, *cur0_tx_indices, *cur1_txs_prunable, *cur1_txs_prunable_tip;
252  bool tx_active0 = false, tx_active1 = false;
253  int dbr;
254 
255  MGINFO("Creating pruned txs_prunable");
256 
258  if (tx_active1) mdb_txn_abort(txn1);
259  if (tx_active0) mdb_txn_abort(txn0);
260  });
261 
262  dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0);
263  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
264  tx_active0 = true;
265  dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
266  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
267  tx_active1 = true;
268 
269  dbr = mdb_dbi_open(txn0, "txs_pruned", MDB_INTEGERKEY, &dbi0_txs_pruned);
270  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
271  mdb_set_compare(txn0, dbi0_txs_pruned, BlockchainLMDB::compare_uint64);
272  dbr = mdb_cursor_open(txn0, dbi0_txs_pruned, &cur0_txs_pruned);
273  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
274 
275  dbr = mdb_dbi_open(txn0, "txs_prunable", MDB_INTEGERKEY, &dbi0_txs_prunable);
276  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
277  mdb_set_compare(txn0, dbi0_txs_prunable, BlockchainLMDB::compare_uint64);
278  dbr = mdb_cursor_open(txn0, dbi0_txs_prunable, &cur0_txs_prunable);
279  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
280 
281  dbr = mdb_dbi_open(txn0, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi0_tx_indices);
282  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
283  mdb_set_dupsort(txn0, dbi0_tx_indices, BlockchainLMDB::compare_hash32);
284  dbr = mdb_cursor_open(txn0, dbi0_tx_indices, &cur0_tx_indices);
285  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
286 
287  dbr = mdb_dbi_open(txn1, "txs_prunable", MDB_INTEGERKEY, &dbi1_txs_prunable);
288  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
289  mdb_set_compare(txn1, dbi1_txs_prunable, BlockchainLMDB::compare_uint64);
290  dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
291  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
292 
293  dbr = mdb_dbi_open(txn1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi1_txs_prunable_tip);
294  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
295  mdb_set_dupsort(txn1, dbi1_txs_prunable_tip, BlockchainLMDB::compare_uint64);
296  dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
297  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
298 
299  dbr = mdb_drop(txn1, dbi1_txs_prunable, 0);
300  if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr)));
301  dbr = mdb_drop(txn1, dbi1_txs_prunable_tip, 0);
302  if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr)));
303 
304  dbr = mdb_dbi_open(txn1, "properties", 0, &dbi1_properties);
305  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
306 
307  MDB_val k, v;
309  static char pruning_seed_key[] = "pruning_seed";
310  k.mv_data = pruning_seed_key;
311  k.mv_size = strlen("pruning_seed") + 1;
312  v.mv_data = (void*)&pruning_seed;
313  v.mv_size = sizeof(pruning_seed);
314  dbr = mdb_put(txn1, dbi1_properties, &k, &v, 0);
315  if (dbr) throw std::runtime_error("Failed to save pruning seed: " + std::string(mdb_strerror(dbr)));
316 
317  MDB_stat stats;
318  dbr = mdb_dbi_open(txn0, "blocks", 0, &dbi0_blocks);
319  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
320  dbr = mdb_stat(txn0, dbi0_blocks, &stats);
321  if (dbr) throw std::runtime_error("Failed to query size of blocks: " + std::string(mdb_strerror(dbr)));
322  mdb_dbi_close(env0, dbi0_blocks);
323  const uint64_t blockchain_height = stats.ms_entries;
324  size_t nrecords = 0, bytes = 0;
325 
327  while (1)
328  {
329  int ret = mdb_cursor_get(cur0_tx_indices, &k, &v, op);
330  op = MDB_NEXT;
331  if (ret == MDB_NOTFOUND)
332  break;
333  if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret)));
334 
335  const txindex *ti = (const txindex*)v.mv_data;
336  const uint64_t block_height = ti->data.block_id;
337  MDB_val_set(kk, ti->data.tx_id);
338  if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height)
339  {
340  MDEBUG(block_height << "/" << blockchain_height << " is in tip");
341  MDB_val_set(vv, block_height);
342  dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0);
343  if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr)));
344  bytes += kk.mv_size + vv.mv_size;
345  }
346  if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk))
347  {
348  MDB_val vv;
349  dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET);
350  if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr)));
351  bytes += kk.mv_size + vv.mv_size;
352  if (resize_point(++nrecords, env1, &txn1, bytes))
353  {
354  dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
355  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
356  dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
357  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
358  }
359  dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0);
360  if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr)));
361  }
362  else
363  {
364  MDEBUG("" << block_height << "/" << blockchain_height << " should be pruned, dropping");
365  }
366  }
367 
368  mdb_cursor_close(cur1_txs_prunable_tip);
369  mdb_cursor_close(cur1_txs_prunable);
370  mdb_cursor_close(cur0_txs_prunable);
371  mdb_cursor_close(cur0_txs_pruned);
372  mdb_cursor_close(cur0_tx_indices);
373  mdb_txn_commit(txn1);
374  tx_active1 = false;
375  mdb_txn_commit(txn0);
376  tx_active0 = false;
377  mdb_dbi_close(env1, dbi1_properties);
378  mdb_dbi_close(env1, dbi1_txs_prunable_tip);
379  mdb_dbi_close(env1, dbi1_txs_prunable);
380  mdb_dbi_close(env0, dbi0_txs_prunable);
381  mdb_dbi_close(env0, dbi0_txs_pruned);
382  mdb_dbi_close(env0, dbi0_tx_indices);
383 }
384 
385 static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags)
386 {
387  std::vector<std::string> options;
388  boost::trim(db_sync_mode);
389  boost::split(options, db_sync_mode, boost::is_any_of(" :"));
390 
391  for(const auto &option : options)
392  MDEBUG("option: " << option);
393 
394  // default to fast:async:1
395  uint64_t DEFAULT_FLAGS = DBF_FAST;
396 
397  db_flags = 0;
398 
399  if(options.size() == 0)
400  {
401  // default to fast:async:1
402  db_flags = DEFAULT_FLAGS;
403  }
404 
405  bool safemode = false;
406  if(options.size() >= 1)
407  {
408  if(options[0] == "safe")
409  {
410  safemode = true;
411  db_flags = DBF_SAFE;
412  }
413  else if(options[0] == "fast")
414  {
415  db_flags = DBF_FAST;
416  }
417  else if(options[0] == "fastest")
418  {
419  db_flags = DBF_FASTEST;
420  records_per_sync = 1000; // default to fastest:async:1000
421  }
422  else
423  return false;
424  }
425 
426  if(options.size() >= 2 && !safemode)
427  {
428  char *endptr;
429  uint64_t bps = strtoull(options[1].c_str(), &endptr, 0);
430  if (*endptr != '\0')
431  return false;
432  records_per_sync = bps;
433  }
434 
435  return true;
436 }
437 
438 int main(int argc, char* argv[])
439 {
440  TRY_ENTRY();
441 
443 
444  std::string default_db_type = "lmdb";
445 
446  std::string available_dbs = cryptonote::blockchain_db_types(", ");
447  available_dbs = "available: " + available_dbs;
448 
449  uint32_t log_level = 0;
450 
452 
453  boost::filesystem::path output_file_path;
454 
455  po::options_description desc_cmd_only("Command line options");
456  po::options_description desc_cmd_sett("Command line options and settings options");
457  const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
458  const command_line::arg_descriptor<std::string> arg_database = {
459  "database", available_dbs.c_str(), default_db_type
460  };
462  "db-sync-mode"
463  , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]."
464  , "fast:1000"
465  };
466  const command_line::arg_descriptor<bool> arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"};
467 
471  command_line::add_arg(desc_cmd_sett, arg_log_level);
472  command_line::add_arg(desc_cmd_sett, arg_database);
473  command_line::add_arg(desc_cmd_sett, arg_db_sync_mode);
474  command_line::add_arg(desc_cmd_sett, arg_copy_pruned_database);
476 
477  po::options_description desc_options("Allowed options");
478  desc_options.add(desc_cmd_only).add(desc_cmd_sett);
479 
480  po::variables_map vm;
481  bool r = command_line::handle_error_helper(desc_options, [&]()
482  {
483  auto parser = po::command_line_parser(argc, argv).options(desc_options);
484  po::store(parser.run(), vm);
485  po::notify(vm);
486  return true;
487  });
488  if (! r)
489  return 1;
490 
492  {
493  std::cout << "Electroneum '" << ELECTRONEUM_RELEASE_NAME << "' (v" << ELECTRONEUM_VERSION_FULL << ")" << ENDL << ENDL;
494  std::cout << desc_options << std::endl;
495  return 1;
496  }
497 
498  mlog_configure(mlog_get_default_log_path("electroneum-blockchain-prune.log"), true);
501  else
502  mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
503 
504  MINFO("Starting...");
505 
506  bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
507  bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on);
508  network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
509  bool opt_copy_pruned_database = command_line::get_arg(vm, arg_copy_pruned_database);
511  while (boost::ends_with(data_dir, "/") || boost::ends_with(data_dir, "\\"))
512  data_dir.pop_back();
513 
514  std::string db_type = command_line::get_arg(vm, arg_database);
516  {
517  MERROR("Invalid database type: " << db_type);
518  return 1;
519  }
520  if (db_type != "lmdb")
521  {
522  MERROR("Unsupported database type: " << db_type << ". Only lmdb is supported");
523  return 1;
524  }
525 
527  uint64_t db_flags = 0;
528  if (!parse_db_sync_mode(db_sync_mode, db_flags))
529  {
530  MERROR("Invalid db sync mode: " << db_sync_mode);
531  return 1;
532  }
533 
534  // If we wanted to use the memory pool, we would set up a fake_core.
535 
536  // Use Blockchain instead of lower-level BlockchainDB for two reasons:
537  // 1. Blockchain has the init() method for easy setup
538  // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash()
539  //
540  // cannot match blockchain_storage setup above with just one line,
541  // e.g.
542  // Blockchain* core_storage = new Blockchain(NULL);
543  // because unlike blockchain_storage constructor, which takes a pointer to
544  // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object.
545  MINFO("Initializing source blockchain (BlockchainDB)");
546  std::array<std::unique_ptr<Blockchain>, 2> core_storage;
547  Blockchain *blockchain = NULL;
548  tx_memory_pool m_mempool(*blockchain);
549  boost::filesystem::path paths[2];
550  bool already_pruned = false;
551  for (size_t n = 0; n < core_storage.size(); ++n)
552  {
553  core_storage[n].reset(new Blockchain(m_mempool));
554 
555  BlockchainDB* db = new_db(db_type);
556  if (db == NULL)
557  {
558  MERROR("Attempted to use non-existent database type: " << db_type);
559  throw std::runtime_error("Attempting to use non-existent database type");
560  }
561  MDEBUG("database: " << db_type);
562 
563  if (n == 1)
564  {
565  paths[1] = boost::filesystem::path(data_dir) / (db->get_db_name() + "-pruned");
566  if (boost::filesystem::exists(paths[1]))
567  {
568  if (!boost::filesystem::is_directory(paths[1]))
569  {
570  MERROR("LMDB needs a directory path, but a file was passed: " << paths[1].string());
571  return 1;
572  }
573  }
574  else
575  {
576  if (!boost::filesystem::create_directories(paths[1]))
577  {
578  MERROR("Failed to create directory: " << paths[1].string());
579  return 1;
580  }
581  }
582  db_path = paths[1].string();
583  }
584  else
585  {
586  paths[0] = boost::filesystem::path(data_dir) / db->get_db_name();
587  }
588 
589  MINFO("Loading blockchain from folder " << paths[n] << " ...");
590 
591  try
592  {
593  db->open(paths[n].string(), n == 0 ? DBF_RDONLY : 0);
594  }
595  catch (const std::exception& e)
596  {
597  MERROR("Error opening database: " << e.what());
598  return 1;
599  }
600  r = core_storage[n]->init(db, net_type);
601 
602  std::string source_dest = n == 0 ? "source" : "pruned";
603  CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize " << source_dest << " blockchain storage");
604  MINFO(source_dest << " blockchain storage initialized OK");
605  if (n == 0 && core_storage[0]->get_blockchain_pruning_seed())
606  {
607  if (!opt_copy_pruned_database)
608  {
609  MERROR("Blockchain is already pruned, use --" << arg_copy_pruned_database.name << " to copy it anyway");
610  return 1;
611  }
612  already_pruned = true;
613  }
614  }
615  core_storage[0]->deinit();
616  core_storage[0].reset(NULL);
617  core_storage[1]->deinit();
618  core_storage[1].reset(NULL);
619 
620  MINFO("Pruning...");
621  MDB_env *env0 = NULL, *env1 = NULL;
622  open(env0, paths[0], db_flags, true);
623  open(env1, paths[1], db_flags, false);
624  copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND);
625  copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
626  copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
627  //copy_table(env0, env1, "txs", MDB_INTEGERKEY);
628  copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND);
629  copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND);
630  // not copied: prunable, prunable_tip
631  copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
632  copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND);
633  copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
634  copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
635  copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
636  copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
637  copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
638  copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND);
639  copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string);
640  if (already_pruned)
641  {
642  copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64);
643  copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64);
644  }
645  else
646  {
647  prune(env0, env1);
648  }
649  close(env1);
650  close(env0);
651 
652  MINFO("Swapping databases, pre-pruning blockchain will be left in " << paths[0].string() + "-old and can be removed if desired");
653  if (replace_file(paths[0].string(), paths[0].string() + "-old") || replace_file(paths[1].string(), paths[0].string()))
654  {
655  MERROR("Blockchain pruned OK, but renaming failed");
656  return 1;
657  }
658 
659  MINFO("Blockchain pruned OK");
660  return 0;
661 
662  CATCH_ENTRY("Pruning error", 1);
663 }
const char *const ELECTRONEUM_RELEASE_NAME
#define MDB_NOTFOUND
Definition: lmdb.h:439
#define MERROR(x)
Definition: misc_log_ex.h:73
std::vector< std::vector< _variant_t > > table
int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
Set the size of the memory map to use for this environment.
void mdb_cursor_close(MDB_cursor *cursor)
Close a cursor handle.
virtual std::string get_db_name() const =0
gets the name of the folder the BlockchainDB&#39;s file(s) should be in
Definition: lmdb.h:411
#define MDB_RDONLY
Definition: lmdb.h:320
#define DBF_RDONLY
#define MINFO(x)
Definition: misc_log_ex.h:75
int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi)
Open a database in the environment.
#define MDB_NODUPDATA
Definition: lmdb.h:369
Lightning memory-mapped database library.
bool set_module_name_and_folder(const std::string &path_to_process_)
Definition: string_tools.h:249
tx_data_t data
Definition: db_lmdb.h:46
::std::string string
Definition: gtest-port.h:1097
uint32_t get_random_stripe()
Definition: pruning.cpp:110
int mdb_txn_commit(MDB_txn *txn)
Commit all the operations of a transaction into the database.
int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, unsigned int flags)
Store by cursor.
auto_scope_leave_caller create_scope_leave_handler(t_scope_leave_handler f)
int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat)
Retrieve statistics for a database.
#define DBF_FASTEST
#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message)
Definition: misc_log_ex.h:181
void mdb_dbi_close(MDB_env *env, MDB_dbi dbi)
Close a database handle. Normally unnecessary. Use with care:
void mlog_set_log(const char *log)
Definition: mlog.cpp:288
#define CATCH_ENTRY(location, return_val)
Definition: misc_log_ex.h:152
std::string mlog_get_default_log_path(const char *default_filename)
Definition: mlog.cpp:72
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size=MAX_LOG_FILE_SIZE, const std::size_t max_log_files=MAX_LOG_FILES)
Definition: mlog.cpp:148
#define CRYPTONOTE_PRUNING_LOG_STRIPES
bool is_v1_tx(const blobdata &tx_blob)
int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
Set a custom data comparison function for a MDB_DUPSORT database.
int mdb_env_info(MDB_env *env, MDB_envinfo *stat)
Return information about the LMDB environment.
const command_line::arg_descriptor< std::string > arg_db_sync_mode
const command_line::arg_descriptor< bool, false > arg_stagenet_on
int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs)
Set the maximum number of named databases for the environment.
const arg_descriptor< bool > arg_help
Non-owning sequence of data. Does not deep copy.
Definition: span.h:56
#define DBF_SAFE
std::string blockchain_db_types(const std::string &sep)
struct MDB_env MDB_env
Opaque structure for a database environment.
Definition: lmdb.h:260
mdb_size_t ms_branch_pages
Definition: lmdb.h:494
boost::filesystem::path data_dir
Definition: main.cpp:50
std::string & trim(std::string &str)
Definition: string_tools.h:288
#define DBF_FAST
#define MGINFO(x)
Definition: misc_log_ex.h:80
virtual void open(const std::string &filename, const int db_flags=0)=0
open a db, or create it if necessary.
#define MDB_MAPASYNC
Definition: lmdb.h:326
#define CRYPTONOTE_PRUNING_TIP_BLOCKS
#define MDEBUG(x)
Definition: misc_log_ex.h:76
struct MDB_txn MDB_txn
Opaque structure for a transaction handle.
Definition: lmdb.h:267
bool on_startup()
Definition: util.cpp:778
Holds cryptonote related classes and helpers.
Definition: ban.cpp:40
int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
Empty or delete+close a database.
Definition: options.h:86
const char *const ELECTRONEUM_VERSION_FULL
bool blockchain_valid_db_type(const std::string &db_type)
int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode)
Open an environment handle.
Statistics for a database in the environment.
Definition: lmdb.h:490
Information about the environment.
Definition: lmdb.h:501
const command_line::arg_descriptor< bool, false > arg_testnet_on
void * mv_data
Definition: lmdb.h:288
BlockchainDB * new_db(const std::string &db_type)
bool handle_error_helper(const boost::program_options::options_description &desc, F parser)
Definition: command_line.h:237
#define TRY_ENTRY()
Definition: misc_log_ex.h:151
unsigned int uint32_t
Definition: stdint.h:126
#define MDB_WRITEMAP
Definition: lmdb.h:324
const command_line::arg_descriptor< std::string > arg_log_level
size_t mv_size
Definition: lmdb.h:287
Definition: lmdb.h:422
boost::shared_ptr< call_befor_die_base > auto_scope_leave_caller
#define MDB_val_set(var, val)
mdb_size_t me_last_pgno
Definition: lmdb.h:504
unsigned __int64 uint64_t
Definition: stdint.h:136
const command_line::arg_descriptor< std::string, false, true, 2 > arg_data_dir
#define MDB_INTEGERKEY
Definition: lmdb.h:349
#define MDB_DUPFIXED
Definition: lmdb.h:351
int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
Set a custom key comparison function for a database.
mdb_size_t ms_overflow_pages
Definition: lmdb.h:496
std::error_code replace_file(const std::string &old_name, const std::string &new_name)
std::rename wrapper for nix and something strange for windows.
Definition: util.cpp:648
unsigned int MDB_dbi
A handle for an individual database in the DB environment.
Definition: lmdb.h:270
int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
Create a transaction for use with the environment.
MDB_cursor_op
Cursor Get operations.
Definition: lmdb.h:398
#define MWARNING(x)
Definition: misc_log_ex.h:74
void add_arg(boost::program_options::options_description &description, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg, bool unique=true)
Definition: command_line.h:188
int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, MDB_cursor_op op)
Retrieve by cursor.
int mdb_env_create(MDB_env **env)
Create an LMDB environment handle.
void mdb_txn_abort(MDB_txn *txn)
Abandon all the operations of the transaction instead of saving them.
Transaction pool, handles transactions which are not part of a block.
Definition: tx_pool.h:94
mdb_size_t ms_leaf_pages
Definition: lmdb.h:495
#define MDB_APPEND
Definition: lmdb.h:377
struct MDB_cursor MDB_cursor
Opaque structure for navigating through a database.
Definition: lmdb.h:273
The BlockchainDB backing store interface declaration/contract.
Generic structure used for passing keys and data in and out of the database.
Definition: lmdb.h:286
unsigned int ms_psize
Definition: lmdb.h:491
#define ENDL
Definition: misc_log_ex.h:149
bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed)
Definition: pruning.cpp:44
T get_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, false, true > &arg)
Definition: command_line.h:271
int mdb_env_stat(MDB_env *env, MDB_stat *stat)
Return statistics about the LMDB environment.
char * mdb_strerror(int err)
Return a string describing a given error code.
mdb_size_t me_mapsize
Definition: lmdb.h:503
uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes)
Definition: pruning.cpp:37
std::string to_string(t_connection_type type)
int main(int argc, char *argv[])
#define MDB_DUPSORT
Definition: lmdb.h:345
int compare_uint64(const MDB_val *a, const MDB_val *b)
mdb_size_t ms_entries
Definition: lmdb.h:497
#define MDB_NOSYNC
Definition: lmdb.h:318
int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned int flags)
Store items into a database.
void mdb_env_close(MDB_env *env)
Close the environment and release the memory map.
bool is_arg_defaulted(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
Definition: command_line.h:265
int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor)
Create a cursor handle.
#define MDB_APPENDDUP
Definition: lmdb.h:379