Electroneum
blockchain_blackball.cpp
Go to the documentation of this file.
1 // Copyright (c) 2014-2019, The Monero Project
2 //
3 // All rights reserved.
4 //
5 // Redistribution and use in source and binary forms, with or without modification, are
6 // permitted provided that the following conditions are met:
7 //
8 // 1. Redistributions of source code must retain the above copyright notice, this list of
9 // conditions and the following disclaimer.
10 //
11 // 2. Redistributions in binary form must reproduce the above copyright notice, this list
12 // of conditions and the following disclaimer in the documentation and/or other
13 // materials provided with the distribution.
14 //
15 // 3. Neither the name of the copyright holder nor the names of its contributors may be
16 // used to endorse or promote products derived from this software without specific
17 // prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
20 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
21 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
22 // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
27 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 
29 #include <boost/range/adaptor/transformed.hpp>
30 #include <boost/algorithm/string.hpp>
34 #include "common/command_line.h"
35 #include "common/varint.h"
36 #include "serialization/crypto.h"
42 #include "blockchain_db/db_types.h"
43 #include "wallet/ringdb.h"
44 #include "version.h"
45 
46 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY
47 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "bcutil"
48 
49 namespace po = boost::program_options;
50 using namespace epee;
51 using namespace cryptonote;
52 
53 static const char zerokey[8] = {0};
54 static const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey };
55 
56 static uint64_t records_per_sync = 200;
57 static uint64_t db_flags = 0;
58 static MDB_dbi dbi_relative_rings;
59 static MDB_dbi dbi_outputs;
60 static MDB_dbi dbi_processed_txidx;
61 static MDB_dbi dbi_spent;
62 static MDB_dbi dbi_per_amount;
63 static MDB_dbi dbi_ring_instances;
64 static MDB_dbi dbi_stats;
65 static MDB_env *env = NULL;
66 
68 {
71  output_data(): amount(0), offset(0) {}
72  output_data(uint64_t a, uint64_t i): amount(a), offset(i) {}
73  bool operator==(const output_data &other) const { return other.amount == amount && other.offset == offset; }
74 };
75 
76 //
77 // relative_rings: key_image -> vector<uint64_t>
78 // outputs: 128 bits -> set of key images
79 // processed_txidx: string -> uint64_t
80 // spent: amount -> offset
81 // ring_instances: vector<uint64_t> -> uint64_t
82 // stats: string -> arbitrary
83 //
84 
85 static bool parse_db_sync_mode(std::string db_sync_mode)
86 {
87  std::vector<std::string> options;
88  boost::trim(db_sync_mode);
89  boost::split(options, db_sync_mode, boost::is_any_of(" :"));
90 
91  for(const auto &option : options)
92  MDEBUG("option: " << option);
93 
94  // default to fast:async:1
95  uint64_t DEFAULT_FLAGS = DBF_FAST;
96 
97  if(options.size() == 0)
98  {
99  // default to fast:async:1
100  db_flags = DEFAULT_FLAGS;
101  }
102 
103  bool safemode = false;
104  if(options.size() >= 1)
105  {
106  if(options[0] == "safe")
107  {
108  safemode = true;
109  db_flags = DBF_SAFE;
110  }
111  else if(options[0] == "fast")
112  {
113  db_flags = DBF_FAST;
114  }
115  else if(options[0] == "fastest")
116  {
117  db_flags = DBF_FASTEST;
118  records_per_sync = 1000; // default to fastest:async:1000
119  }
120  else
121  db_flags = DEFAULT_FLAGS;
122  }
123 
124  if(options.size() >= 2 && !safemode)
125  {
126  char *endptr;
127  uint64_t bps = strtoull(options[1].c_str(), &endptr, 0);
128  if (*endptr == '\0')
129  records_per_sync = bps;
130  }
131 
132  return true;
133 }
134 
135 static std::string get_default_db_path()
136 {
137  boost::filesystem::path dir = tools::get_default_data_dir();
138  // remove .electroneum, replace with .shared-ringdb
139  dir = dir.remove_filename();
140  dir /= ".shared-ringdb";
141  return dir.string();
142 }
143 
144 static std::string get_cache_filename(boost::filesystem::path filename)
145 {
146  if (!boost::filesystem::is_directory(filename))
147  filename.remove_filename();
148  return filename.string();
149 }
150 
151 static int compare_hash32(const MDB_val *a, const MDB_val *b)
152 {
153  const uint32_t *va = (const uint32_t*) a->mv_data;
154  const uint32_t *vb = (const uint32_t*) b->mv_data;
155  for (int n = 7; n >= 0; n--)
156  {
157  if (va[n] == vb[n])
158  continue;
159  return va[n] < vb[n] ? -1 : 1;
160  }
161 
162  return 0;
163 }
164 
165 int compare_uint64(const MDB_val *a, const MDB_val *b)
166 {
167  const uint64_t va = *(const uint64_t *)a->mv_data;
168  const uint64_t vb = *(const uint64_t *)b->mv_data;
169  return (va < vb) ? -1 : va > vb;
170 }
171 
172 static int compare_double64(const MDB_val *a, const MDB_val *b)
173 {
174  const uint64_t va = *(const uint64_t*) a->mv_data;
175  const uint64_t vb = *(const uint64_t*) b->mv_data;
176  if (va == vb)
177  {
178  const uint64_t va = ((const uint64_t*) a->mv_data)[1];
179  const uint64_t vb = ((const uint64_t*) b->mv_data)[1];
180  return va < vb ? -1 : va > vb;
181  }
182  return va < vb ? -1 : va > vb;
183 }
184 
185 static int resize_env(const char *db_path)
186 {
187  MDB_envinfo mei;
188  MDB_stat mst;
189  int ret;
190 
191  size_t needed = 1000ul * 1024 * 1024; // at least 1000 MB
192 
193  ret = mdb_env_info(env, &mei);
194  if (ret)
195  return ret;
196  ret = mdb_env_stat(env, &mst);
197  if (ret)
198  return ret;
199  uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
200  uint64_t mapsize = mei.me_mapsize;
201  if (size_used + needed > mei.me_mapsize)
202  {
203  try
204  {
205  boost::filesystem::path path(db_path);
206  boost::filesystem::space_info si = boost::filesystem::space(path);
207  if(si.available < needed)
208  {
209  MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available");
210  return ENOSPC;
211  }
212  }
213  catch(...)
214  {
215  // print something but proceed.
216  MWARNING("Unable to query free disk space.");
217  }
218 
219  mapsize += needed;
220  }
221  return mdb_env_set_mapsize(env, mapsize);
222 }
223 
224 static void init(std::string cache_filename)
225 {
226  MDB_txn *txn;
227  bool tx_active = false;
228  int dbr;
229 
230  MINFO("Creating spent output cache in " << cache_filename);
231 
233 
234  int flags = 0;
235  if (db_flags & DBF_FAST)
236  flags |= MDB_NOSYNC;
237  if (db_flags & DBF_FASTEST)
238  flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC;
239 
240  dbr = mdb_env_create(&env);
241  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
242  dbr = mdb_env_set_maxdbs(env, 7);
243  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
244  const std::string actual_filename = get_cache_filename(cache_filename);
245  dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664);
246  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '"
247  + actual_filename + "': " + std::string(mdb_strerror(dbr)));
248 
249  dbr = mdb_txn_begin(env, NULL, 0, &txn);
250  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
252  tx_active = true;
253 
254  dbr = mdb_dbi_open(txn, "relative_rings", MDB_CREATE | MDB_INTEGERKEY, &dbi_relative_rings);
255  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
256  mdb_set_compare(txn, dbi_relative_rings, compare_hash32);
257 
258  dbr = mdb_dbi_open(txn, "outputs", MDB_CREATE | MDB_INTEGERKEY, &dbi_outputs);
259  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
260  mdb_set_compare(txn, dbi_outputs, compare_double64);
261 
262  dbr = mdb_dbi_open(txn, "processed_txidx", MDB_CREATE, &dbi_processed_txidx);
263  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
264 
265  dbr = mdb_dbi_open(txn, "spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_spent);
266  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
267  mdb_set_dupsort(txn, dbi_spent, compare_uint64);
268 
269  dbr = mdb_dbi_open(txn, "per_amount", MDB_CREATE | MDB_INTEGERKEY, &dbi_per_amount);
270  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
271  mdb_set_compare(txn, dbi_per_amount, compare_uint64);
272 
273  dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances);
274  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
275 
276  dbr = mdb_dbi_open(txn, "stats", MDB_CREATE, &dbi_stats);
277  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
278 
279  dbr = mdb_txn_commit(txn);
280  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr)));
281  tx_active = false;
282 }
283 
284 static void close()
285 {
286  if (env)
287  {
288  mdb_dbi_close(env, dbi_relative_rings);
289  mdb_dbi_close(env, dbi_outputs);
290  mdb_dbi_close(env, dbi_processed_txidx);
291  mdb_dbi_close(env, dbi_per_amount);
292  mdb_dbi_close(env, dbi_spent);
293  mdb_dbi_close(env, dbi_ring_instances);
294  mdb_dbi_close(env, dbi_stats);
295  mdb_env_close(env);
296  env = NULL;
297  }
298 }
299 
300 static std::string compress_ring(const std::vector<uint64_t> &ring, std::string s = "")
301 {
302  const size_t sz = s.size();
303  s.resize(s.size() + 12 * ring.size());
304  char *ptr = (char*)s.data() + sz;
305  for (uint64_t out: ring)
306  tools::write_varint(ptr, out);
307  if (ptr > s.data() + sz + 12 * ring.size())
308  throw std::runtime_error("varint output overflow");
309  s.resize(ptr - s.data());
310  return s;
311 }
312 
313 static std::string compress_ring(uint64_t amount, const std::vector<uint64_t> &ring)
314 {
315  char s[12], *ptr = s;
316  tools::write_varint(ptr, amount);
317  if (ptr > s + sizeof(s))
318  throw std::runtime_error("varint output overflow");
319  return compress_ring(ring, std::string(s, ptr-s));
320 }
321 
322 static std::vector<uint64_t> decompress_ring(const std::string &s)
323 {
324  std::vector<uint64_t> ring;
325  int read = 0;
326  for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read))
327  {
328  uint64_t out;
329  std::string tmp(i, s.cend());
330  read = tools::read_varint(tmp.begin(), tmp.end(), out);
331  CHECK_AND_ASSERT_THROW_MES(read > 0 && read <= 256, "Internal error decompressing ring");
332  ring.push_back(out);
333  }
334  return ring;
335 }
336 
337 static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, uint64_t &n_txes, const std::function<bool(const cryptonote::transaction_prefix&)> &f)
338 {
339  MDB_env *env;
340  MDB_dbi dbi;
341  MDB_txn *txn;
342  MDB_cursor *cur;
343  int dbr;
344  bool tx_active = false;
345  MDB_val k;
346  MDB_val v;
347 
348  dbr = mdb_env_create(&env);
349  if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
350  dbr = mdb_env_set_maxdbs(env, 2);
351  if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
352  const std::string actual_filename = filename;
353  dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664);
354  if (dbr) throw std::runtime_error("Failed to open rings database file '"
355  + actual_filename + "': " + std::string(mdb_strerror(dbr)));
356 
357  dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
358  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
360  tx_active = true;
361 
362  dbr = mdb_dbi_open(txn, "txs_pruned", MDB_INTEGERKEY, &dbi);
363  if (dbr)
364  dbr = mdb_dbi_open(txn, "txs", MDB_INTEGERKEY, &dbi);
365  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
366  dbr = mdb_cursor_open(txn, dbi, &cur);
367  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
368  MDB_stat stat;
369  dbr = mdb_stat(txn, dbi, &stat);
370  if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr)));
371  n_txes = stat.ms_entries;
372 
373  bool fret = true;
374 
375  k.mv_size = sizeof(uint64_t);
376  k.mv_data = &start_idx;
377  MDB_cursor_op op = MDB_SET;
378  while (1)
379  {
380  int ret = mdb_cursor_get(cur, &k, &v, op);
381  op = MDB_NEXT;
382  if (ret == MDB_NOTFOUND)
383  break;
384  if (ret)
385  throw std::runtime_error("Failed to enumerate transactions: " + std::string(mdb_strerror(ret)));
386 
387  if (k.mv_size != sizeof(uint64_t))
388  throw std::runtime_error("Bad key size");
389  const uint64_t idx = *(uint64_t*)k.mv_data;
390  if (idx < start_idx)
391  continue;
392 
394  blobdata bd;
395  bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
396  std::stringstream ss;
397  ss << bd;
398  binary_archive<false> ba(ss);
399  bool r = do_serialize(ba, tx);
400  CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob");
401 
402  start_idx = *(uint64_t*)k.mv_data;
403  if (!f(tx)) {
404  fret = false;
405  break;
406  }
407  }
408 
409  mdb_cursor_close(cur);
410  dbr = mdb_txn_commit(txn);
411  if (dbr) throw std::runtime_error("Failed to commit db transaction: " + std::string(mdb_strerror(dbr)));
412  tx_active = false;
413  mdb_dbi_close(env, dbi);
414  mdb_env_close(env);
415  return fret;
416 }
417 
418 static uint64_t find_first_diverging_transaction(const std::string &first_filename, const std::string &second_filename)
419 {
420  MDB_env *env[2];
421  MDB_dbi dbi[2];
422  MDB_txn *txn[2];
423  MDB_cursor *cur[2];
424  int dbr;
425  bool tx_active[2] = { false, false };
426  uint64_t n_txes[2];
427  MDB_val k;
428  MDB_val v[2];
429 
431  for (int i = 0; i < 2; ++i)
432  {
433  dbr = mdb_env_create(&env[i]);
434  if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
435  dbr = mdb_env_set_maxdbs(env[i], 2);
436  if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
437  const std::string actual_filename = i ? second_filename : first_filename;
438  dbr = mdb_env_open(env[i], actual_filename.c_str(), 0, 0664);
439  if (dbr) throw std::runtime_error("Failed to open rings database file '"
440  + actual_filename + "': " + std::string(mdb_strerror(dbr)));
441 
442  dbr = mdb_txn_begin(env[i], NULL, MDB_RDONLY, &txn[i]);
443  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
444  txn_dtor[i] = epee::misc_utils::create_scope_leave_handler([&, i](){if (tx_active[i]) mdb_txn_abort(txn[i]);});
445  tx_active[i] = true;
446 
447  dbr = mdb_dbi_open(txn[i], "txs_pruned", MDB_INTEGERKEY, &dbi[i]);
448  if (dbr)
449  dbr = mdb_dbi_open(txn[i], "txs", MDB_INTEGERKEY, &dbi[i]);
450  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
451  dbr = mdb_cursor_open(txn[i], dbi[i], &cur[i]);
452  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
453  MDB_stat stat;
454  dbr = mdb_stat(txn[i], dbi[i], &stat);
455  if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr)));
456  n_txes[i] = stat.ms_entries;
457  }
458 
459  if (n_txes[0] == 0 || n_txes[1] == 0)
460  throw std::runtime_error("No transaction in the database");
461  uint64_t lo = 0, hi = std::min(n_txes[0], n_txes[1]) - 1;
462  while (lo <= hi)
463  {
464  uint64_t mid = (lo + hi) / 2;
465 
466  k.mv_size = sizeof(uint64_t);
467  k.mv_data = (void*)&mid;
468  dbr = mdb_cursor_get(cur[0], &k, &v[0], MDB_SET);
469  if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr)));
470  dbr = mdb_cursor_get(cur[1], &k, &v[1], MDB_SET);
471  if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr)));
472  if (v[0].mv_size == v[1].mv_size && !memcmp(v[0].mv_data, v[1].mv_data, v[0].mv_size))
473  lo = mid + 1;
474  else
475  hi = mid - 1;
476  }
477 
478  for (int i = 0; i < 2; ++i)
479  {
480  mdb_cursor_close(cur[i]);
481  dbr = mdb_txn_commit(txn[i]);
482  if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr)));
483  tx_active[i] = false;
484  mdb_dbi_close(env[i], dbi[i]);
485  mdb_env_close(env[i]);
486  }
487  return hi;
488 }
489 
490 static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v)
491 {
492  std::vector<uint64_t> c;
493  c.reserve(v.size());
494  c.push_back(v[0]);
495  for (size_t n = 1; n < v.size(); ++n)
496  {
497  if (v[n] != 0)
498  c.push_back(v[n]);
499  }
500  if (c.size() < v.size())
501  {
502  MINFO("Ring has duplicate member(s): " <<
503  boost::join(v | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " "));
504  }
505  return c;
506 }
507 
508 static uint64_t get_num_spent_outputs()
509 {
510  MDB_txn *txn;
511  bool tx_active = false;
512 
513  int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
514  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
516  tx_active = true;
517 
518  MDB_cursor *cur;
519  dbr = mdb_cursor_open(txn, dbi_spent, &cur);
520  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr)));
521  MDB_val k, v;
522  mdb_size_t count = 0, tmp;
523 
525  while (1)
526  {
527  dbr = mdb_cursor_get(cur, &k, &v, op);
528  op = MDB_NEXT_NODUP;
529  if (dbr == MDB_NOTFOUND)
530  break;
531  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first/next spent output: " + std::string(mdb_strerror(dbr)));
532  dbr = mdb_cursor_count(cur, &tmp);
533  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr)));
534  count += tmp;
535  }
536 
537  mdb_cursor_close(cur);
538  dbr = mdb_txn_commit(txn);
539  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr)));
540  tx_active = false;
541 
542  return count;
543 }
544 
545 static bool add_spent_output(MDB_cursor *cur, const output_data &od)
546 {
547  MDB_val k = {sizeof(od.amount), (void*)&od.amount};
548  MDB_val v = {sizeof(od.offset), (void*)&od.offset};
549  int dbr = mdb_cursor_put(cur, &k, &v, MDB_NODUPDATA);
550  if (dbr == MDB_KEYEXIST)
551  return false;
552  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to add spent output: " + std::string(mdb_strerror(dbr)));
553  return true;
554 }
555 
556 static bool is_output_spent(MDB_cursor *cur, const output_data &od)
557 {
558  MDB_val k = {sizeof(od.amount), (void*)&od.amount};
559  MDB_val v = {sizeof(od.offset), (void*)&od.offset};
560  int dbr = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH);
561  CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get spent output: " + std::string(mdb_strerror(dbr)));
562  bool spent = dbr == 0;
563  return spent;
564 }
565 
566 static std::vector<output_data> get_spent_outputs(MDB_txn *txn)
567 {
568  MDB_cursor *cur;
569  int dbr = mdb_cursor_open(txn, dbi_spent, &cur);
570  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr)));
571  MDB_val k, v;
572  mdb_size_t count = 0;
573  dbr = mdb_cursor_get(cur, &k, &v, MDB_FIRST);
574  if (dbr != MDB_NOTFOUND)
575  {
576  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first spent output: " + std::string(mdb_strerror(dbr)));
577  dbr = mdb_cursor_count(cur, &count);
578  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr)));
579  }
580  std::vector<output_data> outs;
581  outs.reserve(count);
582  while (1)
583  {
584  outs.push_back({*(const uint64_t*)k.mv_data, *(const uint64_t*)v.mv_data});
585  dbr = mdb_cursor_get(cur, &k, &v, MDB_NEXT);
586  if (dbr == MDB_NOTFOUND)
587  break;
588  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get next spent output: " + std::string(mdb_strerror(dbr)));
589  }
590  mdb_cursor_close(cur);
591  return outs;
592 }
593 
594 static void get_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t &total, uint64_t &spent)
595 {
596  MDB_cursor *cur;
597  int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur);
598  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr)));
599  MDB_val k, v;
600  mdb_size_t count = 0;
601  k.mv_size = sizeof(uint64_t);
602  k.mv_data = (void*)&amount;
603  dbr = mdb_cursor_get(cur, &k, &v, MDB_SET);
604  if (dbr == MDB_NOTFOUND)
605  {
606  total = spent = 0;
607  }
608  else
609  {
610  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get per amount outputs: " + std::string(mdb_strerror(dbr)));
611  total = ((const uint64_t*)v.mv_data)[0];
612  spent = ((const uint64_t*)v.mv_data)[1];
613  }
614  mdb_cursor_close(cur);
615 }
616 
617 static void inc_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t total, uint64_t spent)
618 {
619  MDB_cursor *cur;
620  int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur);
621  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr)));
622  MDB_val k, v;
623  mdb_size_t count = 0;
624  k.mv_size = sizeof(uint64_t);
625  k.mv_data = (void*)&amount;
626  dbr = mdb_cursor_get(cur, &k, &v, MDB_SET);
627  if (dbr == 0)
628  {
629  total += ((const uint64_t*)v.mv_data)[0];
630  spent += ((const uint64_t*)v.mv_data)[1];
631  }
632  else
633  {
634  CHECK_AND_ASSERT_THROW_MES(dbr == MDB_NOTFOUND, "Failed to get per amount outputs: " + std::string(mdb_strerror(dbr)));
635  }
636  uint64_t data[2] = {total, spent};
637  v.mv_size = 2 * sizeof(uint64_t);
638  v.mv_data = (void*)data;
639  dbr = mdb_cursor_put(cur, &k, &v, 0);
640  mdb_cursor_close(cur);
641 }
642 
643 static uint64_t get_processed_txidx(const std::string &name)
644 {
645  MDB_txn *txn;
646  bool tx_active = false;
647 
648  int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
649  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
651  tx_active = true;
652 
653  uint64_t height = 0;
654  MDB_val k, v;
655  k.mv_data = (void*)name.c_str();
656  k.mv_size = name.size();
657  dbr = mdb_get(txn, dbi_processed_txidx, &k, &v);
658  if (dbr != MDB_NOTFOUND)
659  {
660  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get processed height: " + std::string(mdb_strerror(dbr)));
661  height = *(const uint64_t*)v.mv_data;
662  }
663 
664  dbr = mdb_txn_commit(txn);
665  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr)));
666  tx_active = false;
667 
668  return height;
669 }
670 
671 static void set_processed_txidx(MDB_txn *txn, const std::string &name, uint64_t height)
672 {
673  MDB_val k, v;
674  k.mv_data = (void*)name.c_str();
675  k.mv_size = name.size();
676  v.mv_data = (void*)&height;
677  v.mv_size = sizeof(height);
678  int dbr = mdb_put(txn, dbi_processed_txidx, &k, &v, 0);
679  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set processed height: " + std::string(mdb_strerror(dbr)));
680 }
681 
682 static bool get_relative_ring(MDB_txn *txn, const crypto::key_image &ki, std::vector<uint64_t> &ring)
683 {
684  MDB_val k, v;
685  k.mv_data = (void*)&ki;
686  k.mv_size = sizeof(ki);
687  int dbr = mdb_get(txn, dbi_relative_rings, &k, &v);
688  if (dbr == MDB_NOTFOUND)
689  return false;
690  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get relative ring: " + std::string(mdb_strerror(dbr)));
691  ring = decompress_ring(std::string((const char*)v.mv_data, v.mv_size));
692  return true;
693 }
694 
695 static void set_relative_ring(MDB_txn *txn, const crypto::key_image &ki, const std::vector<uint64_t> &ring)
696 {
697  const std::string sring = compress_ring(ring);
698  MDB_val k, v;
699  k.mv_data = (void*)&ki;
700  k.mv_size = sizeof(ki);
701  v.mv_data = (void*)sring.c_str();
702  v.mv_size = sring.size();
703  int dbr = mdb_put(txn, dbi_relative_rings, &k, &v, 0);
704  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set relative ring: " + std::string(mdb_strerror(dbr)));
705 }
706 
707 static std::string keep_under_511(const std::string &s)
708 {
709  if (s.size() <= 511)
710  return s;
712  crypto::cn_fast_hash(s.data(), s.size(), hash);
713  return std::string((const char*)&hash, 32);
714 }
715 
716 static uint64_t get_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring)
717 {
718  const std::string sring = keep_under_511(compress_ring(amount, ring));
719  MDB_val k, v;
720  k.mv_data = (void*)sring.data();
721  k.mv_size = sring.size();
722  int dbr = mdb_get(txn, dbi_ring_instances, &k, &v);
723  if (dbr == MDB_NOTFOUND)
724  return 0;
725  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get ring instances: " + std::string(mdb_strerror(dbr)));
726  return *(const uint64_t*)v.mv_data;
727 }
728 
729 static uint64_t get_ring_subset_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring)
730 {
731  uint64_t instances = get_ring_instances(txn, amount, ring);
732  if (ring.size() > 11)
733  return instances;
734 
735  uint64_t extra = 0;
736  std::vector<uint64_t> subset;
737  subset.reserve(ring.size());
738  for (uint64_t mask = 1; mask < (((uint64_t)1) << ring.size()) - 1; ++mask)
739  {
740  subset.resize(0);
741  for (size_t i = 0; i < ring.size(); ++i)
742  if ((mask >> i) & 1)
743  subset.push_back(ring[i]);
744  extra += get_ring_instances(txn, amount, subset);
745  }
746  return instances + extra;
747 }
748 
749 static uint64_t inc_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring)
750 {
751  const std::string sring = keep_under_511(compress_ring(amount, ring));
752  MDB_val k, v;
753  k.mv_data = (void*)sring.data();
754  k.mv_size = sring.size();
755 
756  int dbr = mdb_get(txn, dbi_ring_instances, &k, &v);
757  CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get ring instances: " + std::string(mdb_strerror(dbr)));
758 
759  uint64_t count;
760  if (dbr == MDB_NOTFOUND)
761  count = 1;
762  else
763  count = 1 + *(const uint64_t*)v.mv_data;
764 
765  v.mv_data = &count;
766  v.mv_size = sizeof(count);
767  dbr = mdb_put(txn, dbi_ring_instances, &k, &v, 0);
768  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set ring instances: " + std::string(mdb_strerror(dbr)));
769 
770  return count;
771 }
772 
773 static std::vector<crypto::key_image> get_key_images(MDB_txn *txn, const output_data &od)
774 {
775  MDB_val k, v;
776  k.mv_data = (void*)&od;
777  k.mv_size = sizeof(od);
778  int dbr = mdb_get(txn, dbi_outputs, &k, &v);
779  CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output: " + std::string(mdb_strerror(dbr)));
780  if (dbr == MDB_NOTFOUND)
781  return {};
782  CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size");
783  std::vector<crypto::key_image> key_images;
784  key_images.reserve(v.mv_size / 32);
785  const crypto::key_image *ki = (const crypto::key_image*)v.mv_data;
786  for (size_t n = 0; n < v.mv_size / 32; ++n)
787  key_images.push_back(*ki++);
788  return key_images;
789 }
790 
791 static void add_key_image(MDB_txn *txn, const output_data &od, const crypto::key_image &ki)
792 {
793  MDB_val k, v;
794  k.mv_data = (void*)&od;
795  k.mv_size = sizeof(od);
796  int dbr = mdb_get(txn, dbi_outputs, &k, &v);
797  CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output");
798  std::string data;
799  if (!dbr)
800  {
801  CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size");
802  data = std::string((const char*)v.mv_data, v.mv_size);
803  }
804  data += std::string((const char*)&ki, sizeof(ki));
805 
806  v.mv_data = (void*)data.data();
807  v.mv_size = data.size();
808  dbr = mdb_put(txn, dbi_outputs, &k, &v, 0);
809  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set outputs: " + std::string(mdb_strerror(dbr)));
810 }
811 
812 static bool get_stat(MDB_txn *txn, const char *key, uint64_t &data)
813 {
814  MDB_val k, v;
815  k.mv_data = (void*)key;
816  k.mv_size = strlen(key);
817  int dbr = mdb_get(txn, dbi_stats, &k, &v);
818  if (dbr == MDB_NOTFOUND)
819  return false;
820  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get stat record");
821  CHECK_AND_ASSERT_THROW_MES(v.mv_size == sizeof(uint64_t), "Unexpected record size");
822  data = *(const uint64_t*)v.mv_data;
823  return true;
824 }
825 
826 static void set_stat(MDB_txn *txn, const char *key, uint64_t data)
827 {
828  MDB_val k, v;
829  k.mv_data = (void*)key;
830  k.mv_size = strlen(key);
831  v.mv_data = (void*)&data;
832  v.mv_size = sizeof(uint64_t);
833  int dbr = mdb_put(txn, dbi_stats, &k, &v, 0);
834  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set stat record");
835 }
836 
837 static void inc_stat(MDB_txn *txn, const char *key)
838 {
839  uint64_t data;
840  if (!get_stat(txn, key, data))
841  data = 0;
842  ++data;
843  set_stat(txn, key, data);
844 }
845 
846 static void open_db(const std::string &filename, MDB_env **env, MDB_txn **txn, MDB_cursor **cur, MDB_dbi *dbi)
847 {
849 
850  int flags = MDB_RDONLY;
851  if (db_flags & DBF_FAST)
852  flags |= MDB_NOSYNC;
853  if (db_flags & DBF_FASTEST)
854  flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC;
855 
856  int dbr = mdb_env_create(env);
857  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
858  dbr = mdb_env_set_maxdbs(*env, 1);
859  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
860  const std::string actual_filename = filename;
861  MINFO("Opening electroneum blockchain at " << actual_filename);
862  dbr = mdb_env_open(*env, actual_filename.c_str(), flags, 0664);
863  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '"
864  + actual_filename + "': " + std::string(mdb_strerror(dbr)));
865 
866  dbr = mdb_txn_begin(*env, NULL, MDB_RDONLY, txn);
867  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
868 
869  dbr = mdb_dbi_open(*txn, "output_amounts", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, dbi);
870  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
871  mdb_set_dupsort(*txn, *dbi, compare_uint64);
872 
873  dbr = mdb_cursor_open(*txn, *dbi, cur);
874  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
875 }
876 
877 static void close_db(MDB_env *env, MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi)
878 {
879  mdb_txn_abort(txn);
880  mdb_cursor_close(cur);
881  mdb_dbi_close(env, dbi);
882  mdb_env_close(env);
883 }
884 
885 static void get_num_outputs(MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi, uint64_t &pre_rct, uint64_t &rct)
886 {
887  uint64_t amount = 0;
888  MDB_val k = { sizeof(amount), (void*)&amount }, v;
889  int dbr = mdb_cursor_get(cur, &k, &v, MDB_SET);
890  if (dbr == MDB_NOTFOUND)
891  {
892  rct = 0;
893  }
894  else
895  {
896  if (dbr) throw std::runtime_error("Record 0 not found: " + std::string(mdb_strerror(dbr)));
897  mdb_size_t count = 0;
898  dbr = mdb_cursor_count(cur, &count);
899  if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr)));
900  rct = count;
901  }
902  MDB_stat s;
903  dbr = mdb_stat(txn, dbi, &s);
904  if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr)));
905  if (s.ms_entries < rct) throw std::runtime_error("Inconsistent records: " + std::string(mdb_strerror(dbr)));
906  pre_rct = s.ms_entries - rct;
907 }
908 
909 static crypto::hash get_genesis_block_hash(const std::string &filename)
910 {
911  MDB_env *env;
912  MDB_dbi dbi;
913  MDB_txn *txn;
914  int dbr;
915  bool tx_active = false;
916 
917  dbr = mdb_env_create(&env);
918  if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
919  dbr = mdb_env_set_maxdbs(env, 1);
920  if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
921  const std::string actual_filename = filename;
922  dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664);
923  if (dbr) throw std::runtime_error("Failed to open rings database file '"
924  + actual_filename + "': " + std::string(mdb_strerror(dbr)));
925 
926  dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
927  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
929  tx_active = true;
930 
931  dbr = mdb_dbi_open(txn, "block_info", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi);
932  mdb_set_dupsort(txn, dbi, compare_uint64);
933  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
934  uint64_t zero = 0;
935  MDB_val k = { sizeof(uint64_t), (void*)&zero}, v;
936  dbr = mdb_get(txn, dbi, &k, &v);
937  if (dbr) throw std::runtime_error("Failed to retrieve genesis block: " + std::string(mdb_strerror(dbr)));
938  crypto::hash genesis_block_hash = *(const crypto::hash*)(((const uint64_t*)v.mv_data) + 5);
939  mdb_dbi_close(env, dbi);
940  mdb_txn_abort(txn);
941  mdb_env_close(env);
942  tx_active = false;
943  return genesis_block_hash;
944 }
945 
946 static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const std::string &filename)
947 {
948  std::vector<std::pair<uint64_t, uint64_t>> outputs;
949  uint64_t amount = std::numeric_limits<uint64_t>::max();
950  FILE *f;
951 
952  f = fopen(filename.c_str(), "r");
953  if (!f)
954  {
955  MERROR("Failed to load outputs from " << filename << ": " << strerror(errno));
956  return {};
957  }
958  while (1)
959  {
960  char s[256];
961  if (!fgets(s, sizeof(s), f))
962  {
963  MERROR("Error reading from " << filename << ": " << strerror(errno));
964  break;
965  }
966  if (feof(f))
967  break;
968  const size_t len = strlen(s);
969  if (len > 0 && s[len - 1] == '\n')
970  s[len - 1] = 0;
971  if (!s[0])
972  continue;
973  std::pair<uint64_t, uint64_t> output;
974  uint64_t offset, num_offsets;
975  if (sscanf(s, "@%" PRIu64, &amount) == 1)
976  {
977  continue;
978  }
979  if (amount == std::numeric_limits<uint64_t>::max())
980  {
981  MERROR("Bad format in " << filename);
982  continue;
983  }
984  if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits<uint64_t>::max() - offset)
985  {
986  while (num_offsets-- > 0)
987  outputs.push_back(std::make_pair(amount, offset++));
988  }
989  else if (sscanf(s, "%" PRIu64, &offset) == 1)
990  {
991  outputs.push_back(std::make_pair(amount, offset));
992  }
993  else
994  {
995  MERROR("Bad format in " << filename);
996  continue;
997  }
998  }
999  fclose(f);
1000  return outputs;
1001 }
1002 
1003 static bool export_spent_outputs(MDB_cursor *cur, const std::string &filename)
1004 {
1005  FILE *f = fopen(filename.c_str(), "w");
1006  if (!f)
1007  {
1008  MERROR("Failed to open " << filename << ": " << strerror(errno));
1009  return false;
1010  }
1011 
1012  uint64_t pending_amount = std::numeric_limits<uint64_t>::max();
1013  std::vector<uint64_t> pending_offsets;
1014  MDB_val k, v;
1015  MDB_cursor_op op = MDB_FIRST;
1016  while (1)
1017  {
1018  int dbr = mdb_cursor_get(cur, &k, &v, op);
1019  if (dbr == MDB_NOTFOUND)
1020  break;
1021  op = MDB_NEXT;
1022  if (dbr)
1023  {
1024  fclose(f);
1025  MERROR("Failed to enumerate spent outputs: " << mdb_strerror(dbr));
1026  return false;
1027  }
1028  const uint64_t amount = *(const uint64_t*)k.mv_data;
1029  const uint64_t offset = *(const uint64_t*)v.mv_data;
1030  if (!pending_offsets.empty() && (amount != pending_amount || pending_offsets.back()+1 != offset))
1031  {
1032  if (pending_offsets.size() == 1)
1033  fprintf(f, "%" PRIu64 "\n", pending_offsets.front());
1034  else
1035  fprintf(f, "%" PRIu64 "*%zu\n", pending_offsets.front(), pending_offsets.size());
1036  pending_offsets.clear();
1037  }
1038  if (pending_amount != amount)
1039  {
1040  fprintf(f, "@%" PRIu64 "\n", amount);
1041  pending_amount = amount;
1042  }
1043  pending_offsets.push_back(offset);
1044  }
1045  if (!pending_offsets.empty())
1046  {
1047  if (pending_offsets.size() == 1)
1048  fprintf(f, "%" PRIu64 "\n", pending_offsets.front());
1049  else
1050  fprintf(f, "%" PRIu64 "*%zu\n", pending_offsets.front(), pending_offsets.size());
1051  pending_offsets.clear();
1052  }
1053  fclose(f);
1054  return true;
1055 }
1056 
1057 int main(int argc, char* argv[])
1058 {
1059  TRY_ENTRY();
1060 
1062 
1063  std::string default_db_type = "lmdb";
1064 
1065  std::string available_dbs = cryptonote::blockchain_db_types(", ");
1066  available_dbs = "available: " + available_dbs;
1067 
1068  uint32_t log_level = 0;
1069 
1071 
1072  boost::filesystem::path output_file_path;
1073 
1074  po::options_description desc_cmd_only("Command line options");
1075  po::options_description desc_cmd_sett("Command line options and settings options");
1076  const command_line::arg_descriptor<std::string> arg_blackball_db_dir = {
1077  "spent-output-db-dir", "Specify spent output database directory",
1078  get_default_db_path(),
1079  };
1080  const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
1081  const command_line::arg_descriptor<std::string> arg_database = {
1082  "database", available_dbs.c_str(), default_db_type
1083  };
1084  const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false};
1085  const command_line::arg_descriptor<bool> arg_check_subsets = {"check-subsets", "Check ring subsets (very expensive)", false};
1086  const command_line::arg_descriptor<bool> arg_verbose = {"verbose", "Verbose output)", false};
1087  const command_line::arg_descriptor<std::vector<std::string> > arg_inputs = {"inputs", "Path to Electroneum DB, and path to any fork DBs"};
1089  "db-sync-mode"
1090  , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]."
1091  , "fast:1000"
1092  };
1093  const command_line::arg_descriptor<std::string> arg_extra_spent_list = {"extra-spent-list", "Optional list of known spent outputs",""};
1094  const command_line::arg_descriptor<std::string> arg_export = {"export", "Filename to export the backball list to"};
1095  const command_line::arg_descriptor<bool> arg_force_chain_reaction_pass = {"force-chain-reaction-pass", "Run the chain reaction pass even if no new blockchain data was processed"};
1096 
1097  command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir);
1098  command_line::add_arg(desc_cmd_sett, arg_log_level);
1099  command_line::add_arg(desc_cmd_sett, arg_database);
1100  command_line::add_arg(desc_cmd_sett, arg_rct_only);
1101  command_line::add_arg(desc_cmd_sett, arg_check_subsets);
1102  command_line::add_arg(desc_cmd_sett, arg_verbose);
1103  command_line::add_arg(desc_cmd_sett, arg_db_sync_mode);
1104  command_line::add_arg(desc_cmd_sett, arg_extra_spent_list);
1105  command_line::add_arg(desc_cmd_sett, arg_export);
1106  command_line::add_arg(desc_cmd_sett, arg_force_chain_reaction_pass);
1107  command_line::add_arg(desc_cmd_sett, arg_inputs);
1109 
1110  po::options_description desc_options("Allowed options");
1111  desc_options.add(desc_cmd_only).add(desc_cmd_sett);
1112 
1113  po::positional_options_description positional_options;
1114  positional_options.add(arg_inputs.name, -1);
1115 
1116  po::variables_map vm;
1117  bool r = command_line::handle_error_helper(desc_options, [&]()
1118  {
1119  auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options);
1120  po::store(parser.run(), vm);
1121  po::notify(vm);
1122  return true;
1123  });
1124  if (! r)
1125  return 1;
1126 
1128  {
1129  std::cout << "Electroneum '" << ELECTRONEUM_RELEASE_NAME << "' (v" << ELECTRONEUM_VERSION_FULL << ")" << ENDL << ENDL;
1130  std::cout << desc_options << std::endl;
1131  return 1;
1132  }
1133 
1134  mlog_configure(mlog_get_default_log_path("electroneum-blockchain-mark-spent-outputs.log"), true);
1137  else
1138  mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
1139 
1140  LOG_PRINT_L0("Starting...");
1141 
1142  output_file_path = command_line::get_arg(vm, arg_blackball_db_dir);
1143  bool opt_rct_only = command_line::get_arg(vm, arg_rct_only);
1144  bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets);
1145  bool opt_verbose = command_line::get_arg(vm, arg_verbose);
1146  bool opt_force_chain_reaction_pass = command_line::get_arg(vm, arg_force_chain_reaction_pass);
1147  std::string opt_export = command_line::get_arg(vm, arg_export);
1148  std::string extra_spent_list = command_line::get_arg(vm, arg_extra_spent_list);
1149  std::vector<std::pair<uint64_t, uint64_t>> extra_spent_outputs = extra_spent_list.empty() ? std::vector<std::pair<uint64_t, uint64_t>>() : load_outputs(extra_spent_list);
1150 
1151  std::string db_type = command_line::get_arg(vm, arg_database);
1153  {
1154  std::cerr << "Invalid database type: " << db_type << std::endl;
1155  return 1;
1156  }
1157 
1159  if (!parse_db_sync_mode(db_sync_mode))
1160  {
1161  MERROR("Invalid db sync mode: " << db_sync_mode);
1162  return 1;
1163  }
1164 
1165  const std::vector<std::string> inputs = command_line::get_arg(vm, arg_inputs);
1166  if (inputs.empty())
1167  {
1168  LOG_PRINT_L0("No inputs given");
1169  return 1;
1170  }
1171 
1172  const std::string cache_dir = (output_file_path / "spent-outputs-cache").string();
1173  init(cache_dir);
1174 
1175  LOG_PRINT_L0("Scanning for spent outputs...");
1176 
1177  size_t done = 0;
1178 
1179  const uint64_t start_blackballed_outputs = get_num_spent_outputs();
1180 
1181  tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0])));
1182 
1183  bool stop_requested = false;
1184  tools::signal_handler::install([&stop_requested](int type) {
1185  stop_requested = true;
1186  });
1187 
1188  int dbr = resize_env(cache_dir.c_str());
1189  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr)));
1190 
1191  // open first db
1192  MDB_env *env0;
1193  MDB_txn *txn0;
1194  MDB_dbi dbi0;
1195  MDB_cursor *cur0;
1196  open_db(inputs[0], &env0, &txn0, &cur0, &dbi0);
1197 
1198  if (!extra_spent_outputs.empty())
1199  {
1200  MINFO("Adding " << extra_spent_outputs.size() << " extra spent outputs");
1201  MDB_txn *txn;
1202  int dbr = mdb_txn_begin(env, NULL, 0, &txn);
1203  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
1204  MDB_cursor *cur;
1205  dbr = mdb_cursor_open(txn, dbi_spent, &cur);
1206  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr)));
1207 
1208  std::vector<std::pair<uint64_t, uint64_t>> blackballs;
1209  for (const std::pair<uint64_t, uint64_t> &output: extra_spent_outputs)
1210  {
1211  if (!is_output_spent(cur, output_data(output.first, output.second)))
1212  {
1213  blackballs.push_back(output);
1214  if (add_spent_output(cur, output_data(output.first, output.second)))
1215  inc_stat(txn, output.first ? "pre-rct-extra" : "rct-ring-extra");
1216  }
1217  }
1218  if (!blackballs.empty())
1219  {
1220  ringdb.blackball(blackballs);
1221  blackballs.clear();
1222  }
1223  mdb_cursor_close(cur);
1224  dbr = mdb_txn_commit(txn);
1225  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr)));
1226  }
1227 
1228  for (size_t n = 0; n < inputs.size(); ++n)
1229  {
1230  const std::string canonical = boost::filesystem::canonical(inputs[n]).string();
1231  uint64_t start_idx = get_processed_txidx(canonical);
1232  if (n > 0 && start_idx == 0)
1233  {
1234  start_idx = find_first_diverging_transaction(inputs[0], inputs[n]);
1235  LOG_PRINT_L0("First diverging transaction at " << start_idx);
1236  }
1237  LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx);
1238  MDB_txn *txn;
1239  int dbr = mdb_txn_begin(env, NULL, 0, &txn);
1240  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
1241  MDB_cursor *cur;
1242  dbr = mdb_cursor_open(txn, dbi_spent, &cur);
1243  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr)));
1244  size_t records = 0;
1245  const std::string filename = inputs[n];
1246  std::vector<std::pair<uint64_t, uint64_t>> blackballs;
1247  uint64_t n_txes;
1248  for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool
1249  {
1250  std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1251  const bool miner_tx = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen);
1252  for (const auto &in: tx.vin)
1253  {
1254  if (in.type() != typeid(txin_to_key))
1255  continue;
1256  const auto &txin = boost::get<txin_to_key>(in);
1257  if (opt_rct_only && txin.amount != 0)
1258  continue;
1259 
1260  const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets);
1261  if (n == 0)
1262  for (uint64_t out: absolute)
1263  add_key_image(txn, output_data(txin.amount, out), txin.k_image);
1264 
1265  std::vector<uint64_t> relative_ring;
1266  std::vector<uint64_t> new_ring = canonicalize(txin.key_offsets);
1267  const uint32_t ring_size = txin.key_offsets.size();
1268  const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring);
1269  uint64_t pa_total = 0, pa_spent = 0;
1270  if (!opt_rct_only)
1271  get_per_amount_outputs(txn, txin.amount, pa_total, pa_spent);
1272  if (n == 0 && ring_size == 1)
1273  {
1274  const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[0]);
1275  if (opt_verbose)
1276  {
1277  MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in a 1-ring");
1278  std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1279  }
1280  blackballs.push_back(output);
1281  if (add_spent_output(cur, output_data(txin.amount, absolute[0])))
1282  inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1");
1283  }
1284  else if (n == 0 && instances == new_ring.size())
1285  {
1286  for (size_t o = 0; o < new_ring.size(); ++o)
1287  {
1288  const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]);
1289  if (opt_verbose)
1290  {
1291  MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings");
1292  std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1293  }
1294  blackballs.push_back(output);
1295  if (add_spent_output(cur, output_data(txin.amount, absolute[o])))
1296  inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings");
1297  }
1298  }
1299  else if (n == 0 && !opt_rct_only && pa_spent + 1 == pa_total)
1300  {
1301  for (size_t o = 0; o < pa_total; ++o)
1302  {
1303  const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, o);
1304  if (opt_verbose)
1305  {
1306  MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to as many outputs of that amount being spent as exist so far");
1307  std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1308  }
1309  blackballs.push_back(output);
1310  if (add_spent_output(cur, output_data(txin.amount, o)))
1311  inc_stat(txn, txin.amount ? "pre-rct-full-count" : "rct-full-count");
1312  }
1313  }
1314  else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size())
1315  {
1316  for (size_t o = 0; o < new_ring.size(); ++o)
1317  {
1318  const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]);
1319  if (opt_verbose)
1320  {
1321  MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings");
1322  std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1323  }
1324  blackballs.push_back(output);
1325  if (add_spent_output(cur, output_data(txin.amount, absolute[o])))
1326  inc_stat(txn, txin.amount ? "pre-rct-subset-rings" : "rct-subset-rings");
1327  }
1328  }
1329  else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring))
1330  {
1331  MDEBUG("Key image " << txin.k_image << " already seen: rings " <<
1332  boost::join(relative_ring | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") <<
1333  ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " "));
1334  std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1335  if (relative_ring != txin.key_offsets)
1336  {
1337  MDEBUG("Rings are different");
1338  std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1339  const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(relative_ring);
1340  const std::vector<uint64_t> r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets);
1341  std::vector<uint64_t> common;
1342  for (uint64_t out: r0)
1343  {
1344  if (std::find(r1.begin(), r1.end(), out) != r1.end())
1345  common.push_back(out);
1346  }
1347  if (common.empty())
1348  {
1349  MERROR("Rings for the same key image are disjoint");
1350  std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1351  }
1352  else if (common.size() == 1)
1353  {
1354  const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, common[0]);
1355  if (opt_verbose)
1356  {
1357  MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in rings with a single common element");
1358  std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1359  }
1360  blackballs.push_back(output);
1361  if (add_spent_output(cur, output_data(txin.amount, common[0])))
1362  inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack");
1363  }
1364  else
1365  {
1366  MDEBUG("The intersection has more than one element, it's still ok");
1367  std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1368  for (const auto &out: r0)
1369  if (std::find(common.begin(), common.end(), out) != common.end())
1370  new_ring.push_back(out);
1372  }
1373  }
1374  }
1375  if (n == 0)
1376  {
1377  set_relative_ring(txn, txin.k_image, new_ring);
1378  if (!opt_rct_only)
1379  inc_per_amount_outputs(txn, txin.amount, 0, 1);
1380  }
1381  }
1382  set_processed_txidx(txn, canonical, start_idx+1);
1383  if (!opt_rct_only)
1384  {
1385  for (const auto &out: tx.vout)
1386  {
1387  uint64_t amount = out.amount;
1388 
1389  if (opt_rct_only && amount != 0)
1390  continue;
1391  if (out.target.type() != typeid(txout_to_key))
1392  continue;
1393  inc_per_amount_outputs(txn, amount, 1, 0);
1394  }
1395  }
1396 
1397  ++records;
1398  if (records >= records_per_sync)
1399  {
1400  if (!blackballs.empty())
1401  {
1402  ringdb.blackball(blackballs);
1403  blackballs.clear();
1404  }
1405  mdb_cursor_close(cur);
1406  dbr = mdb_txn_commit(txn);
1407  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr)));
1408  int dbr = resize_env(cache_dir.c_str());
1409  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr)));
1410  dbr = mdb_txn_begin(env, NULL, 0, &txn);
1411  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
1412  dbr = mdb_cursor_open(txn, dbi_spent, &cur);
1413  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr)));
1414  records = 0;
1415  }
1416 
1417  if (stop_requested)
1418  {
1419  MINFO("Stopping scan...");
1420  return false;
1421  }
1422  return true;
1423  });
1424  mdb_cursor_close(cur);
1425  dbr = mdb_txn_commit(txn);
1426  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr)));
1427  LOG_PRINT_L0("blockchain from " << inputs[n] << " processed till tx idx " << start_idx);
1428  if (stop_requested)
1429  break;
1430  }
1431 
1432  std::vector<output_data> work_spent;
1433 
1434  if (stop_requested)
1435  goto skip_secondary_passes;
1436 
1437  if (opt_force_chain_reaction_pass || get_num_spent_outputs() > start_blackballed_outputs)
1438  {
1439  MDB_txn *txn;
1440  dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
1441  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
1442  work_spent = get_spent_outputs(txn);
1443  mdb_txn_abort(txn);
1444  }
1445 
1446  while (!work_spent.empty())
1447  {
1448  LOG_PRINT_L0("Secondary pass on " << work_spent.size() << " spent outputs");
1449 
1450  int dbr = resize_env(cache_dir.c_str());
1451  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr)));
1452 
1453  MDB_txn *txn;
1454  dbr = mdb_txn_begin(env, NULL, 0, &txn);
1455  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
1456  MDB_cursor *cur;
1457  dbr = mdb_cursor_open(txn, dbi_spent, &cur);
1458  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr)));
1459 
1460  std::vector<std::pair<uint64_t, uint64_t>> blackballs;
1461  std::vector<output_data> scan_spent = std::move(work_spent);
1462  work_spent.clear();
1463  for (const output_data &od: scan_spent)
1464  {
1465  std::vector<crypto::key_image> key_images = get_key_images(txn, od);
1466  for (const crypto::key_image &ki: key_images)
1467  {
1468  std::vector<uint64_t> relative_ring;
1469  CHECK_AND_ASSERT_THROW_MES(get_relative_ring(txn, ki, relative_ring), "Relative ring not found");
1470  std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(relative_ring);
1471  size_t known = 0;
1472  uint64_t last_unknown = 0;
1473  for (uint64_t out: absolute)
1474  {
1475  output_data new_od(od.amount, out);
1476  if (is_output_spent(cur, new_od))
1477  ++known;
1478  else
1479  last_unknown = out;
1480  }
1481  if (known == absolute.size() - 1)
1482  {
1483  const std::pair<uint64_t, uint64_t> output = std::make_pair(od.amount, last_unknown);
1484  if (opt_verbose)
1485  {
1486  MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in a " <<
1487  absolute.size() << "-ring where all other outputs are known to be spent");
1488  }
1489  blackballs.push_back(output);
1490  if (add_spent_output(cur, output_data(od.amount, last_unknown)))
1491  inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction");
1492  work_spent.push_back(output_data(od.amount, last_unknown));
1493  }
1494  }
1495 
1496  if (stop_requested)
1497  {
1498  MINFO("Stopping secondary passes. Secondary passes are not incremental, they will re-run fully.");
1499  return 0;
1500  }
1501  }
1502  if (!blackballs.empty())
1503  {
1504  ringdb.blackball(blackballs);
1505  blackballs.clear();
1506  }
1507  mdb_cursor_close(cur);
1508  dbr = mdb_txn_commit(txn);
1509  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr)));
1510  }
1511 
1512 skip_secondary_passes:
1513  uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs;
1514  LOG_PRINT_L0(std::to_string(diff) << " new outputs marked as spent, " << get_num_spent_outputs() << " total outputs marked as spent");
1515 
1516  MDB_txn *txn;
1517  dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
1518  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
1519  uint64_t pre_rct = 0, rct = 0;
1520  get_num_outputs(txn0, cur0, dbi0, pre_rct, rct);
1521  MINFO("Total pre-rct outputs: " << pre_rct);
1522  MINFO("Total rct outputs: " << rct);
1523  static const struct { const char *key; uint64_t base; } stat_keys[] = {
1524  { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct },
1525  { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct },
1526  { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct },
1527  { "pre-rct-full-count", pre_rct }, { "rct-full-count", rct },
1528  { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct },
1529  { "pre-rct-extra", pre_rct }, { "rct-ring-extra", rct },
1530  { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct },
1531  };
1532  for (const auto &key: stat_keys)
1533  {
1534  uint64_t data;
1535  if (!get_stat(txn, key.key, data))
1536  data = 0;
1537  float percent = key.base ? 100.0f * data / key.base : 0.0f;
1538  MINFO(key.key << ": " << data << " (" << percent << "%)");
1539  }
1540  mdb_txn_abort(txn);
1541 
1542  if (!opt_export.empty())
1543  {
1544  MDB_txn *txn;
1545  int dbr = mdb_txn_begin(env, NULL, 0, &txn);
1546  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
1547  MDB_cursor *cur;
1548  dbr = mdb_cursor_open(txn, dbi_spent, &cur);
1549  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr)));
1550  export_spent_outputs(cur, opt_export);
1551  mdb_cursor_close(cur);
1552  mdb_txn_abort(txn);
1553  }
1554 
1555  LOG_PRINT_L0("Blockchain spent output data exported OK");
1556  close_db(env0, txn0, cur0, dbi0);
1557  close();
1558  return 0;
1559 
1560  CATCH_ENTRY("Error", 1);
1561 }
const char *const ELECTRONEUM_RELEASE_NAME
#define MDB_NOTFOUND
Definition: lmdb.h:439
#define MERROR(x)
Definition: misc_log_ex.h:73
#define CHECK_AND_ASSERT_THROW_MES(expr, message)
Definition: misc_log_ex.h:173
int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
Set the size of the memory map to use for this environment.
void mdb_cursor_close(MDB_cursor *cursor)
Close a cursor handle.
Definition: lmdb.h:411
#define MDB_RDONLY
Definition: lmdb.h:320
int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp)
Return count of duplicates for current key.
#define MINFO(x)
Definition: misc_log_ex.h:75
int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi)
Open a database in the environment.
#define MDB_NODUPDATA
Definition: lmdb.h:369
bool set_module_name_and_folder(const std::string &path_to_process_)
Definition: string_tools.h:249
::std::string string
Definition: gtest-port.h:1097
int mdb_txn_commit(MDB_txn *txn)
Commit all the operations of a transaction into the database.
int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, unsigned int flags)
Store by cursor.
auto_scope_leave_caller create_scope_leave_handler(t_scope_leave_handler f)
int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat)
Retrieve statistics for a database.
#define DBF_FASTEST
#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message)
Definition: misc_log_ex.h:181
void mdb_dbi_close(MDB_env *env, MDB_dbi dbi)
Close a database handle. Normally unnecessary. Use with care:
void mlog_set_log(const char *log)
Definition: mlog.cpp:288
#define CATCH_ENTRY(location, return_val)
Definition: misc_log_ex.h:152
std::string mlog_get_default_log_path(const char *default_filename)
Definition: mlog.cpp:72
uint64_t height
Definition: blockchain.cpp:91
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size=MAX_LOG_FILE_SIZE, const std::size_t max_log_files=MAX_LOG_FILES)
Definition: mlog.cpp:148
provides the implementation of varint&#39;s
const char * key
Definition: hmac_keccak.cpp:39
int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
Set a custom data comparison function for a MDB_DUPSORT database.
#define LOG_PRINT_L0(x)
Definition: misc_log_ex.h:99
int mdb_env_info(MDB_env *env, MDB_envinfo *stat)
Return information about the LMDB environment.
const command_line::arg_descriptor< std::string > arg_db_sync_mode
int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs)
Set the maximum number of named databases for the environment.
const arg_descriptor< bool > arg_help
#define DBF_SAFE
std::string blockchain_db_types(const std::string &sep)
std::vector< uint64_t > relative_output_offsets_to_absolute(const std::vector< uint64_t > &off)
output_data(uint64_t a, uint64_t i)
struct MDB_env MDB_env
Opaque structure for a database environment.
Definition: lmdb.h:260
int main(int argc, char *argv[])
std::string & trim(std::string &str)
Definition: string_tools.h:288
const char * name
#define DBF_FAST
#define MDB_MAPASYNC
Definition: lmdb.h:326
for(i=1;i< 1;++i) fe_sq(t0
#define MDEBUG(x)
Definition: misc_log_ex.h:76
struct MDB_txn MDB_txn
Opaque structure for a transaction handle.
Definition: lmdb.h:267
#define MDB_CREATE
Definition: lmdb.h:357
std::string get_default_data_dir()
Returns the default data directory.
Definition: util.cpp:600
bool on_startup()
Definition: util.cpp:778
Holds cryptonote related classes and helpers.
Definition: ban.cpp:40
Definition: options.h:86
const char *const ELECTRONEUM_VERSION_FULL
bool blockchain_valid_db_type(const std::string &db_type)
int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode)
Open an environment handle.
mdb_size_t count(MDB_cursor *cur)
Statistics for a database in the environment.
Definition: lmdb.h:490
Information about the environment.
Definition: lmdb.h:501
std::string pod_to_hex(const t_pod_type &s)
Definition: string_tools.h:317
void * mv_data
Definition: lmdb.h:288
bool handle_error_helper(const boost::program_options::options_description &desc, F parser)
Definition: command_line.h:237
#define TRY_ENTRY()
Definition: misc_log_ex.h:151
unsigned int uint32_t
Definition: stdint.h:126
#define MDB_WRITEMAP
Definition: lmdb.h:324
const command_line::arg_descriptor< std::string > arg_log_level
size_t mv_size
Definition: lmdb.h:287
Definition: lmdb.h:422
std::vector< uint64_t > absolute_output_offsets_to_relative(const std::vector< uint64_t > &off)
boost::shared_ptr< call_befor_die_base > auto_scope_leave_caller
mdb_size_t me_last_pgno
Definition: lmdb.h:504
unsigned __int64 uint64_t
Definition: stdint.h:136
#define MDB_INTEGERKEY
Definition: lmdb.h:349
#define MDB_DUPFIXED
Definition: lmdb.h:351
int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
Set a custom key comparison function for a database.
unsigned int MDB_dbi
A handle for an individual database in the DB environment.
Definition: lmdb.h:270
int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
Create a transaction for use with the environment.
MDB_cursor_op
Cursor Get operations.
Definition: lmdb.h:398
#define MWARNING(x)
Definition: misc_log_ex.h:74
void do_serialize(boost::mpl::false_, Archive &a, epee::net_utils::network_address &na)
void cn_fast_hash(const void *data, size_t length, char *hash)
void add_arg(boost::program_options::options_description &description, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg, bool unique=true)
Definition: command_line.h:188
int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, MDB_cursor_op op)
Retrieve by cursor.
int mdb_env_create(MDB_env **env)
Create an LMDB environment handle.
void mdb_txn_abort(MDB_txn *txn)
Abandon all the operations of the transaction instead of saving them.
bool operator==(const output_data &other) const
static bool install(T t)
installs a signal handler
Definition: util.h:164
const GenericPointer< typename T::ValueType > T2 T::AllocatorType & a
Definition: pointer.h:1124
std::string blobdata
Definition: blobdatatype.h:39
struct MDB_cursor MDB_cursor
Opaque structure for navigating through a database.
Definition: lmdb.h:273
Generic structure used for passing keys and data in and out of the database.
Definition: lmdb.h:286
const T & move(const T &t)
Definition: gtest-port.h:1317
unsigned int ms_psize
Definition: lmdb.h:491
#define PRIu64
Definition: inttypes.h:142
#define ENDL
Definition: misc_log_ex.h:149
POD_CLASS key_image
Definition: crypto.h:102
T get_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, false, true > &arg)
Definition: command_line.h:271
#define MDB_KEYEXIST
Definition: lmdb.h:437
bool create_directories_if_necessary(const std::string &path)
creates directories for a path
Definition: util.cpp:625
int mdb_env_stat(MDB_env *env, MDB_stat *stat)
Return statistics about the LMDB environment.
char * mdb_strerror(int err)
Return a string describing a given error code.
mdb_size_t me_mapsize
Definition: lmdb.h:503
POD_CLASS hash
Definition: hash.h:50
std::string to_string(t_connection_type type)
else if(0==res)
#define MDB_DUPSORT
Definition: lmdb.h:345
size_t mdb_size_t
Definition: lmdb.h:196
int compare_uint64(const MDB_val *a, const MDB_val *b)
mdb_size_t ms_entries
Definition: lmdb.h:497
key zero()
Definition: rctOps.h:70
int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data)
Get items from a database.
#define MDB_NOSYNC
Definition: lmdb.h:318
int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned int flags)
Store items into a database.
void mdb_env_close(MDB_env *env)
Close the environment and release the memory map.
bool is_arg_defaulted(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
Definition: command_line.h:265
int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor)
Create a cursor handle.
std::enable_if< std::is_integral< T >::value &&std::is_unsigned< T >::value, void >::type write_varint(OutputIt &&dest, T i)
writes a varint to a stream.
Definition: varint.h:70