Electroneum
blockchain_db.cpp
Go to the documentation of this file.
1 // Copyrights(c) 2017-2021, The Electroneum Project
2 // Copyrights(c) 2014-2019, The Monero Project
3 //
4 // All rights reserved.
5 //
6 // Redistribution and use in source and binary forms, with or without modification, are
7 // permitted provided that the following conditions are met:
8 //
9 // 1. Redistributions of source code must retain the above copyright notice, this list of
10 // conditions and the following disclaimer.
11 //
12 // 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 // of conditions and the following disclaimer in the documentation and/or other
14 // materials provided with the distribution.
15 //
16 // 3. Neither the name of the copyright holder nor the names of its contributors may be
17 // used to endorse or promote products derived from this software without specific
18 // prior written permission.
19 //
20 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
21 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
22 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
23 // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
28 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 
30 #include <boost/range/adaptor/reversed.hpp>
31 #include <unordered_set>
32 
33 #include "string_tools.h"
34 #include "blockchain_db.h"
36 #include "profile_tools.h"
37 #include "ringct/rctOps.h"
38 
39 #include "lmdb/db_lmdb.h"
40 #ifdef BERKELEY_DB
41 #include "berkeleydb/db_bdb.h"
42 #endif
43 
44 static const char *db_types[] = {
45  "lmdb",
46 #ifdef BERKELEY_DB
47  "berkeley",
48 #endif
49  NULL
50 };
51 
52 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY
53 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "blockchain.db"
54 
56 
57 namespace cryptonote
58 {
59 
61 {
62  int i;
63  for (i=0; db_types[i]; i++)
64  {
65  if (db_types[i] == db_type)
66  return true;
67  }
68  return false;
69 }
70 
72 {
73  int i;
74  std::string ret = "";
75  for (i=0; db_types[i]; i++)
76  {
77  if (i)
78  ret += sep;
79  ret += db_types[i];
80  }
81  return ret;
82 }
83 
84 std::string arg_db_type_description = "Specify database type, available: " + cryptonote::blockchain_db_types(", ");
86  "db-type"
88 , DEFAULT_DB_TYPE
89 };
91  "db-sync-mode"
92 , "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]]."
93 , "fast:async:250000000bytes"
94 };
96  "db-salvage"
97 , "Try to salvage a blockchain database if it seems corrupted"
98 , false
99 };
100 
102  "addr-db-salvage"
103 , "Try to salvage a addr tx database if it seems corrupted"
104 , false
105 };
106 
108 {
109  if (db_type == "lmdb")
110  return new BlockchainLMDB();
111 #if defined(BERKELEY_DB)
112  if (db_type == "berkeley")
113  return new BlockchainBDB();
114 #endif
115  return NULL;
116 }
117 
118 void BlockchainDB::init_options(boost::program_options::options_description& desc)
119 {
124 }
125 
126 void BlockchainDB::pop_block()
127 {
128  block blk;
129  std::vector<transaction> txs;
130  pop_block(blk, txs);
131 }
132 
133 void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr)
134 {
135  const transaction &tx = txp.first;
136 
137  bool miner_tx = false;
138  crypto::hash tx_hash, tx_prunable_hash;
139  if (!tx_hash_ptr)
140  {
141  // should only need to compute hash for miner transactions
142  tx_hash = get_transaction_hash(tx);
143  LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash);
144  }
145  else
146  {
147  tx_hash = *tx_hash_ptr;
148  }
149 
150  std::vector<std::pair<crypto::hash, uint64_t>> utxos_to_remove;
151  // keep a set of the etn addresses (derived from BOTH ins and outs) associated with the tx for removal from addr_tx db
152  std::unordered_set<cryptonote::account_public_address> addr_tx_addresses;
153 
154  // Sanity check on supported input types
155  for (size_t i = 0; i < tx.vin.size(); ++i)
156  {
157  const txin_v& tx_input = tx.vin[i];
158  if (tx_input.type() == typeid(txin_to_key))
159  {
160  add_spent_key(boost::get<txin_to_key>(tx_input).k_image);
161  }
162  else if (tx_input.type() == typeid(txin_to_key_public))
163  {
164  const auto &txin = boost::get<txin_to_key_public>(tx_input);
165  utxos_to_remove.push_back({txin.tx_hash, txin.relative_offset});
166  add_tx_input(txin.tx_hash, txin.relative_offset, tx.hash, i);
167 
168  //work for addr_tx db
169  transaction parent_tx = get_tx(txin.tx_hash);
170  const auto &txout = boost::get<txout_to_key_public>(parent_tx.vout[txin.relative_offset].target); //previous tx out that this tx in references
171  if(addr_tx_addresses.find(txout.address) == addr_tx_addresses.end()){ //if addr hasn't been used for another input yet, add the unique addr tx record for this address
172  add_addr_tx(tx.hash, addKeys(txout.address.m_view_public_key, txout.address.m_spend_public_key));
173  addr_tx_addresses.insert(txout.address);
174  }
175  }
176  else if (tx_input.type() == typeid(txin_gen))
177  {
178  /* nothing to do here */
179  miner_tx = true;
180  }
181  else
182  {
183  LOG_PRINT_L1("Unsupported input type, removing key images and aborting transaction addition");
184  for (const txin_v& tx_input : tx.vin)
185  {
186  if (tx_input.type() == typeid(txin_to_key))
187  {
188  remove_spent_key(boost::get<txin_to_key>(tx_input).k_image); // inputs are already checked here regardless of version
189  }
190  if (tx_input.type() == typeid(txin_to_key_public)) {
191  //rewind tx inputs added to tx input db if the transaction aborts
192  const auto &txin = boost::get<txin_to_key_public>(tx_input);
193  remove_tx_input(txin.tx_hash, txin.relative_offset);
194  //work for addr_tx db
195  transaction parent_tx = get_tx(txin.tx_hash);
196  const auto &txout = boost::get<txout_to_key_public>(parent_tx.vout[txin.relative_offset].target); //previous tx out that this tx in references
197  if (addr_tx_addresses.find(txout.address) != addr_tx_addresses.end()) { // dont do a remove for every input. there is only one entry per address per tx in the addr tx db
198  remove_addr_tx(tx.hash, addKeys(txout.address.m_view_public_key, txout.address.m_spend_public_key));
199  addr_tx_addresses.erase(txout.address);
200  }
201  }
202  }
203  return;
204  }
205  }
206 
207  if (tx.version == 1)
208  {
209  uint64_t tx_id = add_transaction_data(blk_hash, txp, tx_hash, tx_prunable_hash);
210 
211  std::vector<uint64_t> amount_output_indices(tx.vout.size());
212 
213  // iterate tx.vout using indices instead of C++11 foreach syntax because
214  // we need the index
215  for (uint64_t i = 0; i < tx.vout.size(); ++i)
216  {
217  amount_output_indices[i] = add_output(tx_hash, tx.vout[i], i, tx.unlock_time, NULL);
218  }
219  add_tx_amount_output_indices(tx_id, amount_output_indices);
220  }
221  else if (tx.version >= 2)
222  {
223  add_transaction_data(blk_hash, txp, tx_hash, tx_prunable_hash);
224 
225  // Sanity check on supported output types
226  for (uint64_t i = 0; i < tx.vout.size(); ++i)
227  {
228  if(tx.vout[i].target.type() != typeid(txout_to_key_public))
229  {
230  LOG_PRINT_L1("Unsupported output type, reinstating UTXOs, removing key images and aborting transaction addition");
231  for (const txin_v& tx_input : tx.vin) {
232  if (tx_input.type() == typeid(txin_to_key)) {
233  remove_spent_key(boost::get<txin_to_key>(tx_input).k_image);
234  }
235  if (tx_input.type() == typeid(txin_to_key_public)) {
236  //rewind tx inputs added to tx input db if the transaction aborts
237  const auto &txin = boost::get<txin_to_key_public>(tx_input);
238  remove_tx_input(txin.tx_hash, txin.relative_offset);
239  //work for addr_tx db
240  transaction parent_tx = get_tx(txin.tx_hash);
241  const auto &txout = boost::get<txout_to_key_public>(
242  parent_tx.vout[txin.relative_offset].target); //previous tx out that this tx in references
243  if (addr_tx_addresses.find(txout.address) !=
244  addr_tx_addresses.end()) { // dont do a remove for every input, only a remove for every addr that was uniquely added to the addr tx db
245  remove_addr_tx(tx.hash, addKeys(txout.address.m_view_public_key, txout.address.m_spend_public_key));
246  addr_tx_addresses.erase(txout.address);
247  }
248  }
249  }
250  return;
251  }// if outs are all of the right type, we're ok to proceed by removing the utxos that are now spent
252 
253  for(auto utxo: utxos_to_remove)
254  {
255  remove_chainstate_utxo(utxo.first, utxo.second);
256  }
257 
258  const auto &txout = boost::get<txout_to_key_public>(tx.vout[i].target);
259  add_chainstate_utxo(tx.hash, i, addKeys(txout.address.m_view_public_key, txout.address.m_spend_public_key) , tx.vout[i].amount, txp.first.unlock_time, miner_tx);
260  add_addr_output(tx.hash, i, addKeys(txout.address.m_view_public_key, txout.address.m_spend_public_key), tx.vout[i].amount, txp.first.unlock_time);
261  if(addr_tx_addresses.find(txout.address) == addr_tx_addresses.end()){ //if addr hasn't been used for another input yet, add the unique addr tx record for this address
262  add_addr_tx(tx.hash, addKeys(txout.address.m_view_public_key, txout.address.m_spend_public_key));
263  addr_tx_addresses.insert(txout.address);
264  }
265  }//end of v2+ processing
266  }
267 }
268 
269 uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
270  , size_t block_weight
271  , uint64_t long_term_block_weight
272  , const difficulty_type& cumulative_difficulty
273  , const uint64_t& coins_generated
274  , const std::vector<std::pair<transaction, blobdata>>& txs
275  )
276 {
277  const block &blk = blck.first;
278 
279  // sanity
280  if (blk.tx_hashes.size() != txs.size())
281  throw std::runtime_error("Inconsistent tx/hashes sizes");
282 
283  TIME_MEASURE_START(time1);
284  crypto::hash blk_hash = get_block_hash(blk);
285  TIME_MEASURE_FINISH(time1);
286  time_blk_hash += time1;
287 
288  uint64_t prev_height = height();
289 
290  // call out to add the transactions
291 
293 
294  add_transaction(blk_hash, std::make_pair(blk.miner_tx, tx_to_blob(blk.miner_tx)));
295  int tx_i = 0;
296  crypto::hash tx_hash = crypto::null_hash;
297  for (const std::pair<transaction, blobdata>& tx : txs)
298  {
299  tx_hash = blk.tx_hashes[tx_i];
300  add_transaction(blk_hash, tx, &tx_hash);
301  ++tx_i;
302  }
303  TIME_MEASURE_FINISH(time1);
304  time_add_transaction += time1;
305 
306  // call out to subclass implementation to add the block & metadata
308  add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, 0, blk_hash);
309  TIME_MEASURE_FINISH(time1);
310  time_add_block1 += time1;
311  //voting mechanism
312  m_hardfork->add(blk, prev_height);
313 
314  ++num_calls;
315 
316  return prev_height;
317 }
318 
320 {
321  m_hardfork = hf;
322 }
323 
324 void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs)
325 {
326  blk = get_top_block();
327 
328  remove_block();
329 
330  for (const auto& h : boost::adaptors::reverse(blk.tx_hashes))
331  {
333  if (!get_tx(h, tx) && !get_pruned_tx(h, tx))
334  throw DB_ERROR("Failed to get pruned or unpruned transaction from the db");
335  txs.push_back(std::move(tx));
336  remove_transaction(h);
337  }
338  remove_transaction(get_transaction_hash(blk.miner_tx));
339 }
340 
342 {
343  return m_open;
344 }
345 
346 void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
347 {
348  transaction tx = get_pruned_tx(tx_hash);
349 
350  // keep a set of the etn addresses (derived from BOTH ins and outs) associated with the tx for removal from addr_tx db
351  std::unordered_set<cryptonote::account_public_address> addr_tx_addresses;
352 
353  for (const txin_v& tx_input : tx.vin)
354  {
355  if (tx_input.type() == typeid(txin_to_key))
356  {
357  remove_spent_key(boost::get<txin_to_key>(tx_input).k_image);
358  }
359  else if (tx_input.type() == typeid(txin_to_key_public))
360  {
361  const auto &txin = boost::get<txin_to_key_public>(tx_input); // input being used in the tx to be removed.
362 
363  transaction parent_tx = get_tx(txin.tx_hash);
364  const auto &txout = boost::get<txout_to_key_public>(parent_tx.vout[txin.relative_offset].target); //previous tx out that this tx in references
365  //reinstate that out as a utxo
366  bool reinstate_coinbase = cryptonote::is_coinbase(get_pruned_tx(txin.tx_hash));
367  add_chainstate_utxo(txin.tx_hash, txin.relative_offset, addKeys(txout.address.m_view_public_key, txout.address.m_spend_public_key), txin.amount, get_tx_unlock_time(txin.tx_hash), reinstate_coinbase);
368  remove_tx_input(txin.tx_hash, txin.relative_offset);
369 
370 
371  if(addr_tx_addresses.find(txout.address) == addr_tx_addresses.end()){
372  remove_addr_tx(txin.tx_hash, addKeys(txout.address.m_view_public_key, txout.address.m_spend_public_key));
373  addr_tx_addresses.insert(txout.address);
374  }
375  }
376  }
377 
378  if (tx.version >= 2)
379  {
380  for (uint64_t i = 0; i < tx.vout.size(); ++i)
381  {
382  const auto &txout = boost::get<txout_to_key_public>(tx.vout[i].target);
383 
384  remove_chainstate_utxo(tx_hash, i);
385  remove_addr_output(tx_hash, i, addKeys(txout.address.m_view_public_key, txout.address.m_spend_public_key), tx.vout[i].amount, tx.unlock_time);
386 
387  // remove addr tx entries for outputs involving addr that weren't used for ins
388  if(addr_tx_addresses.find(txout.address) == addr_tx_addresses.end()){
389  remove_addr_tx(tx_hash, addKeys(txout.address.m_view_public_key, txout.address.m_spend_public_key));
390  addr_tx_addresses.insert(txout.address);
391  }
392  }
393  }
394 
395  // need tx as tx.vout has the tx outputs, and the output amounts are needed
396  remove_transaction_data(tx_hash, tx);
397 }
398 
400 {
402  block b;
404  throw DB_ERROR("Failed to parse block from blob retrieved from the db");
405 
406  return b;
407 }
408 
410 {
411  blobdata bd = get_block_blob(h);
412  block b;
414  throw DB_ERROR("Failed to parse block from blob retrieved from the db");
415 
416  return b;
417 }
418 
420 {
421  blobdata bd;
422  if (!get_tx_blob(h, bd))
423  return false;
424  if (!parse_and_validate_tx_from_blob(bd, tx))
425  throw DB_ERROR("Failed to parse transaction from blob retrieved from the db");
426 
427  return true;
428 }
429 
431 {
432  blobdata bd;
433  if (!get_pruned_tx_blob(h, bd))
434  return false;
436  throw DB_ERROR("Failed to parse transaction base from blob retrieved from the db");
437 
438  return true;
439 }
440 
442 {
443  transaction tx;
444  if (!get_tx(h, tx))
445  throw TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str());
446  return tx;
447 }
448 
450 {
451  transaction tx;
452  if (!get_pruned_tx(h, tx))
453  throw TX_DNE(std::string("pruned tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str());
454  return tx;
455 }
456 
458 {
459  num_calls = 0;
460  time_blk_hash = 0;
461  time_tx_exists = 0;
462  time_add_block1 = 0;
463  time_add_transaction = 0;
464  time_commit1 = 0;
465 }
466 
468 {
470  << "*********************************"
471  << ENDL
472  << "num_calls: " << num_calls
473  << ENDL
474  << "time_blk_hash: " << time_blk_hash << "ms"
475  << ENDL
476  << "time_tx_exists: " << time_tx_exists << "ms"
477  << ENDL
478  << "time_add_block1: " << time_add_block1 << "ms"
479  << ENDL
480  << "time_add_transaction: " << time_add_transaction << "ms"
481  << ENDL
482  << "time_commit1: " << time_commit1 << "ms"
483  << ENDL
484  << "*********************************"
485  << ENDL
486  );
487 }
488 
490 {
491  if (is_read_only()) {
492  LOG_PRINT_L1("Database is opened read only - skipping fixup check");
493  return;
494  }
495 
496  // Apply fixes to DB if needed.
497 
498  batch_stop();
499 }
500 
501 } // namespace cryptonote
bool is_coinbase(const transaction &tx)
virtual bool get_tx_blob(const crypto::hash &h, cryptonote::blobdata &tx) const =0
fetches the transaction blob with the given hash
std::vector< crypto::hash > tx_hashes
uint64_t get_tick_count()
virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t &height) const =0
fetch a block blob by height
#define LOG_PRINT_L1(x)
Definition: misc_log_ex.h:100
::std::string string
Definition: gtest-port.h:1097
uint64_t height
Definition: blockchain.cpp:91
boost::variant< txin_gen, txin_to_script, txin_to_scripthash, txin_to_key, txin_to_key_public > txin_v
const command_line::arg_descriptor< std::string > arg_db_sync_mode
virtual void remove_addr_tx(const crypto::hash tx_hash, const crypto::public_key &combined_key)=0
public_key addKeys(const public_key &A, const public_key &B)
Definition: crypto.h:339
std::string blockchain_db_types(const std::string &sep)
virtual bool get_pruned_tx_blob(const crypto::hash &h, cryptonote::blobdata &tx) const =0
fetches the pruned transaction blob with the given hash
void add_transaction(const crypto::hash &blk_hash, const std::pair< transaction, blobdata > &tx, const crypto::hash *tx_hash_ptr=NULL, const crypto::hash *tx_prunable_hash_ptr=NULL)
helper function for add_transactions, to add each individual transaction
thrown when a requested transaction does not exist
Holds cryptonote related classes and helpers.
Definition: ban.cpp:40
void show_stats()
show profiling stats
bool blockchain_valid_db_type(const std::string &db_type)
blobdata tx_to_blob(const transaction &tx)
std::string pod_to_hex(const t_pod_type &s)
Definition: string_tools.h:317
BlockchainDB * new_db(const std::string &db_type)
virtual block get_block_from_height(const uint64_t &height) const
fetch a block by height
bool add(const cryptonote::block &block, uint64_t height)
add a new block
Definition: hardfork.cpp:164
bool get_block_hash(const block &b, crypto::hash &res)
uint64_t time_tx_exists
a performance metric
virtual bool is_read_only() const =0
is BlockchainDB in read-only mode?
virtual transaction get_tx(const crypto::hash &h) const
fetches the transaction with the given hash
unsigned __int64 uint64_t
Definition: stdint.h:136
virtual block get_top_block() const =0
fetch the top block
virtual uint64_t height() const =0
fetch the current blockchain height
virtual void batch_stop()=0
ends a batch transaction
virtual uint64_t get_tx_unlock_time(const crypto::hash &h) const =0
fetch a transaction&#39;s unlock time/height
virtual transaction get_pruned_tx(const crypto::hash &h) const
fetches the transaction base with the given hash
virtual void add_addr_tx(const crypto::hash tx_hash, const crypto::public_key &combined_key)=0
#define TIME_MEASURE_START(var_name)
Definition: profile_tools.h:61
bool parse_and_validate_tx_base_from_blob(const blobdata &tx_blob, transaction &tx)
bool m_open
Whether or not the BlockchainDB is open/ready for use.
const command_line::arg_descriptor< std::string > arg_db_type
bool parse_and_validate_tx_from_blob(const blobdata &tx_blob, transaction &tx)
#define LOG_PRINT_L3(x)
Definition: misc_log_ex.h:102
#define TIME_MEASURE_FINISH(var_name)
Definition: profile_tools.h:64
void add_arg(boost::program_options::options_description &description, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg, bool unique=true)
Definition: command_line.h:188
bool is_open() const
Gets the current open/ready state of the BlockchainDB.
virtual void set_hard_fork(HardFork *hf)
std::string blobdata
Definition: blobdatatype.h:39
A generic BlockchainDB exception.
The BlockchainDB backing store interface declaration/contract.
const T & move(const T &t)
Definition: gtest-port.h:1317
boost::multiprecision::uint128_t difficulty_type
Definition: difficulty.h:43
#define ENDL
Definition: misc_log_ex.h:149
const command_line::arg_descriptor< bool > arg_addr_db_salvage
crypto::hash get_transaction_hash(const transaction &t)
virtual void fixup()
fix up anything that may be wrong due to past bugs
const command_line::arg_descriptor< bool > arg_db_salvage
static void init_options(boost::program_options::options_description &desc)
init command line options
std::string arg_db_type_description
POD_CLASS hash
Definition: hash.h:50
uint64_t time_commit1
a performance metric
virtual block get_block(const crypto::hash &h) const
fetches the block with the given hash
virtual cryptonote::blobdata get_block_blob(const crypto::hash &h) const =0
fetches the block with the given hash
void reset_stats()
reset profiling stats
bool parse_and_validate_block_from_blob(const blobdata &b_blob, block &b, crypto::hash *block_hash)