Electroneum
blockchain_usage.cpp
Go to the documentation of this file.
1 // Copyright (c) 2014-2019, The Monero Project
2 //
3 // All rights reserved.
4 //
5 // Redistribution and use in source and binary forms, with or without modification, are
6 // permitted provided that the following conditions are met:
7 //
8 // 1. Redistributions of source code must retain the above copyright notice, this list of
9 // conditions and the following disclaimer.
10 //
11 // 2. Redistributions in binary form must reproduce the above copyright notice, this list
12 // of conditions and the following disclaimer in the documentation and/or other
13 // materials provided with the distribution.
14 //
15 // 3. Neither the name of the copyright holder nor the names of its contributors may be
16 // used to endorse or promote products derived from this software without specific
17 // prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
20 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
21 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
22 // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
27 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 
29 #include <boost/range/adaptor/transformed.hpp>
30 #include <boost/algorithm/string.hpp>
31 #include "common/command_line.h"
32 #include "common/varint.h"
37 #include "blockchain_db/db_types.h"
38 #include "version.h"
39 
40 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY
41 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "bcutil"
42 
43 namespace po = boost::program_options;
44 using namespace epee;
45 using namespace cryptonote;
46 
47 struct output_data
48 {
49  uint64_t amount;
51  mutable bool coinbase;
52  mutable uint64_t height;
53  output_data(uint64_t a, uint64_t i, bool cb, uint64_t h): amount(a), index(i), coinbase(cb), height(h) {}
54  bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; }
55  void info(bool c, uint64_t h) const { coinbase = c; height =h; }
56 };
57 namespace std
58 {
59  template<> struct hash<output_data>
60  {
61  size_t operator()(const output_data &od) const
62  {
63  const uint64_t data[2] = {od.amount, od.index};
64  crypto::hash h;
65  crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h);
66  return reinterpret_cast<const std::size_t &>(h);
67  }
68  };
69 }
70 
71 struct reference
72 {
76  reference(uint64_t h, uint64_t rs, uint64_t p): height(h), ring_size(rs), position(p) {}
77 };
78 
79 int main(int argc, char* argv[])
80 {
81  TRY_ENTRY();
82 
84 
85  std::string default_db_type = "lmdb";
86 
87  std::string available_dbs = cryptonote::blockchain_db_types(", ");
88  available_dbs = "available: " + available_dbs;
89 
90  uint32_t log_level = 0;
91 
93 
94  boost::filesystem::path output_file_path;
95 
96  po::options_description desc_cmd_only("Command line options");
97  po::options_description desc_cmd_sett("Command line options and settings options");
98  const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
99  const command_line::arg_descriptor<std::string> arg_database = {
100  "database", available_dbs.c_str(), default_db_type
101  };
102  const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false};
103  const command_line::arg_descriptor<std::string> arg_input = {"input", ""};
104 
107  command_line::add_arg(desc_cmd_sett, arg_log_level);
108  command_line::add_arg(desc_cmd_sett, arg_database);
109  command_line::add_arg(desc_cmd_sett, arg_rct_only);
110  command_line::add_arg(desc_cmd_sett, arg_input);
112 
113  po::options_description desc_options("Allowed options");
114  desc_options.add(desc_cmd_only).add(desc_cmd_sett);
115 
116  po::positional_options_description positional_options;
117  positional_options.add(arg_input.name, -1);
118 
119  po::variables_map vm;
120  bool r = command_line::handle_error_helper(desc_options, [&]()
121  {
122  auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options);
123  po::store(parser.run(), vm);
124  po::notify(vm);
125  return true;
126  });
127  if (! r)
128  return 1;
129 
131  {
132  std::cout << "Electroneum '" << ELECTRONEUM_RELEASE_NAME << "' (v" << ELECTRONEUM_VERSION_FULL << ")" << ENDL << ENDL;
133  std::cout << desc_options << std::endl;
134  return 1;
135  }
136 
137  mlog_configure(mlog_get_default_log_path("electroneum-blockchain-usage.log"), true);
140  else
141  mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
142 
143  LOG_PRINT_L0("Starting...");
144 
145  bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
146  bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on);
147  network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
148  bool opt_rct_only = command_line::get_arg(vm, arg_rct_only);
149 
150  std::string db_type = command_line::get_arg(vm, arg_database);
152  {
153  std::cerr << "Invalid database type: " << db_type << std::endl;
154  return 1;
155  }
156 
157  // If we wanted to use the memory pool, we would set up a fake_core.
158 
159  // Use Blockchain instead of lower-level BlockchainDB for two reasons:
160  // 1. Blockchain has the init() method for easy setup
161  // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash()
162  //
163  // cannot match blockchain_storage setup above with just one line,
164  // e.g.
165  // Blockchain* core_storage = new Blockchain(NULL);
166  // because unlike blockchain_storage constructor, which takes a pointer to
167  // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object.
168  LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
169  const std::string input = command_line::get_arg(vm, arg_input);
170  std::unique_ptr<Blockchain> core_storage;
171  tx_memory_pool m_mempool(*core_storage);
172  core_storage.reset(new Blockchain(m_mempool));
173  BlockchainDB* db = new_db(db_type);
174  if (db == NULL)
175  {
176  LOG_ERROR("Attempted to use non-existent database type: " << db_type);
177  throw std::runtime_error("Attempting to use non-existent database type");
178  }
179  LOG_PRINT_L0("database: " << db_type);
180 
181  const std::string filename = input;
182  LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
183 
184  try
185  {
186  db->open(filename, DBF_RDONLY);
187  }
188  catch (const std::exception& e)
189  {
190  LOG_PRINT_L0("Error opening database: " << e.what());
191  return 1;
192  }
193  r = core_storage->init(db, net_type);
194 
195  CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
196  LOG_PRINT_L0("Source blockchain storage initialized OK");
197 
198  LOG_PRINT_L0("Building usage patterns...");
199 
200  size_t done = 0;
201  std::unordered_map<output_data, std::list<reference>> outputs;
202  std::unordered_map<uint64_t,uint64_t> indices;
203 
204  LOG_PRINT_L0("Reading blockchain from " << input);
205  core_storage->for_all_transactions([&](const crypto::hash &hash, const cryptonote::transaction &tx)->bool
206  {
207  const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen);
208  const uint64_t height = core_storage->get_db().get_tx_block_height(hash);
209 
210  // create new outputs
211  for (const auto &out: tx.vout)
212  {
213  if (opt_rct_only && out.amount)
214  continue;
215  uint64_t index = indices[out.amount]++;
216  output_data od(out.amount, indices[out.amount], coinbase, height);
217  auto itb = outputs.emplace(od, std::list<reference>());
218  itb.first->first.info(coinbase, height);
219  }
220 
221  for (const auto &in: tx.vin)
222  {
223  if (in.type() != typeid(txin_to_key))
224  continue;
225  const auto &txin = boost::get<txin_to_key>(in);
226  if (opt_rct_only && txin.amount != 0)
227  continue;
228 
229  const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets);
230  for (size_t n = 0; n < txin.key_offsets.size(); ++n)
231  {
232  output_data od(txin.amount, absolute[n], coinbase, height);
233  outputs[od].push_back(reference(height, txin.key_offsets.size(), n));
234  }
235  }
236  return true;
237  }, true);
238 
239  std::unordered_map<uint64_t, uint64_t> counts;
240  size_t total = 0;
241  for (const auto &out: outputs)
242  {
243  counts[out.second.size()]++;
244  total++;
245  }
246  if (total > 0)
247  {
248  for (const auto &c: counts)
249  {
250  float percent = 100.f * c.second / total;
251  MINFO(std::to_string(c.second) << " outputs used " << c.first << " times (" << percent << "%)");
252  }
253  }
254  else
255  {
256  MINFO("No outputs to process");
257  }
258 
259  LOG_PRINT_L0("Blockchain usage exported OK");
260  return 0;
261 
262  CATCH_ENTRY("Export error", 1);
263 }
const char *const ELECTRONEUM_RELEASE_NAME
uint64_t position
output_data(uint64_t a, uint64_t i, bool cb, uint64_t h)
reference(uint64_t h, uint64_t rs, uint64_t p)
virtual uint64_t get_tx_block_height(const crypto::hash &h) const =0
fetches the height of a transaction&#39;s block
#define DBF_RDONLY
#define MINFO(x)
Definition: misc_log_ex.h:75
bool set_module_name_and_folder(const std::string &path_to_process_)
Definition: string_tools.h:249
::std::string string
Definition: gtest-port.h:1097
#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message)
Definition: misc_log_ex.h:181
void mlog_set_log(const char *log)
Definition: mlog.cpp:288
#define CATCH_ENTRY(location, return_val)
Definition: misc_log_ex.h:152
std::string mlog_get_default_log_path(const char *default_filename)
Definition: mlog.cpp:72
uint64_t height
Definition: blockchain.cpp:91
bool for_all_transactions(std::function< bool(const crypto::hash &, const cryptonote::transaction &)>, bool pruned) const
perform a check on all transactions in the blockchain
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size=MAX_LOG_FILE_SIZE, const std::size_t max_log_files=MAX_LOG_FILES)
Definition: mlog.cpp:148
provides the implementation of varint&#39;s
#define LOG_PRINT_L0(x)
Definition: misc_log_ex.h:99
STL namespace.
const command_line::arg_descriptor< bool, false > arg_stagenet_on
const arg_descriptor< bool > arg_help
std::string blockchain_db_types(const std::string &sep)
std::vector< uint64_t > relative_output_offsets_to_absolute(const std::vector< uint64_t > &off)
virtual void open(const std::string &filename, const int db_flags=0)=0
open a db, or create it if necessary.
bool on_startup()
Definition: util.cpp:778
int main(int argc, char *argv[])
Holds cryptonote related classes and helpers.
Definition: ban.cpp:40
const char *const ELECTRONEUM_VERSION_FULL
bool blockchain_valid_db_type(const std::string &db_type)
const command_line::arg_descriptor< bool, false > arg_testnet_on
BlockchainDB * new_db(const std::string &db_type)
bool handle_error_helper(const boost::program_options::options_description &desc, F parser)
Definition: command_line.h:237
#define TRY_ENTRY()
Definition: misc_log_ex.h:151
unsigned int uint32_t
Definition: stdint.h:126
const command_line::arg_descriptor< std::string > arg_log_level
uint64_t height
unsigned __int64 uint64_t
Definition: stdint.h:136
void cn_fast_hash(const void *data, size_t length, char *hash)
void add_arg(boost::program_options::options_description &description, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg, bool unique=true)
Definition: command_line.h:188
const BlockchainDB & get_db() const
get a reference to the BlockchainDB in use by Blockchain
Definition: blockchain.h:963
bool operator==(const output_data &other) const
void info(bool c, uint64_t h) const
const GenericPointer< typename T::ValueType > T2 T::AllocatorType & a
Definition: pointer.h:1124
Transaction pool, handles transactions which are not part of a block.
Definition: tx_pool.h:94
size_t operator()(const output_data &od) const
The BlockchainDB backing store interface declaration/contract.
#define LOG_ERROR(x)
Definition: misc_log_ex.h:98
#define ENDL
Definition: misc_log_ex.h:149
T get_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, false, true > &arg)
Definition: command_line.h:271
bool init(BlockchainDB *db, const network_type nettype=MAINNET, bool offline=false, const cryptonote::test_options *test_options=NULL, difficulty_type fixed_difficulty=0, const GetCheckpointsCallback &get_checkpoints=nullptr, bool ignore_bsig=false, bool fallback_to_pow=false)
Initialize the Blockchain state.
Definition: blockchain.cpp:334
POD_CLASS hash
Definition: hash.h:50
uint64_t ring_size
std::string to_string(t_connection_type type)
bool is_arg_defaulted(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
Definition: command_line.h:265