Electroneum
bootstrap_file.cpp
Go to the documentation of this file.
1 // Copyrights(c) 2017-2021, The Electroneum Project
2 // Copyrights(c) 2014-2019, The Monero Project
3 //
4 // All rights reserved.
5 //
6 // Redistribution and use in source and binary forms, with or without modification, are
7 // permitted provided that the following conditions are met:
8 //
9 // 1. Redistributions of source code must retain the above copyright notice, this list of
10 // conditions and the following disclaimer.
11 //
12 // 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 // of conditions and the following disclaimer in the documentation and/or other
14 // materials provided with the distribution.
15 //
16 // 3. Neither the name of the copyright holder nor the names of its contributors may be
17 // used to endorse or promote products derived from this software without specific
18 // prior written permission.
19 //
20 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
21 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
22 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
23 // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
28 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 
31 #include "serialization/binary_utils.h" // dump_binary(), parse_binary()
32 #include "serialization/json_utils.h" // dump_json()
33 
34 #include "bootstrap_file.h"
35 
36 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY
37 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "bcutil"
38 
39 namespace po = boost::program_options;
40 
41 using namespace cryptonote;
42 using namespace epee;
43 
44 namespace
45 {
46  // This number was picked by taking the leading 4 bytes from this output:
47  // echo Electroneum bootstrap file | sha1sum
48  const uint32_t blockchain_raw_magic = 0x28721586;
49  const uint32_t header_size = 1024;
50 
51  std::string refresh_string = "\r \r";
52 }
53 
54 
55 
56 bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
57 {
58  const boost::filesystem::path dir_path = file_path.parent_path();
59  if (!dir_path.empty())
60  {
61  if (boost::filesystem::exists(dir_path))
62  {
63  if (!boost::filesystem::is_directory(dir_path))
64  {
65  MFATAL("export directory path is a file: " << dir_path);
66  return false;
67  }
68  }
69  else
70  {
71  if (!boost::filesystem::create_directory(dir_path))
72  {
73  MFATAL("Failed to create directory " << dir_path);
74  return false;
75  }
76  }
77  }
78 
79  m_raw_data_file = new std::ofstream();
80 
81  bool do_initialize_file = false;
82  uint64_t num_blocks = 0;
83 
84  if (! boost::filesystem::exists(file_path))
85  {
86  MDEBUG("creating file");
87  do_initialize_file = true;
88  num_blocks = 0;
89  }
90  else
91  {
92  num_blocks = count_blocks(file_path.string());
93  MDEBUG("appending to existing file with height: " << num_blocks-1 << " total blocks: " << num_blocks);
94  }
95  m_height = num_blocks;
96 
97  if (do_initialize_file)
98  m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
99  else
100  m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::app | std::ios::ate);
101 
102  if (m_raw_data_file->fail())
103  return false;
104 
105  m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
106  if (m_output_stream == nullptr)
107  return false;
108 
109  if (do_initialize_file)
110  initialize_file();
111 
112  return true;
113 }
114 
115 
117 {
118  const uint32_t file_magic = blockchain_raw_magic;
119 
120  std::string blob;
121  if (! ::serialization::dump_binary(file_magic, blob))
122  {
123  throw std::runtime_error("Error in serialization of file magic");
124  }
125  *m_raw_data_file << blob;
126 
128  bfi.major_version = 1;
129  bfi.minor_version = 0;
130  bfi.header_size = header_size;
131 
133  bbi.block_first = 0;
134  bbi.block_last = 0;
135  bbi.block_last_pos = 0;
136 
137  buffer_type buffer2;
138  boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>> output_stream_header(buffer2);
139 
140  uint32_t bd_size = 0;
141 
143  MDEBUG("bootstrap::file_info size: " << bd.size());
144  bd_size = bd.size();
145 
146  if (! ::serialization::dump_binary(bd_size, blob))
147  {
148  throw std::runtime_error("Error in serialization of bootstrap::file_info size");
149  }
150  output_stream_header << blob;
151  output_stream_header << bd;
152 
154  MDEBUG("bootstrap::blocks_info size: " << bd.size());
155  bd_size = bd.size();
156 
157  if (! ::serialization::dump_binary(bd_size, blob))
158  {
159  throw std::runtime_error("Error in serialization of bootstrap::blocks_info size");
160  }
161  output_stream_header << blob;
162  output_stream_header << bd;
163 
164  output_stream_header.flush();
165  output_stream_header << std::string(header_size-buffer2.size(), 0); // fill in rest with null bytes
166  output_stream_header.flush();
167  std::copy(buffer2.begin(), buffer2.end(), std::ostreambuf_iterator<char>(*m_raw_data_file));
168 
169  return true;
170 }
171 
173 {
174  m_output_stream->flush();
175 
176  uint32_t chunk_size = m_buffer.size();
177  // MTRACE("chunk_size " << chunk_size);
178  if (chunk_size > BUFFER_SIZE)
179  {
180  MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
181  }
182 
183  std::string blob;
184  if (! ::serialization::dump_binary(chunk_size, blob))
185  {
186  throw std::runtime_error("Error in serialization of chunk size");
187  }
188  *m_raw_data_file << blob;
189 
190  if (m_max_chunk < chunk_size)
191  {
192  m_max_chunk = chunk_size;
193  }
194  long pos_before = m_raw_data_file->tellp();
195  std::copy(m_buffer.begin(), m_buffer.end(), std::ostreambuf_iterator<char>(*m_raw_data_file));
196  m_raw_data_file->flush();
197  long pos_after = m_raw_data_file->tellp();
198  long num_chars_written = pos_after - pos_before;
199  if (static_cast<unsigned long>(num_chars_written) != chunk_size)
200  {
201  MFATAL("Error writing chunk: height: " << m_cur_height << " chunk_size: " << chunk_size << " num chars written: " << num_chars_written);
202  throw std::runtime_error("Error writing chunk");
203  }
204 
205  m_buffer.clear();
206  delete m_output_stream;
207  m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
208  MDEBUG("flushed chunk: chunk_size: " << chunk_size);
209 }
210 
212 {
214  bp.block = block;
215 
216  std::vector<transaction> txs;
217 
218  uint64_t block_height = boost::get<txin_gen>(block.miner_tx.vin.front()).height;
219 
220 
221  // now add all regular transactions
222  for (const auto& tx_id : block.tx_hashes)
223  {
224  if (tx_id == crypto::null_hash)
225  {
226  throw std::runtime_error("Aborting: tx == null_hash");
227  }
228  transaction tx = m_blockchain_storage->get_db().get_tx(tx_id);
229 
230  txs.push_back(tx);
231  }
232 
233  // these non-coinbase txs will be serialized using this structure
234  bp.txs = txs;
235 
236  // These three attributes are currently necessary for a fast import that adds blocks without verification.
237  bool include_extra_block_data = true;
238  if (include_extra_block_data)
239  {
240  size_t block_weight = m_blockchain_storage->get_db().get_block_weight(block_height);
241  difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height);
242  uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height);
243 
244  bp.block_weight = block_weight;
245  bp.cumulative_difficulty = cumulative_difficulty;
246  bp.coins_generated = coins_generated;
247  }
248 
250  m_output_stream->write((const char*)bd.data(), bd.size());
251 }
252 
254 {
255  if (m_raw_data_file->fail())
256  return false;
257 
258  m_raw_data_file->flush();
259  delete m_output_stream;
260  delete m_raw_data_file;
261  return true;
262 }
263 
264 
265 bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop)
266 {
267  uint64_t num_blocks_written = 0;
268  m_max_chunk = 0;
269  m_blockchain_storage = _blockchain_storage;
270  m_tx_pool = _tx_pool;
271  uint64_t progress_interval = 100;
272  MINFO("Storing blocks raw data...");
273  if (!BootstrapFile::open_writer(output_file))
274  {
275  MFATAL("failed to open raw file for write");
276  return false;
277  }
278  block b;
279 
280  // block_start, block_stop use 0-based height. m_height uses 1-based height. So to resume export
281  // from last exported block, block_start doesn't need to add 1 here, as it's already at the next
282  // height.
283  uint64_t block_start = m_height;
284  uint64_t block_stop = 0;
285  MINFO("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()-1);
286  if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
287  {
288  MINFO("Using requested block height: " << requested_block_stop);
289  block_stop = requested_block_stop;
290  }
291  else
292  {
293  block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
294  MINFO("Using block height of source blockchain: " << block_stop);
295  }
296  for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
297  {
298  // this method's height refers to 0-based height (genesis block = height 0)
299  crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height);
300  m_blockchain_storage->get_block_by_hash(hash, b);
301  write_block(b);
302  if (m_cur_height % NUM_BLOCKS_PER_CHUNK == 0) {
303  flush_chunk();
304  num_blocks_written += NUM_BLOCKS_PER_CHUNK;
305  }
306  if (m_cur_height % progress_interval == 0) {
307  std::cout << refresh_string;
308  std::cout << "block " << m_cur_height << "/" << block_stop << "\r" << std::flush;
309  }
310  }
311  // NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later supported.
312  if (m_cur_height % NUM_BLOCKS_PER_CHUNK != 0)
313  {
314  flush_chunk();
315  }
316  // print message for last block, which may not have been printed yet due to progress_interval
317  std::cout << refresh_string;
318  std::cout << "block " << m_cur_height-1 << "/" << block_stop << ENDL;
319 
320  MINFO("Number of blocks exported: " << num_blocks_written);
321  if (num_blocks_written > 0)
322  MINFO("Largest chunk: " << m_max_chunk << " bytes");
323 
324  return BootstrapFile::close();
325 }
326 
327 uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version)
328 {
329  uint32_t file_magic;
330 
331  std::string str1;
332  char buf1[2048];
333  import_file.read(buf1, sizeof(file_magic));
334  if (! import_file)
335  throw std::runtime_error("Error reading expected number of bytes");
336  str1.assign(buf1, sizeof(file_magic));
337 
338  if (! ::serialization::parse_binary(str1, file_magic))
339  throw std::runtime_error("Error in deserialization of file_magic");
340 
341  if (file_magic != blockchain_raw_magic)
342  {
343  MFATAL("bootstrap file not recognized");
344  throw std::runtime_error("Aborting");
345  }
346  else
347  MINFO("bootstrap file recognized");
348 
349  uint32_t buflen_file_info;
350 
351  import_file.read(buf1, sizeof(buflen_file_info));
352  str1.assign(buf1, sizeof(buflen_file_info));
353  if (! import_file)
354  throw std::runtime_error("Error reading expected number of bytes");
355  if (! ::serialization::parse_binary(str1, buflen_file_info))
356  throw std::runtime_error("Error in deserialization of buflen_file_info");
357  MINFO("bootstrap::file_info size: " << buflen_file_info);
358 
359  if (buflen_file_info > sizeof(buf1))
360  throw std::runtime_error("Error: bootstrap::file_info size exceeds buffer size");
361  import_file.read(buf1, buflen_file_info);
362  if (! import_file)
363  throw std::runtime_error("Error reading expected number of bytes");
364  str1.assign(buf1, buflen_file_info);
366  if (! ::serialization::parse_binary(str1, bfi))
367  throw std::runtime_error("Error in deserialization of bootstrap::file_info");
368  MINFO("bootstrap file v" << unsigned(bfi.major_version) << "." << unsigned(bfi.minor_version));
369  MINFO("bootstrap magic size: " << sizeof(file_magic));
370  MINFO("bootstrap header size: " << bfi.header_size);
371 
372  uint64_t full_header_size = sizeof(file_magic) + bfi.header_size;
373  import_file.seekg(full_header_size);
374 
375  major_version = bfi.major_version;
376  minor_version = bfi.minor_version;
377  return full_header_size;
378 }
379 
380 uint64_t BootstrapFile::count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit)
381 {
382  uint64_t bytes_read = 0;
383  uint32_t chunk_size;
384  char buf1[sizeof(chunk_size)];
385  std::string str1;
386  h = 0;
387  while (1)
388  {
389  import_file.read(buf1, sizeof(chunk_size));
390  if (!import_file) {
391  std::cout << refresh_string;
392  MDEBUG("End of file reached");
393  quit = true;
394  break;
395  }
396  bytes_read += sizeof(chunk_size);
397  str1.assign(buf1, sizeof(chunk_size));
398  if (! ::serialization::parse_binary(str1, chunk_size))
399  throw std::runtime_error("Error in deserialization of chunk_size");
400  MDEBUG("chunk_size: " << chunk_size);
401 
402  if (chunk_size > BUFFER_SIZE)
403  {
404  std::cout << refresh_string;
405  MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE
406  << " height: " << h-1 << ", offset " << bytes_read);
407  throw std::runtime_error("Aborting: chunk size exceeds buffer size");
408  }
409  if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD)
410  {
411  std::cout << refresh_string;
412  MDEBUG("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD << " << height: "
413  << h-1 << ", offset " << bytes_read);
414  }
415  else if (chunk_size <= 0) {
416  std::cout << refresh_string;
417  MDEBUG("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h-1 << ", offset " << bytes_read);
418  throw std::runtime_error("Aborting");
419  }
420  // skip to next expected block size value
421  import_file.seekg(chunk_size, std::ios_base::cur);
422  if (! import_file) {
423  std::cout << refresh_string;
424  MFATAL("ERROR: unexpected end of file: bytes read before error: "
425  << import_file.gcount() << " of chunk_size " << chunk_size);
426  throw std::runtime_error("Aborting");
427  }
428  bytes_read += chunk_size;
430  if (h >= blocks)
431  break;
432  }
433  return bytes_read;
434 }
435 
437 {
438  std::streampos dummy_pos;
439  uint64_t dummy_height = 0;
440  return count_blocks(import_file_path, dummy_pos, dummy_height);
441 }
442 
443 // If seek_height is non-zero on entry, return a stream position <= this height when finished.
444 // And return the actual height corresponding to this position. Allows the caller to locate its
445 // starting position without having to reread the entire file again.
446 uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::streampos &start_pos, uint64_t& seek_height)
447 {
448  boost::filesystem::path raw_file_path(import_file_path);
449  boost::system::error_code ec;
450  if (!boost::filesystem::exists(raw_file_path, ec))
451  {
452  MFATAL("bootstrap file not found: " << raw_file_path);
453  throw std::runtime_error("Aborting");
454  }
455  std::ifstream import_file;
456  import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in);
457 
458  uint64_t start_height = seek_height;
459  uint64_t h = 0;
460  if (import_file.fail())
461  {
462  MFATAL("import_file.open() fail");
463  throw std::runtime_error("Aborting");
464  }
465 
466  uint64_t full_header_size; // 4 byte magic + length of header structures
467  uint8_t major_version, minor_version;
468  full_header_size = seek_to_first_chunk(import_file, major_version, minor_version);
469 
470  MINFO("Scanning blockchain from bootstrap file...");
471  bool quit = false;
472  uint64_t bytes_read = 0, blocks;
473  int progress_interval = 10;
474 
475  while (! quit)
476  {
477  if (start_height && h + progress_interval >= start_height - 1)
478  {
479  start_height = 0;
480  start_pos = import_file.tellg();
481  seek_height = h;
482  }
483  bytes_read += count_bytes(import_file, progress_interval, blocks, quit);
484  h += blocks;
485  std::cout << "\r" << "block height: " << h-1 <<
486  " \r" <<
487  std::flush;
488 
489  // std::cout << refresh_string;
490  MDEBUG("Number bytes scanned: " << bytes_read);
491  }
492 
493  import_file.close();
494 
495  std::cout << ENDL;
496  std::cout << "Done scanning bootstrap file" << ENDL;
497  std::cout << "Full header length: " << full_header_size << " bytes" << ENDL;
498  std::cout << "Scanned for blocks: " << bytes_read << " bytes" << ENDL;
499  std::cout << "Total: " << full_header_size + bytes_read << " bytes" << ENDL;
500  std::cout << "Number of blocks: " << h << ENDL;
501  std::cout << ENDL;
502 
503  // NOTE: h is the number of blocks.
504  // Note that a block's stored height is zero-based, but parts of the code use
505  // one-based height.
506  return h;
507 }
std::vector< crypto::hash > tx_hashes
#define MINFO(x)
Definition: misc_log_ex.h:75
uint64_t count_bytes(std::ifstream &import_file, uint64_t blocks, uint64_t &h, bool &quit)
bool open_writer(const boost::filesystem::path &file_path)
#define MFATAL(x)
Definition: misc_log_ex.h:72
::std::string string
Definition: gtest-port.h:1097
uint64_t height
Definition: blockchain.cpp:91
uint64_t num_blocks(const std::vector< test_event_entry > &events)
Definition: chaingen.cpp:1044
std::vector< char > buffer_type
#define BUFFER_SIZE
bool parse_binary(const std::string &blob, T &v)
Definition: binary_utils.h:41
unsigned char uint8_t
Definition: stdint.h:124
void copy(key &AA, const key &A)
Definition: rctOps.h:79
bool dump_binary(T &v, std::string &blob)
Definition: binary_utils.h:51
#define MDEBUG(x)
Definition: misc_log_ex.h:76
uint64_t count_blocks(const std::string &dir_path, std::streampos &start_pos, uint64_t &seek_height)
Holds cryptonote related classes and helpers.
Definition: ban.cpp:40
#define NUM_BLOCKS_PER_CHUNK
unsigned int uint32_t
Definition: stdint.h:126
#define CHUNK_SIZE_WARNING_THRESHOLD
unsigned __int64 uint64_t
Definition: stdint.h:136
bool t_serializable_object_to_blob(const t_object &to, blobdata &b_blob)
#define MWARNING(x)
Definition: misc_log_ex.h:74
Transaction pool, handles transactions which are not part of a block.
Definition: tx_pool.h:94
std::string blobdata
Definition: blobdatatype.h:39
bool store_blockchain_raw(cryptonote::Blockchain *cs, cryptonote::tx_memory_pool *txp, boost::filesystem::path &output_file, uint64_t use_block_height=0)
boost::multiprecision::uint128_t difficulty_type
Definition: difficulty.h:43
#define ENDL
Definition: misc_log_ex.h:149
uint64_t seek_to_first_chunk(std::ifstream &import_file, uint8_t &major_version, uint8_t &minor_version)
POD_CLASS hash
Definition: hash.h:50
void write_block(block &block)