36 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY 37 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "bcutil" 39 namespace po = boost::program_options;
48 const uint32_t blockchain_raw_magic = 0x28721586;
58 const boost::filesystem::path dir_path = file_path.parent_path();
59 if (!dir_path.empty())
61 if (boost::filesystem::exists(dir_path))
63 if (!boost::filesystem::is_directory(dir_path))
65 MFATAL(
"export directory path is a file: " << dir_path);
71 if (!boost::filesystem::create_directory(dir_path))
73 MFATAL(
"Failed to create directory " << dir_path);
79 m_raw_data_file =
new std::ofstream();
81 bool do_initialize_file =
false;
84 if (! boost::filesystem::exists(file_path))
87 do_initialize_file =
true;
97 if (do_initialize_file)
98 m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
100 m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::app | std::ios::ate);
102 if (m_raw_data_file->fail())
105 m_output_stream =
new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
106 if (m_output_stream ==
nullptr)
109 if (do_initialize_file)
118 const uint32_t file_magic = blockchain_raw_magic;
123 throw std::runtime_error(
"Error in serialization of file magic");
125 *m_raw_data_file << blob;
138 boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>> output_stream_header(buffer2);
143 MDEBUG(
"bootstrap::file_info size: " << bd.size());
148 throw std::runtime_error(
"Error in serialization of bootstrap::file_info size");
150 output_stream_header << blob;
151 output_stream_header << bd;
154 MDEBUG(
"bootstrap::blocks_info size: " << bd.size());
159 throw std::runtime_error(
"Error in serialization of bootstrap::blocks_info size");
161 output_stream_header << blob;
162 output_stream_header << bd;
164 output_stream_header.flush();
165 output_stream_header <<
std::string(header_size-buffer2.size(), 0);
166 output_stream_header.flush();
167 std::copy(buffer2.begin(), buffer2.end(), std::ostreambuf_iterator<char>(*m_raw_data_file));
174 m_output_stream->flush();
176 uint32_t chunk_size = m_buffer.size();
186 throw std::runtime_error(
"Error in serialization of chunk size");
188 *m_raw_data_file << blob;
190 if (m_max_chunk < chunk_size)
192 m_max_chunk = chunk_size;
194 long pos_before = m_raw_data_file->tellp();
195 std::copy(m_buffer.begin(), m_buffer.end(), std::ostreambuf_iterator<char>(*m_raw_data_file));
196 m_raw_data_file->flush();
197 long pos_after = m_raw_data_file->tellp();
198 long num_chars_written = pos_after - pos_before;
199 if (static_cast<unsigned long>(num_chars_written) != chunk_size)
201 MFATAL(
"Error writing chunk: height: " << m_cur_height <<
" chunk_size: " << chunk_size <<
" num chars written: " << num_chars_written);
202 throw std::runtime_error(
"Error writing chunk");
206 delete m_output_stream;
207 m_output_stream =
new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
208 MDEBUG(
"flushed chunk: chunk_size: " << chunk_size);
216 std::vector<transaction> txs;
224 if (tx_id == crypto::null_hash)
226 throw std::runtime_error(
"Aborting: tx == null_hash");
228 transaction tx = m_blockchain_storage->get_db().get_tx(tx_id);
237 bool include_extra_block_data =
true;
238 if (include_extra_block_data)
240 size_t block_weight = m_blockchain_storage->get_db().get_block_weight(block_height);
241 difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height);
242 uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height);
250 m_output_stream->write((
const char*)bd.data(), bd.size());
255 if (m_raw_data_file->fail())
258 m_raw_data_file->flush();
259 delete m_output_stream;
260 delete m_raw_data_file;
269 m_blockchain_storage = _blockchain_storage;
270 m_tx_pool = _tx_pool;
272 MINFO(
"Storing blocks raw data...");
275 MFATAL(
"failed to open raw file for write");
285 MINFO(
"source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()-1);
286 if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
288 MINFO(
"Using requested block height: " << requested_block_stop);
289 block_stop = requested_block_stop;
293 block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
294 MINFO(
"Using block height of source blockchain: " << block_stop);
296 for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
299 crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height);
300 m_blockchain_storage->get_block_by_hash(
hash, b);
306 if (m_cur_height % progress_interval == 0) {
307 std::cout << refresh_string;
308 std::cout <<
"block " << m_cur_height <<
"/" << block_stop <<
"\r" << std::flush;
317 std::cout << refresh_string;
318 std::cout <<
"block " << m_cur_height-1 <<
"/" << block_stop <<
ENDL;
320 MINFO(
"Number of blocks exported: " << num_blocks_written);
321 if (num_blocks_written > 0)
322 MINFO(
"Largest chunk: " << m_max_chunk <<
" bytes");
333 import_file.read(buf1,
sizeof(file_magic));
335 throw std::runtime_error(
"Error reading expected number of bytes");
336 str1.assign(buf1,
sizeof(file_magic));
339 throw std::runtime_error(
"Error in deserialization of file_magic");
341 if (file_magic != blockchain_raw_magic)
343 MFATAL(
"bootstrap file not recognized");
344 throw std::runtime_error(
"Aborting");
347 MINFO(
"bootstrap file recognized");
351 import_file.read(buf1,
sizeof(buflen_file_info));
352 str1.assign(buf1,
sizeof(buflen_file_info));
354 throw std::runtime_error(
"Error reading expected number of bytes");
356 throw std::runtime_error(
"Error in deserialization of buflen_file_info");
357 MINFO(
"bootstrap::file_info size: " << buflen_file_info);
359 if (buflen_file_info >
sizeof(buf1))
360 throw std::runtime_error(
"Error: bootstrap::file_info size exceeds buffer size");
361 import_file.read(buf1, buflen_file_info);
363 throw std::runtime_error(
"Error reading expected number of bytes");
364 str1.assign(buf1, buflen_file_info);
367 throw std::runtime_error(
"Error in deserialization of bootstrap::file_info");
369 MINFO(
"bootstrap magic size: " <<
sizeof(file_magic));
373 import_file.seekg(full_header_size);
377 return full_header_size;
384 char buf1[
sizeof(chunk_size)];
389 import_file.read(buf1,
sizeof(chunk_size));
391 std::cout << refresh_string;
392 MDEBUG(
"End of file reached");
396 bytes_read +=
sizeof(chunk_size);
397 str1.assign(buf1,
sizeof(chunk_size));
399 throw std::runtime_error(
"Error in deserialization of chunk_size");
400 MDEBUG(
"chunk_size: " << chunk_size);
404 std::cout << refresh_string;
406 <<
" height: " << h-1 <<
", offset " << bytes_read);
407 throw std::runtime_error(
"Aborting: chunk size exceeds buffer size");
411 std::cout << refresh_string;
413 << h-1 <<
", offset " << bytes_read);
415 else if (chunk_size <= 0) {
416 std::cout << refresh_string;
417 MDEBUG(
"ERROR: chunk_size " << chunk_size <<
" <= 0" <<
" height: " << h-1 <<
", offset " << bytes_read);
418 throw std::runtime_error(
"Aborting");
421 import_file.seekg(chunk_size, std::ios_base::cur);
423 std::cout << refresh_string;
424 MFATAL(
"ERROR: unexpected end of file: bytes read before error: " 425 << import_file.gcount() <<
" of chunk_size " << chunk_size);
426 throw std::runtime_error(
"Aborting");
428 bytes_read += chunk_size;
438 std::streampos dummy_pos;
440 return count_blocks(import_file_path, dummy_pos, dummy_height);
448 boost::filesystem::path raw_file_path(import_file_path);
449 boost::system::error_code ec;
450 if (!boost::filesystem::exists(raw_file_path, ec))
452 MFATAL(
"bootstrap file not found: " << raw_file_path);
453 throw std::runtime_error(
"Aborting");
455 std::ifstream import_file;
456 import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in);
458 uint64_t start_height = seek_height;
460 if (import_file.fail())
462 MFATAL(
"import_file.open() fail");
463 throw std::runtime_error(
"Aborting");
467 uint8_t major_version, minor_version;
468 full_header_size = seek_to_first_chunk(import_file, major_version, minor_version);
470 MINFO(
"Scanning blockchain from bootstrap file...");
473 int progress_interval = 10;
477 if (start_height && h + progress_interval >= start_height - 1)
480 start_pos = import_file.tellg();
483 bytes_read += count_bytes(import_file, progress_interval,
blocks, quit);
485 std::cout <<
"\r" <<
"block height: " << h-1 <<
490 MDEBUG(
"Number bytes scanned: " << bytes_read);
496 std::cout <<
"Done scanning bootstrap file" <<
ENDL;
497 std::cout <<
"Full header length: " << full_header_size <<
" bytes" <<
ENDL;
498 std::cout <<
"Scanned for blocks: " << bytes_read <<
" bytes" <<
ENDL;
499 std::cout <<
"Total: " << full_header_size + bytes_read <<
" bytes" <<
ENDL;
500 std::cout <<
"Number of blocks: " << h <<
ENDL;
std::vector< crypto::hash > tx_hashes
uint64_t count_bytes(std::ifstream &import_file, uint64_t blocks, uint64_t &h, bool &quit)
bool open_writer(const boost::filesystem::path &file_path)
uint64_t num_blocks(const std::vector< test_event_entry > &events)
std::vector< char > buffer_type
bool parse_binary(const std::string &blob, T &v)
void copy(key &AA, const key &A)
bool dump_binary(T &v, std::string &blob)
uint64_t count_blocks(const std::string &dir_path, std::streampos &start_pos, uint64_t &seek_height)
Holds cryptonote related classes and helpers.
std::vector< txin_v > vin
#define NUM_BLOCKS_PER_CHUNK
difficulty_type cumulative_difficulty
#define CHUNK_SIZE_WARNING_THRESHOLD
std::vector< transaction > txs
unsigned __int64 uint64_t
bool t_serializable_object_to_blob(const t_object &to, blobdata &b_blob)
Transaction pool, handles transactions which are not part of a block.
bool store_blockchain_raw(cryptonote::Blockchain *cs, cryptonote::tx_memory_pool *txp, boost::filesystem::path &output_file, uint64_t use_block_height=0)
boost::multiprecision::uint128_t difficulty_type
uint64_t seek_to_first_chunk(std::ifstream &import_file, uint8_t &major_version, uint8_t &minor_version)
void write_block(block &block)