Electroneum
download.cpp
Go to the documentation of this file.
1 // Copyright (c) 2017-Present, Electroneum
2 // Copyright (c) 2017-2019, The Monero Project
3 //
4 // All rights reserved.
5 //
6 // Redistribution and use in source and binary forms, with or without modification, are
7 // permitted provided that the following conditions are met:
8 //
9 // 1. Redistributions of source code must retain the above copyright notice, this list of
10 // conditions and the following disclaimer.
11 //
12 // 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 // of conditions and the following disclaimer in the documentation and/or other
14 // materials provided with the distribution.
15 //
16 // 3. Neither the name of the copyright holder nor the names of its contributors may be
17 // used to endorse or promote products derived from this software without specific
18 // prior written permission.
19 //
20 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
21 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
22 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
23 // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
28 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 
30 #include <string>
31 #include <atomic>
32 #include <boost/filesystem.hpp>
33 #include <boost/thread/thread.hpp>
34 #include "file_io_utils.h"
35 #include "net/http_client.h"
36 #include "download.h"
37 
38 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY
39 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "net.dl"
40 
41 namespace tools
42 {
44  {
47  std::function<void(const std::string&, const std::string&, bool)> result_cb;
48  std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb;
49  bool stop;
50  bool stopped;
51  bool success;
52  boost::thread thread;
53  boost::mutex mutex;
54 
55  download_thread_control(const std::string &path, const std::string &uri, std::function<void(const std::string&, const std::string&, bool)> result_cb, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb):
57  ~download_thread_control() { if (thread.joinable()) thread.detach(); }
58  };
59 
60  static void download_thread(download_async_handle control)
61  {
62  static std::atomic<unsigned int> thread_id(0);
63 
64  MLOG_SET_THREAD_NAME("DL" + std::to_string(thread_id++));
65 
66  struct stopped_setter
67  {
68  stopped_setter(const download_async_handle &control): control(control) {}
69  ~stopped_setter() { control->stopped = true; }
70  download_async_handle control;
71  } stopped_setter(control);
72 
73  try
74  {
75  boost::unique_lock<boost::mutex> lock(control->mutex);
76  std::ios_base::openmode mode = std::ios_base::out | std::ios_base::binary;
77  uint64_t existing_size = 0;
78  if (epee::file_io_utils::get_file_size(control->path, existing_size) && existing_size > 0)
79  {
80  MINFO("Resuming downloading " << control->uri << " to " << control->path << " from " << existing_size);
81  mode |= std::ios_base::app;
82  }
83  else
84  {
85  MINFO("Downloading " << control->uri << " to " << control->path);
86  mode |= std::ios_base::trunc;
87  }
88  std::ofstream f;
89  f.open(control->path, mode);
90  if (!f.good()) {
91  MERROR("Failed to open file " << control->path);
92  control->result_cb(control->path, control->uri, control->success);
93  return;
94  }
95  class download_client: public epee::net_utils::http::http_simple_client
96  {
97  public:
98  download_client(download_async_handle control, std::ofstream &f, uint64_t offset = 0):
99  control(control), f(f), content_length(-1), total(0), offset(offset) {}
100  virtual ~download_client() { f.close(); }
101  virtual bool on_header(const epee::net_utils::http::http_response_info &headers)
102  {
103  for (const auto &kv: headers.m_header_info.m_etc_fields)
104  MDEBUG("Header: " << kv.first << ": " << kv.second);
105  ssize_t length;
107  {
108  MINFO("Content-Length: " << length);
109  content_length = length;
110  boost::filesystem::path path(control->path);
111  boost::filesystem::space_info si = boost::filesystem::space(path);
112  if (si.available < (size_t)content_length)
113  {
114  const uint64_t avail = (si.available + 1023) / 1024, needed = (content_length + 1023) / 1024;
115  MERROR("Not enough space to download " << needed << " kB to " << path << " (" << avail << " kB available)");
116  return false;
117  }
118  }
119  if (offset > 0)
120  {
121  // we requested a range, so check if we're getting it, otherwise truncate
122  bool got_range = false;
123  const std::string prefix = "bytes=" + std::to_string(offset) + "-";
124  for (const auto &kv: headers.m_header_info.m_etc_fields)
125  {
126  if (kv.first == "Content-Range" && strncmp(kv.second.c_str(), prefix.c_str(), prefix.size()))
127  {
128  got_range = true;
129  break;
130  }
131  }
132  if (!got_range)
133  {
134  MWARNING("We did not get the requested range, downloading from start");
135  f.close();
136  f.open(control->path, std::ios_base::out | std::ios_base::binary | std::ios_base::trunc);
137  }
138  }
139  return true;
140  }
141  virtual bool handle_target_data(std::string &piece_of_transfer)
142  {
143  try
144  {
145  boost::lock_guard<boost::mutex> lock(control->mutex);
146  if (control->stop)
147  return false;
148  f << piece_of_transfer;
149  total += piece_of_transfer.size();
150  if (control->progress_cb && !control->progress_cb(control->path, control->uri, total, content_length))
151  return false;
152  return f.good();
153  }
154  catch (const std::exception &e)
155  {
156  MERROR("Error writing data: " << e.what());
157  return false;
158  }
159  }
160  private:
161  download_async_handle control;
162  std::ofstream &f;
163  ssize_t content_length;
164  size_t total;
165  uint64_t offset;
166  } client(control, f, existing_size);
168  if (!epee::net_utils::parse_url(control->uri, u_c))
169  {
170  MERROR("Failed to parse URL " << control->uri);
171  control->result_cb(control->path, control->uri, control->success);
172  return;
173  }
174  if (u_c.host.empty())
175  {
176  MERROR("Failed to determine address from URL " << control->uri);
177  control->result_cb(control->path, control->uri, control->success);
178  return;
179  }
180 
181  lock.unlock();
182 
185  MDEBUG("Connecting to " << u_c.host << ":" << port);
186  client.set_server(u_c.host, std::to_string(port), boost::none, ssl);
187  if (!client.connect(std::chrono::seconds(30)))
188  {
189  boost::lock_guard<boost::mutex> lock(control->mutex);
190  MERROR("Failed to connect to " << control->uri);
191  control->result_cb(control->path, control->uri, control->success);
192  return;
193  }
194  MDEBUG("GETting " << u_c.uri);
197  if (existing_size > 0)
198  {
199  const std::string range = "bytes=" + std::to_string(existing_size) + "-";
200  MDEBUG("Asking for range: " << range);
201  fields.push_back(std::make_pair("Range", range));
202  }
203  if (!client.invoke_get(u_c.uri, std::chrono::seconds(30), "", &info, fields))
204  {
205  boost::lock_guard<boost::mutex> lock(control->mutex);
206  MERROR("Failed to connect to " << control->uri);
207  client.disconnect();
208  control->result_cb(control->path, control->uri, control->success);
209  return;
210  }
211  if (control->stop)
212  {
213  boost::lock_guard<boost::mutex> lock(control->mutex);
214  MDEBUG("Download cancelled");
215  client.disconnect();
216  control->result_cb(control->path, control->uri, control->success);
217  return;
218  }
219  if (!info)
220  {
221  boost::lock_guard<boost::mutex> lock(control->mutex);
222  MERROR("Failed invoking GET command to " << control->uri << ", no status info returned");
223  client.disconnect();
224  control->result_cb(control->path, control->uri, control->success);
225  return;
226  }
227  MDEBUG("response code: " << info->m_response_code);
228  MDEBUG("response length: " << info->m_header_info.m_content_length);
229  MDEBUG("response comment: " << info->m_response_comment);
230  MDEBUG("response body: " << info->m_body);
231  for (const auto &f: info->m_additional_fields)
232  MDEBUG("additional field: " << f.first << ": " << f.second);
233  if (info->m_response_code != 200 && info->m_response_code != 206)
234  {
235  boost::lock_guard<boost::mutex> lock(control->mutex);
236  MERROR("Status code " << info->m_response_code);
237  client.disconnect();
238  control->result_cb(control->path, control->uri, control->success);
239  return;
240  }
241  client.disconnect();
242  f.close();
243  MDEBUG("Download complete");
244  lock.lock();
245  control->success = true;
246  control->result_cb(control->path, control->uri, control->success);
247  return;
248  }
249  catch (const std::exception &e)
250  {
251  MERROR("Exception in download thread: " << e.what());
252  // fall through and call result_cb not from the catch block to avoid another exception
253  }
254  boost::lock_guard<boost::mutex> lock(control->mutex);
255  control->result_cb(control->path, control->uri, control->success);
256  }
257 
258  bool download(const std::string &path, const std::string &url, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> cb)
259  {
260  bool success = false;
261  download_async_handle handle = download_async(path, url, [&success](const std::string&, const std::string&, bool result) {success = result;}, cb);
262  download_wait(handle);
263  return success;
264  }
265 
266  download_async_handle download_async(const std::string &path, const std::string &url, std::function<void(const std::string&, const std::string&, bool)> result, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress)
267  {
268  download_async_handle control = std::make_shared<download_thread_control>(path, url, result, progress);
269  control->thread = boost::thread([control](){ download_thread(control); });
270  return control;
271  }
272 
274  {
275  CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
276  boost::lock_guard<boost::mutex> lock(control->mutex);
277  return control->stopped;
278  }
279 
281  {
282  CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
283  boost::lock_guard<boost::mutex> lock(control->mutex);
284  return !control->success;
285  }
286 
288  {
289  CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
290  {
291  boost::lock_guard<boost::mutex> lock(control->mutex);
292  if (control->stopped)
293  return true;
294  }
295  control->thread.join();
296  return true;
297  }
298 
300  {
301  CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
302  {
303  boost::lock_guard<boost::mutex> lock(control->mutex);
304  if (control->stopped)
305  return true;
306  control->stop = true;
307  }
308  control->thread.join();
309  return true;
310  }
311 }
#define MERROR(x)
Definition: misc_log_ex.h:73
const std::string path
Definition: download.cpp:45
std::list< std::pair< std::string, std::string > > fields_list
Definition: http_base.h:66
#define MINFO(x)
Definition: misc_log_ex.h:75
::std::string string
Definition: gtest-port.h:1097
#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message)
Definition: misc_log_ex.h:181
bool download(const std::string &path, const std::string &url, std::function< bool(const std::string &, const std::string &, size_t, ssize_t)> cb)
Definition: download.cpp:258
download_thread_control(const std::string &path, const std::string &uri, std::function< void(const std::string &, const std::string &, bool)> result_cb, std::function< bool(const std::string &, const std::string &, size_t, ssize_t)> progress_cb)
Definition: download.cpp:55
unsigned short uint16_t
Definition: stdint.h:125
#define MDEBUG(x)
Definition: misc_log_ex.h:76
Various Tools.
Definition: tools.cpp:31
unsigned __int64 uint64_t
Definition: stdint.h:136
std::function< bool(const std::string &, const std::string &, size_t, ssize_t)> progress_cb
Definition: download.cpp:48
#define false
Definition: stdbool.h:38
bool download_finished(const download_async_handle &control)
Definition: download.cpp:273
#define MWARNING(x)
Definition: misc_log_ex.h:74
bool download_wait(const download_async_handle &control)
Definition: download.cpp:287
boost::endian::big_uint16_t port
Definition: socks.cpp:60
CXA_THROW_INFO_T * info
Definition: stack_trace.cpp:91
expect< void > success() noexcept
Definition: expect.h:397
bool download_cancel(const download_async_handle &control)
Definition: download.cpp:299
bool parse_url(const std::string url_str, http::url_content &content)
#define MLOG_SET_THREAD_NAME(x)
Definition: misc_log_ex.h:115
std::string to_string(t_connection_type type)
bool get_file_size(const std::string &path_to_file, uint64_t &size)
std::shared_ptr< download_thread_control > download_async_handle
Definition: download.h:36
PUSH_WARNINGS bool get_xtype_from_string(OUT XType &val, const std::string &str_id)
Definition: string_tools.h:125
download_async_handle download_async(const std::string &path, const std::string &url, std::function< void(const std::string &, const std::string &, bool)> result, std::function< bool(const std::string &, const std::string &, size_t, ssize_t)> progress)
Definition: download.cpp:266
std::function< void(const std::string &, const std::string &, bool)> result_cb
Definition: download.cpp:47
bool download_error(const download_async_handle &control)
Definition: download.cpp:280