Electroneum
http_client.h
Go to the documentation of this file.
1 // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 // * Redistributions of source code must retain the above copyright
7 // notice, this list of conditions and the following disclaimer.
8 // * Redistributions in binary form must reproduce the above copyright
9 // notice, this list of conditions and the following disclaimer in the
10 // documentation and/or other materials provided with the distribution.
11 // * Neither the name of the Andrey N. Sabelnikov nor the
12 // names of its contributors may be used to endorse or promote products
13 // derived from this software without specific prior written permission.
14 //
15 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY
19 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 //
26 
27 
28 
29 #pragma once
30 #include <ctype.h>
31 #include <boost/shared_ptr.hpp>
32 #include <boost/regex.hpp>
33 #include <boost/lexical_cast.hpp>
34 #include <boost/optional/optional.hpp>
35 #include <boost/utility/string_ref.hpp>
36 //#include <mbstring.h>
37 #include <algorithm>
38 #include <cctype>
39 #include <functional>
40 
41 #include "net_helper.h"
42 #include "http_client_base.h"
43 
44 #ifdef HTTP_ENABLE_GZIP
45 #include "gzip_encoding.h"
46 #endif
47 
48 #include "string_tools.h"
49 #include "reg_exp_definer.h"
50 #include "http_base.h"
51 #include "http_auth.h"
52 #include "to_nonconst_iterator.h"
53 #include "net_parse_helpers.h"
54 #include "syncobj.h"
55 
56 //#include "shlwapi.h"
57 
58 //#pragma comment(lib, "shlwapi.lib")
59 
60 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY
61 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "net.http"
62 
64 
65 
66 namespace epee
67 {
68 namespace net_utils
69 {
70 
71  /*struct url
72  {
73  public:
74  void parse(const std::string& url_s)
75  {
76  const string prot_end("://");
77  string::const_iterator prot_i = search(url_s.begin(), url_s.end(),
78  prot_end.begin(), prot_end.end());
79  protocol_.reserve(distance(url_s.begin(), prot_i));
80  transform(url_s.begin(), prot_i,
81  back_inserter(protocol_),
82  ptr_fun<int,int>(tolower)); // protocol is icase
83  if( prot_i == url_s.end() )
84  return;
85  advance(prot_i, prot_end.length());
86  string::const_iterator path_i = find(prot_i, url_s.end(), '/');
87  host_.reserve(distance(prot_i, path_i));
88  transform(prot_i, path_i,
89  back_inserter(host_),
90  ptr_fun<int,int>(tolower)); // host is icase
91  string::const_iterator query_i = find(path_i, url_s.end(), '?');
92  path_.assign(path_i, query_i);
93  if( query_i != url_s.end() )
94  ++query_i;
95  query_.assign(query_i, url_s.end());
96  }
97 
98  std::string protocol_;
99  std::string host_;
100  std::string path_;
101  std::string query_;
102  };*/
103 
104 
105 
106 
107  //---------------------------------------------------------------------------
108  static inline const char* get_hex_vals()
109  {
110  static const char hexVals[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
111  return hexVals;
112  }
113 
114  static inline const char* get_unsave_chars()
115  {
116  //static char unsave_chars[] = "\"<>%\\^[]`+$,@:;/!#?=&";
117  static const char unsave_chars[] = "\"<>%\\^[]`+$,@:;!#&";
118  return unsave_chars;
119  }
120 
121  static inline bool is_unsafe(unsigned char compare_char)
122  {
123  if(compare_char <= 32 || compare_char >= 123)
124  return true;
125 
126  const char* punsave = get_unsave_chars();
127 
128  for(int ichar_pos = 0; 0!=punsave[ichar_pos] ;ichar_pos++)
129  if(compare_char == punsave[ichar_pos])
130  return true;
131 
132  return false;
133  }
134 
135  static inline
136  std::string dec_to_hex(char num, int radix)
137  {
138  int temp=0;
139  std::string csTmp;
140  int num_char;
141 
142  num_char = (int) num;
143  if (num_char < 0)
144  num_char = 256 + num_char;
145 
146  while (num_char >= radix)
147  {
148  temp = num_char % radix;
149  num_char = (int)floor((float)num_char / (float)radix);
150  csTmp = get_hex_vals()[temp];
151  }
152 
153  csTmp += get_hex_vals()[num_char];
154 
155  if(csTmp.size() < 2)
156  {
157  csTmp += '0';
158  }
159 
160  std::reverse(csTmp.begin(), csTmp.end());
161  //_mbsrev((unsigned char*)csTmp.data());
162 
163  return csTmp;
164  }
165  static inline int get_index(const char *s, char c) { const char *ptr = (const char*)memchr(s, c, 16); return ptr ? ptr-s : -1; }
166  static inline
167  std::string hex_to_dec_2bytes(const char *s)
168  {
169  const char *hex = get_hex_vals();
170  int i0 = get_index(hex, toupper(s[0]));
171  int i1 = get_index(hex, toupper(s[1]));
172  if (i0 < 0 || i1 < 0)
173  return std::string("%") + std::string(1, s[0]) + std::string(1, s[1]);
174  return std::string(1, i0 * 16 | i1);
175  }
176 
177  static inline std::string convert(char val)
178  {
179  std::string csRet;
180  csRet += "%";
181  csRet += dec_to_hex(val, 16);
182  return csRet;
183  }
184  static inline std::string conver_to_url_format(const std::string& uri)
185  {
186 
187  std::string result;
188 
189  for(size_t i = 0; i!= uri.size(); i++)
190  {
191  if(is_unsafe(uri[i]))
192  result += convert(uri[i]);
193  else
194  result += uri[i];
195 
196  }
197 
198  return result;
199  }
200  static inline std::string convert_from_url_format(const std::string& uri)
201  {
202 
203  std::string result;
204 
205  for(size_t i = 0; i!= uri.size(); i++)
206  {
207  if(uri[i] == '%' && i + 2 < uri.size())
208  {
209  result += hex_to_dec_2bytes(uri.c_str() + i + 1);
210  i += 2;
211  }
212  else
213  result += uri[i];
214 
215  }
216 
217  return result;
218  }
219 
220  static inline std::string convert_to_url_format_force_all(const std::string& uri)
221  {
222 
223  std::string result;
224 
225  for(size_t i = 0; i!= uri.size(); i++)
226  {
227  result += convert(uri[i]);
228  }
229 
230  return result;
231  }
232 
233 
234 
235 
236 
237  namespace http
238  {
239 
240  template<typename net_client_type>
242  {
243  private:
244  enum reciev_machine_state
245  {
246  reciev_machine_state_header,
247  reciev_machine_state_body_content_len,
248  reciev_machine_state_body_connection_close,
249  reciev_machine_state_body_chunked,
250  reciev_machine_state_done,
251  reciev_machine_state_error
252  };
253 
254 
255 
256  enum chunked_state{
257  http_chunked_state_chunk_head,
258  http_chunked_state_chunk_body,
259  http_chunked_state_done,
260  http_chunked_state_undefined
261  };
262 
263 
264  net_client_type m_net_client;
265  std::string m_host_buff;
266  std::string m_port;
267  http_client_auth m_auth;
268  std::string m_header_cache;
269  http_response_info m_response_info;
270  size_t m_len_in_summary;
271  size_t m_len_in_remain;
272  //std::string* m_ptarget_buffer;
273  boost::shared_ptr<i_sub_handler> m_pcontent_encoding_handler;
274  reciev_machine_state m_state;
275  chunked_state m_chunked_state;
276  std::string m_chunked_cache;
277  bool m_auto_connect;
278  critical_section m_lock;
279 
280  public:
282  : i_target_handler()
283  , m_net_client()
284  , m_host_buff()
285  , m_port()
286  , m_auth()
287  , m_header_cache()
288  , m_response_info()
289  , m_len_in_summary(0)
290  , m_len_in_remain(0)
291  , m_pcontent_encoding_handler(nullptr)
292  , m_state()
293  , m_chunked_state()
294  , m_chunked_cache()
295  , m_auto_connect(true)
296  , m_lock()
297  {}
298 
299  const std::string &get_host() const { return m_host_buff; };
300  const std::string &get_port() const { return m_port; };
301 
302  bool set_server(const std::string& address, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect)
303  {
304  http::url_content parsed{};
305  const bool r = parse_url(address, parsed);
306  CHECK_AND_ASSERT_MES(r, false, "failed to parse url: " << address);
307  set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), std::move(ssl_options));
308  return true;
309  }
310 
311  void set_server(std::string host, std::string port, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect)
312  {
313  CRITICAL_REGION_LOCAL(m_lock);
314  disconnect();
315  m_host_buff = std::move(host);
316  m_port = std::move(port);
317  m_auth = user ? http_client_auth{std::move(*user)} : http_client_auth{};
318  m_net_client.set_ssl(std::move(ssl_options));
319  }
320 
321  void set_auto_connect(bool auto_connect)
322  {
323  m_auto_connect = auto_connect;
324  }
325 
326  template<typename F>
327  void set_connector(F connector)
328  {
329  CRITICAL_REGION_LOCAL(m_lock);
330  m_net_client.set_connector(std::move(connector));
331  }
332 
333  bool connect(std::chrono::milliseconds timeout)
334  {
335  CRITICAL_REGION_LOCAL(m_lock);
336  return m_net_client.connect(m_host_buff, m_port, timeout);
337  }
338  //---------------------------------------------------------------------------
339  bool disconnect()
340  {
341  CRITICAL_REGION_LOCAL(m_lock);
342  return m_net_client.disconnect();
343  }
344  //---------------------------------------------------------------------------
345  bool is_connected(bool *ssl = NULL)
346  {
347  CRITICAL_REGION_LOCAL(m_lock);
348  return m_net_client.is_connected(ssl);
349  }
350  //---------------------------------------------------------------------------
351  virtual bool handle_target_data(std::string& piece_of_transfer)
352  {
353  CRITICAL_REGION_LOCAL(m_lock);
354  m_response_info.m_body += piece_of_transfer;
355  piece_of_transfer.clear();
356  return true;
357  }
358  //---------------------------------------------------------------------------
359  virtual bool on_header(const http_response_info &headers)
360  {
361  return true;
362  }
363  //---------------------------------------------------------------------------
364  inline
365  bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
366  {
367  CRITICAL_REGION_LOCAL(m_lock);
368  return invoke(uri, "GET", body, timeout, ppresponse_info, additional_params);
369  }
370 
371  //---------------------------------------------------------------------------
372  inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
373  {
374  CRITICAL_REGION_LOCAL(m_lock);
375  if(!is_connected())
376  {
377  if (!m_auto_connect)
378  {
379  MWARNING("Auto connect attempt to " << m_host_buff << ":" << m_port << " disabled");
380  return false;
381  }
382  MDEBUG("Reconnecting...");
383  if(!connect(timeout))
384  {
385  MDEBUG("Failed to connect to " << m_host_buff << ":" << m_port);
386  return false;
387  }
388  }
389 
390  std::string req_buff{};
391  req_buff.reserve(2048);
392  req_buff.append(method.data(), method.size()).append(" ").append(uri.data(), uri.size()).append(" HTTP/1.1\r\n");
393  add_field(req_buff, "Host", m_host_buff);
394  add_field(req_buff, "Content-Length", std::to_string(body.size()));
395 
396  //handle "additional_params"
397  for(const auto& field : additional_params)
398  add_field(req_buff, field);
399 
400  for (unsigned sends = 0; sends < 2; ++sends)
401  {
402  const std::size_t initial_size = req_buff.size();
403  const auto auth = m_auth.get_auth_field(method, uri);
404  if (auth)
405  add_field(req_buff, *auth);
406 
407  req_buff += "\r\n";
408  //--
409 
410  bool res = m_net_client.send(req_buff, timeout);
411  CHECK_AND_NO_ASSERT_MES_L(res, false, 1, "HTTP_CLIENT: Failed to SEND");
412  if(body.size())
413  res = m_net_client.send(body, timeout);
414  CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
415 
416  m_response_info.clear();
417  m_state = reciev_machine_state_header;
418  if (!handle_reciev(timeout))
419  return false;
420  if (m_response_info.m_response_code != 401)
421  {
422  if(ppresponse_info)
423  *ppresponse_info = std::addressof(m_response_info);
424  return true;
425  }
426 
427  switch (m_auth.handle_401(m_response_info))
428  {
430  break;
432  sends = 2;
433  break;
434  default:
436  LOG_ERROR("Bad server response for authentication");
437  return false;
438  }
439  req_buff.resize(initial_size); // rollback for new auth generation
440  }
441  LOG_ERROR("Client has incorrect username/password for server requiring authentication");
442  return false;
443  }
444  //---------------------------------------------------------------------------
445  inline bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
446  {
447  CRITICAL_REGION_LOCAL(m_lock);
448  return invoke(uri, "POST", body, timeout, ppresponse_info, additional_params);
449  }
450  //---------------------------------------------------------------------------
451  bool test(const std::string &s, std::chrono::milliseconds timeout) // TEST FUNC ONLY
452  {
453  CRITICAL_REGION_LOCAL(m_lock);
454  m_net_client.set_test_data(s);
455  m_state = reciev_machine_state_header;
456  return handle_reciev(timeout);
457  }
458  //---------------------------------------------------------------------------
460  {
461  return m_net_client.get_bytes_sent();
462  }
463  //---------------------------------------------------------------------------
465  {
466  return m_net_client.get_bytes_received();
467  }
468  //---------------------------------------------------------------------------
469  private:
470  //---------------------------------------------------------------------------
471  inline bool handle_reciev(std::chrono::milliseconds timeout)
472  {
473  CRITICAL_REGION_LOCAL(m_lock);
474  bool keep_handling = true;
475  bool need_more_data = true;
476  std::string recv_buffer;
477  while(keep_handling)
478  {
479  if(need_more_data)
480  {
481  if(!m_net_client.recv(recv_buffer, timeout))
482  {
483  MERROR("Unexpected recv fail");
484  m_state = reciev_machine_state_error;
485  }
486  if(!recv_buffer.size())
487  {
488  //connection is going to be closed
489  if(reciev_machine_state_body_connection_close != m_state)
490  {
491  m_state = reciev_machine_state_error;
492  }
493  }
494  need_more_data = false;
495  }
496  switch(m_state)
497  {
498  case reciev_machine_state_header:
499  keep_handling = handle_header(recv_buffer, need_more_data);
500  break;
501  case reciev_machine_state_body_content_len:
502  keep_handling = handle_body_content_len(recv_buffer, need_more_data);
503  break;
504  case reciev_machine_state_body_connection_close:
505  keep_handling = handle_body_connection_close(recv_buffer, need_more_data);
506  break;
507  case reciev_machine_state_body_chunked:
508  keep_handling = handle_body_body_chunked(recv_buffer, need_more_data);
509  break;
510  case reciev_machine_state_done:
511  keep_handling = false;
512  break;
513  case reciev_machine_state_error:
514  keep_handling = false;
515  break;
516  }
517 
518  }
519  m_header_cache.clear();
520  if(m_state != reciev_machine_state_error)
521  {
522  if(m_response_info.m_header_info.m_connection.size() && !string_tools::compare_no_case("close", m_response_info.m_header_info.m_connection))
523  disconnect();
524 
525  return true;
526  }
527  else
528  {
529  LOG_PRINT_L3("Returning false because of wrong state machine. state: " << m_state);
530  return false;
531  }
532  }
533  //---------------------------------------------------------------------------
534  inline
535  bool handle_header(std::string& recv_buff, bool& need_more_data)
536  {
537 
538  CRITICAL_REGION_LOCAL(m_lock);
539  if(!recv_buff.size())
540  {
541  LOG_ERROR("Connection closed at handle_header");
542  m_state = reciev_machine_state_error;
543  return false;
544  }
545 
546  m_header_cache += recv_buff;
547  recv_buff.clear();
548  std::string::size_type pos = m_header_cache.find("\r\n\r\n");
549  if(pos != std::string::npos)
550  {
551  recv_buff.assign(m_header_cache.begin()+pos+4, m_header_cache.end());
552  m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end());
553 
554  analize_cached_header_and_invoke_state();
555  if (!on_header(m_response_info))
556  {
557  MDEBUG("Connection cancelled by on_header");
558  m_state = reciev_machine_state_done;
559  return false;
560  }
561  m_header_cache.clear();
562  if(!recv_buff.size() && (m_state != reciev_machine_state_error && m_state != reciev_machine_state_done))
563  need_more_data = true;
564 
565  return true;
566  }else
567  need_more_data = true;
568  return true;
569  }
570  //---------------------------------------------------------------------------
571  inline
572  bool handle_body_content_len(std::string& recv_buff, bool& need_more_data)
573  {
574  CRITICAL_REGION_LOCAL(m_lock);
575  if(!recv_buff.size())
576  {
577  MERROR("Warning: Content-Len mode, but connection unexpectedly closed");
578  m_state = reciev_machine_state_done;
579  return true;
580  }
581  CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_len_in_remain >= recv_buff.size()");
582  m_len_in_remain -= recv_buff.size();
583  if (!m_pcontent_encoding_handler->update_in(recv_buff))
584  {
585  m_state = reciev_machine_state_done;
586  return false;
587  }
588 
589  if(m_len_in_remain == 0)
590  m_state = reciev_machine_state_done;
591  else
592  need_more_data = true;
593 
594  return true;
595  }
596  //---------------------------------------------------------------------------
597  inline
598  bool handle_body_connection_close(std::string& recv_buff, bool& need_more_data)
599  {
600  CRITICAL_REGION_LOCAL(m_lock);
601  if(!recv_buff.size())
602  {
603  m_state = reciev_machine_state_done;
604  return true;
605  }
606  need_more_data = true;
607  m_pcontent_encoding_handler->update_in(recv_buff);
608 
609 
610  return true;
611  }
612  //---------------------------------------------------------------------------
613  inline bool is_hex_symbol(char ch)
614  {
615 
616  if( (ch >= '0' && ch <='9')||(ch >= 'A' && ch <='F')||(ch >= 'a' && ch <='f'))
617  return true;
618  else
619  return false;
620  }
621  //---------------------------------------------------------------------------
622  inline
623  bool get_len_from_chunk_head(const std::string &chunk_head, size_t& result_size)
624  {
625  std::stringstream str_stream;
626  str_stream << std::hex;
627  if(!(str_stream << chunk_head && str_stream >> result_size))
628  return false;
629 
630  return true;
631  }
632  //---------------------------------------------------------------------------
633  inline
634  bool get_chunk_head(std::string& buff, size_t& chunk_size, bool& is_matched)
635  {
636  is_matched = false;
637  size_t offset = 0;
638  for(std::string::iterator it = buff.begin(); it!= buff.end(); it++, offset++)
639  {
640  if(!is_hex_symbol(*it))
641  {
642  if(*it == '\r' || *it == ' ' )
643  {
644  offset--;
645  continue;
646  }
647  else if(*it == '\n')
648  {
649  std::string chunk_head = buff.substr(0, offset);
650  if(!get_len_from_chunk_head(chunk_head, chunk_size))
651  return false;
652 
653  if(0 == chunk_size)
654  {
655  //Here is a small confusion
656  //In brief - if the chunk is the last one we need to get terminating sequence
657  //along with the cipher, generally in the "ddd\r\n\r\n" form
658 
659  for(it++;it != buff.end(); it++)
660  {
661  if('\r' == *it)
662  continue;
663  else if('\n' == *it)
664  break;
665  else
666  {
667  LOG_ERROR("http_stream_filter: Wrong last chunk terminator");
668  return false;
669  }
670  }
671 
672  if(it == buff.end())
673  return true;
674  }
675 
676  buff.erase(buff.begin(), ++it);
677 
678  is_matched = true;
679  return true;
680  }
681  else
682  return false;
683  }
684  }
685 
686  return true;
687  }
688  //---------------------------------------------------------------------------
689  inline
690  bool handle_body_body_chunked(std::string& recv_buff, bool& need_more_data)
691  {
692  CRITICAL_REGION_LOCAL(m_lock);
693  if(!recv_buff.size())
694  {
695  MERROR("Warning: CHUNKED mode, but connection unexpectedly closed");
696  m_state = reciev_machine_state_done;
697  return true;
698  }
699  m_chunked_cache += recv_buff;
700  recv_buff.clear();
701  bool is_matched = false;
702 
703  while(true)
704  {
705  if(!m_chunked_cache.size())
706  {
707  need_more_data = true;
708  break;
709  }
710 
711  switch(m_chunked_state)
712  {
713  case http_chunked_state_chunk_head:
714  if(m_chunked_cache[0] == '\n' || m_chunked_cache[0] == '\r')
715  {
716  //optimize a bit
717  if(m_chunked_cache[0] == '\r' && m_chunked_cache.size()>1 && m_chunked_cache[1] == '\n')
718  m_chunked_cache.erase(0, 2);
719  else
720  m_chunked_cache.erase(0, 1);
721  break;
722  }
723  if(!get_chunk_head(m_chunked_cache, m_len_in_remain, is_matched))
724  {
725  LOG_ERROR("http_stream_filter::handle_chunked(*) Failed to get length from chunked head:" << m_chunked_cache);
726  m_state = reciev_machine_state_error;
727  return false;
728  }
729 
730  if(!is_matched)
731  {
732  need_more_data = true;
733  return true;
734  }else
735  {
736  m_chunked_state = http_chunked_state_chunk_body;
737  if(m_len_in_remain == 0)
738  {//last chunk, let stop the stream and fix the chunk queue.
739  m_state = reciev_machine_state_done;
740  return true;
741  }
742  m_chunked_state = http_chunked_state_chunk_body;
743  break;
744  }
745  break;
746  case http_chunked_state_chunk_body:
747  {
748  std::string chunk_body;
749  if(m_len_in_remain >= m_chunked_cache.size())
750  {
751  m_len_in_remain -= m_chunked_cache.size();
752  chunk_body.swap(m_chunked_cache);
753  }else
754  {
755  chunk_body.assign(m_chunked_cache, 0, m_len_in_remain);
756  m_chunked_cache.erase(0, m_len_in_remain);
757  m_len_in_remain = 0;
758  }
759 
760  if (!m_pcontent_encoding_handler->update_in(chunk_body))
761  {
762  m_state = reciev_machine_state_error;
763  return false;
764  }
765 
766  if(!m_len_in_remain)
767  m_chunked_state = http_chunked_state_chunk_head;
768  }
769  break;
770  case http_chunked_state_done:
771  m_state = reciev_machine_state_done;
772  return true;
773  case http_chunked_state_undefined:
774  default:
775  LOG_ERROR("http_stream_filter::handle_chunked(): Wrong state" << m_chunked_state);
776  return false;
777  }
778  }
779 
780  return true;
781  }
782  //---------------------------------------------------------------------------
783  inline bool parse_header(http_header_info& body_info, const std::string& m_cache_to_process)
784  {
785  MTRACE("http_stream_filter::parse_cached_header(*)");
786 
787  const char *ptr = m_cache_to_process.c_str();
788  while (ptr[0] != '\r' || ptr[1] != '\n')
789  {
790  // optional \n
791  if (*ptr == '\n')
792  ++ptr;
793  // an identifier composed of letters or -
794  const char *key_pos = ptr;
795  while (isalnum(*ptr) || *ptr == '_' || *ptr == '-')
796  ++ptr;
797  const char *key_end = ptr;
798  // optional space (not in RFC, but in previous code)
799  if (*ptr == ' ')
800  ++ptr;
801  CHECK_AND_ASSERT_MES(*ptr == ':', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process);
802  ++ptr;
803  // optional whitespace, but not newlines - line folding is obsolete, let's ignore it
804  while (isblank(*ptr))
805  ++ptr;
806  const char *value_pos = ptr;
807  while (*ptr != '\r' && *ptr != '\n')
808  ++ptr;
809  const char *value_end = ptr;
810  // optional trailing whitespace
811  while (value_end > value_pos && isblank(*(value_end-1)))
812  --value_end;
813  if (*ptr == '\r')
814  ++ptr;
815  CHECK_AND_ASSERT_MES(*ptr == '\n', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process);
816  ++ptr;
817 
818  const std::string key = std::string(key_pos, key_end - key_pos);
819  const std::string value = std::string(value_pos, value_end - value_pos);
820  if (!key.empty())
821  {
822  if (!string_tools::compare_no_case(key, "Connection"))
823  body_info.m_connection = value;
824  else if(!string_tools::compare_no_case(key, "Referrer"))
825  body_info.m_referer = value;
826  else if(!string_tools::compare_no_case(key, "Content-Length"))
827  body_info.m_content_length = value;
828  else if(!string_tools::compare_no_case(key, "Content-Type"))
829  body_info.m_content_type = value;
830  else if(!string_tools::compare_no_case(key, "Transfer-Encoding"))
831  body_info.m_transfer_encoding = value;
832  else if(!string_tools::compare_no_case(key, "Content-Encoding"))
833  body_info.m_content_encoding = value;
834  else if(!string_tools::compare_no_case(key, "Host"))
835  body_info.m_host = value;
836  else if(!string_tools::compare_no_case(key, "Cookie"))
837  body_info.m_cookie = value;
838  else if(!string_tools::compare_no_case(key, "User-Agent"))
839  body_info.m_user_agent = value;
840  else if(!string_tools::compare_no_case(key, "Origin"))
841  body_info.m_origin = value;
842  else
843  body_info.m_etc_fields.emplace_back(key, value);
844  }
845  }
846  return true;
847  }
848  //---------------------------------------------------------------------------
849  inline bool analize_first_response_line()
850  {
851  //First line response, look like this: "HTTP/1.1 200 OK"
852  const char *ptr = m_header_cache.c_str();
853  CHECK_AND_ASSERT_MES(!memcmp(ptr, "HTTP/", 5), false, "Invalid first response line: " + m_header_cache);
854  ptr += 5;
855  CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line: " + m_header_cache);
856  unsigned long ul;
857  char *end;
858  ul = strtoul(ptr, &end, 10);
859  CHECK_AND_ASSERT_MES(ul <= INT_MAX && *end =='.', false, "Invalid first response line: " + m_header_cache);
860  m_response_info.m_http_ver_hi = ul;
861  ptr = end + 1;
862  CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr);
863  ul = strtoul(ptr, &end, 10);
864  CHECK_AND_ASSERT_MES(ul <= INT_MAX && isblank(*end), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr);
865  m_response_info.m_http_ver_lo = ul;
866  ptr = end + 1;
867  while (isblank(*ptr))
868  ++ptr;
869  CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line: " + m_header_cache);
870  ul = strtoul(ptr, &end, 10);
871  CHECK_AND_ASSERT_MES(ul >= 100 && ul <= 999 && isspace(*end), false, "Invalid first response line: " + m_header_cache);
872  m_response_info.m_response_code = ul;
873  ptr = end;
874  // ignore the optional text, till the end
875  while (*ptr != '\r' && *ptr != '\n')
876  ++ptr;
877  if (*ptr == '\r')
878  ++ptr;
879  CHECK_AND_ASSERT_MES(*ptr == '\n', false, "Invalid first response line: " << m_header_cache);
880  ++ptr;
881 
882  m_header_cache.erase(0, ptr - m_header_cache.c_str());
883  return true;
884  }
885  inline
886  bool set_reply_content_encoder()
887  {
888  STATIC_REGEXP_EXPR_1(rexp_match_gzip, "^.*?((gzip)|(deflate))", boost::regex::icase | boost::regex::normal);
889  boost::smatch result; // 12 3
890  if(boost::regex_search( m_response_info.m_header_info.m_content_encoding, result, rexp_match_gzip, boost::match_default) && result[0].matched)
891  {
892 #ifdef HTTP_ENABLE_GZIP
893  m_pcontent_encoding_handler.reset(new content_encoding_gzip(this, result[3].matched));
894 #else
895  m_pcontent_encoding_handler.reset(new do_nothing_sub_handler(this));
896  LOG_ERROR("GZIP encoding not supported in this build, please add zlib to your project and define HTTP_ENABLE_GZIP");
897  return false;
898 #endif
899  }
900  else
901  {
902  m_pcontent_encoding_handler.reset(new do_nothing_sub_handler(this));
903  }
904 
905  return true;
906  }
907  inline
908  bool analize_cached_header_and_invoke_state()
909  {
910  m_response_info.clear();
911  analize_first_response_line();
912  std::string fake_str; //gcc error workaround
913 
914  bool res = parse_header(m_response_info.m_header_info, m_header_cache);
915  CHECK_AND_ASSERT_MES(res, false, "http_stream_filter::analize_cached_reply_header_and_invoke_state(): failed to anilize reply header: " << m_header_cache);
916 
917  set_reply_content_encoder();
918 
919  m_len_in_summary = 0;
920  bool content_len_valid = false;
921  if(m_response_info.m_header_info.m_content_length.size())
922  content_len_valid = string_tools::get_xtype_from_string(m_len_in_summary, m_response_info.m_header_info.m_content_length);
923 
924 
925 
926  if(!m_len_in_summary && ((m_response_info.m_response_code>=100&&m_response_info.m_response_code<200)
927  || 204 == m_response_info.m_response_code
928  || 304 == m_response_info.m_response_code) )
929  {//There will be no response body, server will display the local page with error
930  m_state = reciev_machine_state_done;
931  return true;
932  }else if(m_response_info.m_header_info.m_transfer_encoding.size())
933  {
935  if(string_tools::compare_no_case(m_response_info.m_header_info.m_transfer_encoding, "chunked"))
936  {
937  LOG_ERROR("Wrong Transfer-Encoding:" << m_response_info.m_header_info.m_transfer_encoding);
938  m_state = reciev_machine_state_error;
939  return false;
940  }
941  m_state = reciev_machine_state_body_chunked;
942  m_chunked_state = http_chunked_state_chunk_head;
943  return true;
944  }
945  else if(!m_response_info.m_header_info.m_content_length.empty())
946  {
947  //In the response header the length was specified
948  if(!content_len_valid)
949  {
950  LOG_ERROR("http_stream_filter::analize_cached_reply_header_and_invoke_state(): Failed to get_len_from_content_lenght();, m_query_info.m_content_length="<<m_response_info.m_header_info.m_content_length);
951  m_state = reciev_machine_state_error;
952  return false;
953  }
954  if(!m_len_in_summary)
955  {
956  m_state = reciev_machine_state_done;
957  return true;
958  }
959  else
960  {
961  m_len_in_remain = m_len_in_summary;
962  m_state = reciev_machine_state_body_content_len;
963  return true;
964  }
965  }else if(!m_response_info.m_header_info.m_connection.empty() && is_connection_close_field(m_response_info.m_header_info.m_connection))
966  { //By indirect signs we suspect that data transfer will end with a connection break
967  m_state = reciev_machine_state_body_connection_close;
968  }else if(is_multipart_body(m_response_info.m_header_info, fake_str))
969  {
970  m_state = reciev_machine_state_error;
971  LOG_ERROR("Unsupported MULTIPART BODY.");
972  return false;
973  }else
974  { //Apparently there are no signs of the form of transfer, will receive data until the connection is closed
975  m_state = reciev_machine_state_error;
976  MERROR("Undefined transfer type, consider http_body_transfer_connection_close method. header: " << m_header_cache);
977  return false;
978  }
979  return false;
980  }
981  inline
982  bool is_connection_close_field(const std::string& str)
983  {
984  STATIC_REGEXP_EXPR_1(rexp_match_close, "^\\s*close", boost::regex::icase | boost::regex::normal);
985  boost::smatch result;
986  if(boost::regex_search( str, result, rexp_match_close, boost::match_default) && result[0].matched)
987  return true;
988  else
989  return false;
990  }
991  inline
992  bool is_multipart_body(const http_header_info& head_info, OUT std::string& boundary)
993  {
994  //Check whether this is multi part - if yes, capture boundary immediately
995  STATIC_REGEXP_EXPR_1(rexp_match_multipart_type, "^\\s*multipart/([\\w\\-]+); boundary=((\"(.*?)\")|(\\\\\"(.*?)\\\\\")|([^\\s;]*))", boost::regex::icase | boost::regex::normal);
996  boost::smatch result;
997  if(boost::regex_search(head_info.m_content_type, result, rexp_match_multipart_type, boost::match_default) && result[0].matched)
998  {
999  if(result[4].matched)
1000  boundary = result[4];
1001  else if(result[6].matched)
1002  boundary = result[6];
1003  else if(result[7].matched)
1004  boundary = result[7];
1005  else
1006  {
1007  LOG_ERROR("Failed to match boundary in content-type=" << head_info.m_content_type);
1008  return false;
1009  }
1010  return true;
1011  }
1012  else
1013  return false;
1014 
1015  return true;
1016  }
1017  };
1019  }
1020 }
1021 }
const char * res
Definition: hmac_keccak.cpp:41
#define MERROR(x)
Definition: misc_log_ex.h:73
bool set_server(const std::string &address, boost::optional< login > user, ssl_options_t ssl_options=ssl_support_t::e_ssl_support_autodetect)
Definition: http_client.h:302
#define STATIC_REGEXP_EXPR_1(var_name, xpr_text, reg_exp_flags)
std::list< std::pair< std::string, std::string > > fields_list
Definition: http_base.h:66
#define MTRACE(x)
Definition: misc_log_ex.h:77
::std::string string
Definition: gtest-port.h:1097
Implements RFC 2617 digest auth. Digests from RFC 7616 can be added.
Definition: http_auth.h:95
virtual bool on_header(const http_response_info &headers)
Definition: http_client.h:359
#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message)
Definition: misc_log_ex.h:181
const char * key
Definition: hmac_keccak.cpp:39
bool invoke_post(const boost::string_ref uri, const std::string &body, std::chrono::milliseconds timeout, const http_response_info **ppresponse_info=NULL, const fields_list &additional_params=fields_list())
Definition: http_client.h:445
bool connect(std::chrono::milliseconds timeout)
Definition: http_client.h:333
std::string & trim(std::string &str)
Definition: string_tools.h:288
#define CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, l, message)
Definition: misc_log_ex.h:185
#define OUT
Definition: string_tools.h:55
#define MDEBUG(x)
Definition: misc_log_ex.h:76
bool test(const std::string &s, std::chrono::milliseconds timeout)
Definition: http_client.h:451
http_simple_client_template< blocked_mode_client > http_simple_client
Definition: http_client.h:1018
return true
status handle_401(const http_response_info &response)
Definition: http_auth.h:144
unsigned __int64 uint64_t
Definition: stdint.h:136
bool compare_no_case(const std::string &str1, const std::string &str2)
Definition: string_tools.h:221
#define CRITICAL_REGION_LOCAL(x)
Definition: syncobj.h:228
#define LOG_PRINT_L3(x)
Definition: misc_log_ex.h:102
#define MWARNING(x)
Definition: misc_log_ex.h:74
boost::endian::big_uint16_t port
Definition: socks.cpp:60
const T & move(const T &t)
Definition: gtest-port.h:1317
bool parse_url(const std::string url_str, http::url_content &content)
#define LOG_ERROR(x)
Definition: misc_log_ex.h:98
bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string &body, std::chrono::milliseconds timeout, const http_response_info **ppresponse_info=NULL, const fields_list &additional_params=fields_list())
Definition: http_client.h:372
const GenericPointer< typename T::ValueType > T2 value
Definition: pointer.h:1225
epee::critical_section gregexp_lock
#define F(s)
bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string &body=std::string(), const http_response_info **ppresponse_info=NULL, const fields_list &additional_params=fields_list())
Definition: http_client.h:365
const char * address
Definition: multisig.cpp:37
std::string to_string(t_connection_type type)
boost::optional< std::pair< std::string, std::string > > get_auth_field(const boost::string_ref method, const boost::string_ref uri)
Definition: http_auth.h:158
virtual bool handle_target_data(std::string &piece_of_transfer)
Definition: http_client.h:351
#define INT_MAX
Definition: fake-rfc2553.h:119
PUSH_WARNINGS bool get_xtype_from_string(OUT XType &val, const std::string &str_id)
Definition: string_tools.h:125
void set_server(std::string host, std::string port, boost::optional< login > user, ssl_options_t ssl_options=ssl_support_t::e_ssl_support_autodetect)
Definition: http_client.h:311