31 #include <boost/shared_ptr.hpp> 32 #include <boost/regex.hpp> 33 #include <boost/lexical_cast.hpp> 34 #include <boost/optional/optional.hpp> 35 #include <boost/utility/string_ref.hpp> 44 #ifdef HTTP_ENABLE_GZIP 60 #undef ELECTRONEUM_DEFAULT_LOG_CATEGORY 61 #define ELECTRONEUM_DEFAULT_LOG_CATEGORY "net.http" 108 static inline const char* get_hex_vals()
110 static const char hexVals[16] = {
'0',
'1',
'2',
'3',
'4',
'5',
'6',
'7',
'8',
'9',
'A',
'B',
'C',
'D',
'E',
'F'};
114 static inline const char* get_unsave_chars()
117 static const char unsave_chars[] =
"\"<>%\\^[]`+$,@:;!#&";
121 static inline bool is_unsafe(
unsigned char compare_char)
123 if(compare_char <= 32 || compare_char >= 123)
126 const char* punsave = get_unsave_chars();
128 for(
int ichar_pos = 0; 0!=punsave[ichar_pos] ;ichar_pos++)
129 if(compare_char == punsave[ichar_pos])
142 num_char = (int) num;
144 num_char = 256 + num_char;
146 while (num_char >= radix)
148 temp = num_char % radix;
149 num_char = (int)floor((
float)num_char / (float)radix);
150 csTmp = get_hex_vals()[temp];
153 csTmp += get_hex_vals()[num_char];
160 std::reverse(csTmp.begin(), csTmp.end());
165 static inline int get_index(
const char *s,
char c) {
const char *ptr = (
const char*)memchr(s, c, 16);
return ptr ? ptr-s : -1; }
169 const char *hex = get_hex_vals();
170 int i0 = get_index(hex, toupper(s[0]));
171 int i1 = get_index(hex, toupper(s[1]));
172 if (i0 < 0 || i1 < 0)
181 csRet += dec_to_hex(val, 16);
189 for(
size_t i = 0; i!= uri.size(); i++)
191 if(is_unsafe(uri[i]))
192 result += convert(uri[i]);
205 for(
size_t i = 0; i!= uri.size(); i++)
207 if(uri[i] ==
'%' && i + 2 < uri.size())
209 result += hex_to_dec_2bytes(uri.c_str() + i + 1);
225 for(
size_t i = 0; i!= uri.size(); i++)
227 result += convert(uri[i]);
240 template<
typename net_client_type>
244 enum reciev_machine_state
246 reciev_machine_state_header,
247 reciev_machine_state_body_content_len,
248 reciev_machine_state_body_connection_close,
249 reciev_machine_state_body_chunked,
250 reciev_machine_state_done,
251 reciev_machine_state_error
257 http_chunked_state_chunk_head,
258 http_chunked_state_chunk_body,
259 http_chunked_state_done,
260 http_chunked_state_undefined
264 net_client_type m_net_client;
270 size_t m_len_in_summary;
271 size_t m_len_in_remain;
273 boost::shared_ptr<i_sub_handler> m_pcontent_encoding_handler;
274 reciev_machine_state m_state;
275 chunked_state m_chunked_state;
289 , m_len_in_summary(0)
291 , m_pcontent_encoding_handler(nullptr)
295 , m_auto_connect(
true)
318 m_net_client.set_ssl(
std::move(ssl_options));
323 m_auto_connect = auto_connect;
330 m_net_client.set_connector(
std::move(connector));
333 bool connect(std::chrono::milliseconds timeout)
336 return m_net_client.connect(m_host_buff,
m_port, timeout);
342 return m_net_client.disconnect();
348 return m_net_client.is_connected(ssl);
354 m_response_info.
m_body += piece_of_transfer;
355 piece_of_transfer.clear();
368 return invoke(uri,
"GET", body, timeout, ppresponse_info, additional_params);
379 MWARNING(
"Auto connect attempt to " << m_host_buff <<
":" <<
m_port <<
" disabled");
382 MDEBUG(
"Reconnecting...");
385 MDEBUG(
"Failed to connect to " << m_host_buff <<
":" <<
m_port);
391 req_buff.reserve(2048);
392 req_buff.append(method.data(), method.size()).append(
" ").append(uri.data(), uri.size()).append(
" HTTP/1.1\r\n");
393 add_field(req_buff,
"Host", m_host_buff);
394 add_field(req_buff,
"Content-Length",
std::to_string(body.size()));
397 for(
const auto& field : additional_params)
398 add_field(req_buff, field);
400 for (
unsigned sends = 0; sends < 2; ++sends)
402 const std::size_t initial_size = req_buff.size();
405 add_field(req_buff, *auth);
410 bool res = m_net_client.send(req_buff, timeout);
413 res = m_net_client.send(body, timeout);
416 m_response_info.
clear();
417 m_state = reciev_machine_state_header;
418 if (!handle_reciev(timeout))
423 *ppresponse_info = std::addressof(m_response_info);
436 LOG_ERROR(
"Bad server response for authentication");
439 req_buff.resize(initial_size);
441 LOG_ERROR(
"Client has incorrect username/password for server requiring authentication");
448 return invoke(uri,
"POST", body, timeout, ppresponse_info, additional_params);
454 m_net_client.set_test_data(s);
455 m_state = reciev_machine_state_header;
456 return handle_reciev(timeout);
461 return m_net_client.get_bytes_sent();
466 return m_net_client.get_bytes_received();
471 inline bool handle_reciev(std::chrono::milliseconds timeout)
474 bool keep_handling =
true;
475 bool need_more_data =
true;
481 if(!m_net_client.recv(recv_buffer, timeout))
483 MERROR(
"Unexpected recv fail");
484 m_state = reciev_machine_state_error;
486 if(!recv_buffer.size())
489 if(reciev_machine_state_body_connection_close != m_state)
491 m_state = reciev_machine_state_error;
494 need_more_data =
false;
498 case reciev_machine_state_header:
499 keep_handling = handle_header(recv_buffer, need_more_data);
501 case reciev_machine_state_body_content_len:
502 keep_handling = handle_body_content_len(recv_buffer, need_more_data);
504 case reciev_machine_state_body_connection_close:
505 keep_handling = handle_body_connection_close(recv_buffer, need_more_data);
507 case reciev_machine_state_body_chunked:
508 keep_handling = handle_body_body_chunked(recv_buffer, need_more_data);
510 case reciev_machine_state_done:
511 keep_handling =
false;
513 case reciev_machine_state_error:
514 keep_handling =
false;
519 m_header_cache.clear();
520 if(m_state != reciev_machine_state_error)
529 LOG_PRINT_L3(
"Returning false because of wrong state machine. state: " << m_state);
535 bool handle_header(
std::string& recv_buff,
bool& need_more_data)
539 if(!recv_buff.size())
541 LOG_ERROR(
"Connection closed at handle_header");
542 m_state = reciev_machine_state_error;
546 m_header_cache += recv_buff;
548 std::string::size_type pos = m_header_cache.find(
"\r\n\r\n");
549 if(pos != std::string::npos)
551 recv_buff.assign(m_header_cache.begin()+pos+4, m_header_cache.end());
552 m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end());
554 analize_cached_header_and_invoke_state();
557 MDEBUG(
"Connection cancelled by on_header");
558 m_state = reciev_machine_state_done;
561 m_header_cache.clear();
562 if(!recv_buff.size() && (m_state != reciev_machine_state_error && m_state != reciev_machine_state_done))
563 need_more_data =
true;
567 need_more_data =
true;
572 bool handle_body_content_len(
std::string& recv_buff,
bool& need_more_data)
575 if(!recv_buff.size())
577 MERROR(
"Warning: Content-Len mode, but connection unexpectedly closed");
578 m_state = reciev_machine_state_done;
581 CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(),
false,
"m_len_in_remain >= recv_buff.size()");
582 m_len_in_remain -= recv_buff.size();
583 if (!m_pcontent_encoding_handler->update_in(recv_buff))
585 m_state = reciev_machine_state_done;
589 if(m_len_in_remain == 0)
590 m_state = reciev_machine_state_done;
592 need_more_data =
true;
598 bool handle_body_connection_close(
std::string& recv_buff,
bool& need_more_data)
601 if(!recv_buff.size())
603 m_state = reciev_machine_state_done;
606 need_more_data =
true;
607 m_pcontent_encoding_handler->update_in(recv_buff);
613 inline bool is_hex_symbol(
char ch)
616 if( (ch >=
'0' && ch <=
'9')||(ch >=
'A' && ch <=
'F')||(ch >=
'a' && ch <=
'f'))
623 bool get_len_from_chunk_head(
const std::string &chunk_head,
size_t& result_size)
625 std::stringstream str_stream;
626 str_stream << std::hex;
627 if(!(str_stream << chunk_head && str_stream >> result_size))
634 bool get_chunk_head(
std::string& buff,
size_t& chunk_size,
bool& is_matched)
638 for(std::string::iterator it = buff.begin(); it!= buff.end(); it++, offset++)
640 if(!is_hex_symbol(*it))
642 if(*it ==
'\r' || *it ==
' ' )
650 if(!get_len_from_chunk_head(chunk_head, chunk_size))
659 for(it++;it != buff.end(); it++)
667 LOG_ERROR(
"http_stream_filter: Wrong last chunk terminator");
676 buff.erase(buff.begin(), ++it);
690 bool handle_body_body_chunked(
std::string& recv_buff,
bool& need_more_data)
693 if(!recv_buff.size())
695 MERROR(
"Warning: CHUNKED mode, but connection unexpectedly closed");
696 m_state = reciev_machine_state_done;
699 m_chunked_cache += recv_buff;
701 bool is_matched =
false;
705 if(!m_chunked_cache.size())
707 need_more_data =
true;
711 switch(m_chunked_state)
713 case http_chunked_state_chunk_head:
714 if(m_chunked_cache[0] ==
'\n' || m_chunked_cache[0] ==
'\r')
717 if(m_chunked_cache[0] ==
'\r' && m_chunked_cache.size()>1 && m_chunked_cache[1] ==
'\n')
718 m_chunked_cache.erase(0, 2);
720 m_chunked_cache.erase(0, 1);
723 if(!get_chunk_head(m_chunked_cache, m_len_in_remain, is_matched))
725 LOG_ERROR(
"http_stream_filter::handle_chunked(*) Failed to get length from chunked head:" << m_chunked_cache);
726 m_state = reciev_machine_state_error;
732 need_more_data =
true;
736 m_chunked_state = http_chunked_state_chunk_body;
737 if(m_len_in_remain == 0)
739 m_state = reciev_machine_state_done;
742 m_chunked_state = http_chunked_state_chunk_body;
746 case http_chunked_state_chunk_body:
749 if(m_len_in_remain >= m_chunked_cache.size())
751 m_len_in_remain -= m_chunked_cache.size();
752 chunk_body.swap(m_chunked_cache);
755 chunk_body.assign(m_chunked_cache, 0, m_len_in_remain);
756 m_chunked_cache.erase(0, m_len_in_remain);
760 if (!m_pcontent_encoding_handler->update_in(chunk_body))
762 m_state = reciev_machine_state_error;
767 m_chunked_state = http_chunked_state_chunk_head;
770 case http_chunked_state_done:
771 m_state = reciev_machine_state_done;
773 case http_chunked_state_undefined:
775 LOG_ERROR(
"http_stream_filter::handle_chunked(): Wrong state" << m_chunked_state);
783 inline bool parse_header(http_header_info& body_info,
const std::string& m_cache_to_process)
785 MTRACE(
"http_stream_filter::parse_cached_header(*)");
787 const char *ptr = m_cache_to_process.c_str();
788 while (ptr[0] !=
'\r' || ptr[1] !=
'\n')
794 const char *key_pos = ptr;
795 while (isalnum(*ptr) || *ptr ==
'_' || *ptr ==
'-')
797 const char *key_end = ptr;
801 CHECK_AND_ASSERT_MES(*ptr ==
':',
true,
"http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process);
804 while (isblank(*ptr))
806 const char *value_pos = ptr;
807 while (*ptr !=
'\r' && *ptr !=
'\n')
809 const char *value_end = ptr;
811 while (value_end > value_pos && isblank(*(value_end-1)))
815 CHECK_AND_ASSERT_MES(*ptr ==
'\n',
true,
"http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process);
823 body_info.m_connection =
value;
825 body_info.m_referer =
value;
827 body_info.m_content_length =
value;
829 body_info.m_content_type =
value;
831 body_info.m_transfer_encoding =
value;
833 body_info.m_content_encoding =
value;
835 body_info.m_host =
value;
837 body_info.m_cookie =
value;
839 body_info.m_user_agent =
value;
841 body_info.m_origin =
value;
843 body_info.m_etc_fields.emplace_back(
key,
value);
849 inline bool analize_first_response_line()
852 const char *ptr = m_header_cache.c_str();
853 CHECK_AND_ASSERT_MES(!memcmp(ptr,
"HTTP/", 5),
false,
"Invalid first response line: " + m_header_cache);
858 ul = strtoul(ptr, &end, 10);
863 ul = strtoul(ptr, &end, 10);
867 while (isblank(*ptr))
870 ul = strtoul(ptr, &end, 10);
875 while (*ptr !=
'\r' && *ptr !=
'\n')
882 m_header_cache.erase(0, ptr - m_header_cache.c_str());
886 bool set_reply_content_encoder()
888 STATIC_REGEXP_EXPR_1(rexp_match_gzip,
"^.*?((gzip)|(deflate))", boost::regex::icase | boost::regex::normal);
889 boost::smatch result;
892 #ifdef HTTP_ENABLE_GZIP 893 m_pcontent_encoding_handler.reset(
new content_encoding_gzip(
this, result[3].matched));
895 m_pcontent_encoding_handler.reset(
new do_nothing_sub_handler(
this));
896 LOG_ERROR(
"GZIP encoding not supported in this build, please add zlib to your project and define HTTP_ENABLE_GZIP");
902 m_pcontent_encoding_handler.reset(
new do_nothing_sub_handler(
this));
908 bool analize_cached_header_and_invoke_state()
910 m_response_info.
clear();
911 analize_first_response_line();
915 CHECK_AND_ASSERT_MES(
res,
false,
"http_stream_filter::analize_cached_reply_header_and_invoke_state(): failed to anilize reply header: " << m_header_cache);
917 set_reply_content_encoder();
919 m_len_in_summary = 0;
920 bool content_len_valid =
false;
930 m_state = reciev_machine_state_done;
938 m_state = reciev_machine_state_error;
941 m_state = reciev_machine_state_body_chunked;
942 m_chunked_state = http_chunked_state_chunk_head;
948 if(!content_len_valid)
950 LOG_ERROR(
"http_stream_filter::analize_cached_reply_header_and_invoke_state(): Failed to get_len_from_content_lenght();, m_query_info.m_content_length="<<m_response_info.
m_header_info.
m_content_length);
951 m_state = reciev_machine_state_error;
954 if(!m_len_in_summary)
956 m_state = reciev_machine_state_done;
961 m_len_in_remain = m_len_in_summary;
962 m_state = reciev_machine_state_body_content_len;
967 m_state = reciev_machine_state_body_connection_close;
968 }
else if(is_multipart_body(m_response_info.
m_header_info, fake_str))
970 m_state = reciev_machine_state_error;
971 LOG_ERROR(
"Unsupported MULTIPART BODY.");
975 m_state = reciev_machine_state_error;
976 MERROR(
"Undefined transfer type, consider http_body_transfer_connection_close method. header: " << m_header_cache);
982 bool is_connection_close_field(
const std::string& str)
984 STATIC_REGEXP_EXPR_1(rexp_match_close,
"^\\s*close", boost::regex::icase | boost::regex::normal);
985 boost::smatch result;
986 if(boost::regex_search( str, result, rexp_match_close, boost::match_default) && result[0].matched)
992 bool is_multipart_body(
const http_header_info& head_info,
OUT std::string& boundary)
995 STATIC_REGEXP_EXPR_1(rexp_match_multipart_type,
"^\\s*multipart/([\\w\\-]+); boundary=((\"(.*?)\")|(\\\\\"(.*?)\\\\\")|([^\\s;]*))", boost::regex::icase | boost::regex::normal);
996 boost::smatch result;
997 if(boost::regex_search(head_info.m_content_type, result, rexp_match_multipart_type, boost::match_default) && result[0].matched)
999 if(result[4].matched)
1000 boundary = result[4];
1001 else if(result[6].matched)
1002 boundary = result[6];
1003 else if(result[7].matched)
1004 boundary = result[7];
1007 LOG_ERROR(
"Failed to match boundary in content-type=" << head_info.m_content_type);
bool set_server(const std::string &address, boost::optional< login > user, ssl_options_t ssl_options=ssl_support_t::e_ssl_support_autodetect)
#define STATIC_REGEXP_EXPR_1(var_name, xpr_text, reg_exp_flags)
bool is_connected(bool *ssl=NULL)
std::list< std::pair< std::string, std::string > > fields_list
http_simple_client_template()
http_header_info m_header_info
Implements RFC 2617 digest auth. Digests from RFC 7616 can be added.
virtual bool on_header(const http_response_info &headers)
#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message)
const std::string & get_port() const
void set_auto_connect(bool auto_connect)
bool invoke_post(const boost::string_ref uri, const std::string &body, std::chrono::milliseconds timeout, const http_response_info **ppresponse_info=NULL, const fields_list &additional_params=fields_list())
bool connect(std::chrono::milliseconds timeout)
uint64_t get_bytes_sent() const
#define CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, l, message)
bool test(const std::string &s, std::chrono::milliseconds timeout)
http_simple_client_template< blocked_mode_client > http_simple_client
status handle_401(const http_response_info &response)
unsigned __int64 uint64_t
#define CRITICAL_REGION_LOCAL(x)
const std::string & get_host() const
boost::endian::big_uint16_t port
uint64_t get_bytes_received() const
const T & move(const T &t)
bool parse_url(const std::string url_str, http::url_content &content)
bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string &body, std::chrono::milliseconds timeout, const http_response_info **ppresponse_info=NULL, const fields_list &additional_params=fields_list())
const GenericPointer< typename T::ValueType > T2 value
epee::critical_section gregexp_lock
bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string &body=std::string(), const http_response_info **ppresponse_info=NULL, const fields_list &additional_params=fields_list())
std::string to_string(t_connection_type type)
boost::optional< std::pair< std::string, std::string > > get_auth_field(const boost::string_ref method, const boost::string_ref uri)
void set_connector(F connector)
virtual bool handle_target_data(std::string &piece_of_transfer)
void set_server(std::string host, std::string port, boost::optional< login > user, ssl_options_t ssl_options=ssl_support_t::e_ssl_support_autodetect)