libzypp 17.31.8
request.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8----------------------------------------------------------------------*/
13#include <zypp-core/zyppng/base/EventDispatcher>
14#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
15#include <zypp-core/zyppng/core/String>
17#include <zypp-curl/CurlConfig>
18#include <zypp-curl/auth/CurlAuthData>
19#include <zypp-media/MediaConfig>
20#include <zypp-core/base/String.h>
21#include <zypp-core/base/StringV.h>
22#include <zypp-core/Pathname.h>
23#include <curl/curl.h>
24#include <stdio.h>
25#include <fcntl.h>
26#include <sstream>
27#include <utility>
28
29#include <iostream>
30#include <boost/variant.hpp>
31#include <boost/variant/polymorphic_get.hpp>
32
33
34namespace zyppng {
35
36 void dump(const char *text,
37 FILE *stream, unsigned char *ptr, size_t size)
38 {
39 size_t i;
40 size_t c;
41 unsigned int width=0x10;
42
43 DBG << zypp::str::form ( "%s, %10.10ld bytes (0x%8.8lx)\n", text, (long)size, (long)size );
44
45 for(i=0; i<size; i+= width) {
46
47 DBG << zypp::str::form ( "%4.4lx: ", (long)i );
48
49 /* show hex to the left */
50 for(c = 0; c < width; c++) {
51 if(i+c < size)
52 DBG << zypp::str::form("%02x ", ptr[i+c]);
53 else
54 DBG << (" ");
55 }
56
57 /* show data on the right */
58 for(c = 0; (c < width) && (i+c < size); c++) {
59 char x = (ptr[i+c] >= 0x20 && ptr[i+c] < 0x80) ? ptr[i+c] : '.';
60 DBG << x;
61 }
62
63 DBG << std::endl;
64 }
65 }
66
67 int log_curl_all(CURL *handle, curl_infotype type,
68 char *data, size_t size,
69 void *userp)
70 {
71 const char *text;
72
73 long maxlvl = userp == nullptr ? 3 : *((long *)userp);
74
75 (void)handle; /* prevent compiler warning */
76 (void)userp;
77
78 switch (type) {
79 case CURLINFO_TEXT:
80 DBG << handle << " " << "== Info: ";
81 DBG << handle << " " << std::string( data, size ) << std::endl;
82
83 default: /* in case a new one is introduced to shock us */
84 return 0;
85
86 case CURLINFO_HEADER_OUT:
87 text = "=> Send header: ";
88 DBG << handle << " " << std::string( data, size ) << std::endl;
89 return 0;
90 case CURLINFO_DATA_OUT:
91 text = "=> Send data";
92 break;
93 case CURLINFO_SSL_DATA_OUT:
94 text = "=> Send SSL data";
95 break;
96 case CURLINFO_HEADER_IN:
97 text = "<= Recv header: ";
98 DBG << handle << " " << std::string( data, size ) << std::endl;
99 return 0;
100 case CURLINFO_DATA_IN:
101 text = "<= Recv data";
102 break;
103 case CURLINFO_SSL_DATA_IN:
104 text = "<= Recv SSL data";
105 break;
106 }
107
108 if ( maxlvl > 4 )
109 dump(text, stderr, (unsigned char *)data, size);
110 else
111 DBG << handle << " " << " " << size << " bytes " << std::endl;
112 return 0;
113 }
114
115 namespace {
116 static size_t nwr_headerCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
117 if ( !userdata )
118 return 0;
119
120 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
121 return that->headerCallback( ptr, size, nmemb );
122 }
123 static size_t nwr_writeCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
124 if ( !userdata )
125 return 0;
126
127 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
128 return that->writeCallback( ptr, size, nmemb );
129 }
130
131 //helper for std::visit
132 template<class T> struct always_false : std::false_type {};
133 }
134
135 std::vector<char> peek_data_fd( FILE *fd, off_t offset, size_t count )
136 {
137 if ( !fd )
138 return {};
139
140 fflush( fd );
141
142 std::vector<char> data( count + 1 , '\0' );
143
144 ssize_t l = -1;
145 while ((l = pread( fileno( fd ), data.data(), count, offset ) ) == -1 && errno == EINTR)
146 ;
147 if (l == -1)
148 return {};
149
150 return data;
151 }
152
153 NetworkRequest::Range NetworkRequest::Range::make(size_t start, size_t len, zyppng::NetworkRequest::DigestPtr &&digest, zyppng::NetworkRequest::CheckSumBytes &&expectedChkSum, std::any &&userData, std::optional<size_t> digestCompareLen, std::optional<size_t> dataBlockPadding )
154 {
155 return NetworkRequest::Range {
156 .start = start,
157 .len = len,
158 .bytesWritten = 0,
159 ._digest = std::move( digest ),
160 ._checksum = std::move( expectedChkSum ),
161 ._relevantDigestLen = std::move( digestCompareLen ),
162 ._chksumPad = std::move( dataBlockPadding ),
163 .userData = std::move( userData ),
164 ._rangeState = State::Pending
165 };
166 }
167
169 : _outFile( std::move(prevState._outFile) )
170 , _downloaded( prevState._downloaded )
171 , _rangeAttemptIdx( prevState._rangeAttemptIdx )
172 { }
173
175 : _requireStatusPartial( prevState._requireStatusPartial )
176 { }
177
179 : _outFile( std::move(prevState._outFile) )
180 , _requireStatusPartial( true )
181 , _downloaded( prevState._downloaded )
182 , _rangeAttemptIdx( prevState._rangeAttemptIdx )
183 { }
184
186 : BasePrivate(p)
187 , _url ( std::move(url) )
188 , _targetFile ( std::move( targetFile) )
189 , _fMode ( std::move(fMode) )
190 , _headers( std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( nullptr, &curl_slist_free_all ) )
191 { }
192
194 {
195 if ( _easyHandle ) {
196 //clean up for now, later we might reuse handles
197 curl_easy_cleanup( _easyHandle );
198 //reset in request but make sure the request was not enqueued again and got a new handle
199 _easyHandle = nullptr;
200 }
201 }
202
203 bool NetworkRequestPrivate::initialize( std::string &errBuf )
204 {
205 reset();
206
207 if ( _easyHandle )
208 //will reset to defaults but keep live connections, session ID and DNS caches
209 curl_easy_reset( _easyHandle );
210 else
211 _easyHandle = curl_easy_init();
212 return setupHandle ( errBuf );
213 }
214
215 bool NetworkRequestPrivate::setupHandle( std::string &errBuf )
216 {
217 curl_easy_setopt( _easyHandle, CURLOPT_ERRORBUFFER, this->_errorBuf.data() );
218
219 const std::string urlScheme = _url.getScheme();
220 if ( urlScheme == "http" || urlScheme == "https" )
222
223 try {
224
225 setCurlOption( CURLOPT_PRIVATE, this );
227 setCurlOption( CURLOPT_XFERINFODATA, this );
228 setCurlOption( CURLOPT_NOPROGRESS, 0L);
229 setCurlOption( CURLOPT_FAILONERROR, 1L);
230 setCurlOption( CURLOPT_NOSIGNAL, 1L);
231
232 std::string urlBuffer( _url.asString() );
233 setCurlOption( CURLOPT_URL, urlBuffer.c_str() );
234
235 setCurlOption( CURLOPT_WRITEFUNCTION, nwr_writeCallback );
236 setCurlOption( CURLOPT_WRITEDATA, this );
237
239 setCurlOption( CURLOPT_CONNECT_ONLY, 1L );
240 setCurlOption( CURLOPT_FRESH_CONNECT, 1L );
241 }
243 // instead of returning no data with NOBODY, we return
244 // little data, that works with broken servers, and
245 // works for ftp as well, because retrieving only headers
246 // ftp will return always OK code ?
247 // See http://curl.haxx.se/docs/knownbugs.html #58
249 setCurlOption( CURLOPT_NOBODY, 1L );
250 else
251 setCurlOption( CURLOPT_RANGE, "0-1" );
252 }
253
255 if ( _requestedRanges.size() ) {
256 if ( ! prepareNextRangeBatch ( errBuf ))
257 return false;
258 } else {
259 std::visit( [&]( auto &arg ){
260 using T = std::decay_t<decltype(arg)>;
261 if constexpr ( std::is_same_v<T, pending_t> ) {
262 arg._requireStatusPartial = false;
263 } else {
264 DBG << _easyHandle << " " << "NetworkRequestPrivate::setupHandle called in unexpected state" << std::endl;
265 }
266 }, _runningMode );
269 }
270 }
271
272 //make a local copy of the settings, so headers are not added multiple times
274
275 if ( _dispatcher ) {
276 locSet.setUserAgentString( _dispatcher->agentString().c_str() );
277
278 // add custom headers as configured (bsc#955801)
279 const auto &cHeaders = _dispatcher->hostSpecificHeaders();
280 if ( auto i = cHeaders.find(_url.getHost()); i != cHeaders.end() ) {
281 for ( const auto &[key, value] : i->second ) {
283 "%s: %s", key.c_str(), value.c_str() )
284 ));
285 }
286 }
287 }
288
289 locSet.addHeader("Pragma:");
290
293
294 {
295 char *ptr = getenv("ZYPP_MEDIA_CURL_DEBUG");
296 _curlDebug = (ptr && *ptr) ? zypp::str::strtonum<long>( ptr) : 0L;
297 if( _curlDebug > 0)
298 {
299 setCurlOption( CURLOPT_VERBOSE, 1L);
300 if ( _curlDebug >= 3 )
301 setCurlOption( CURLOPT_DEBUGFUNCTION, log_curl_all );
302 else
303 setCurlOption( CURLOPT_DEBUGFUNCTION, ::internal::log_curl);
304 setCurlOption( CURLOPT_DEBUGDATA, &_curlDebug);
305 }
306 }
307
310 {
311 case 4: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ); break;
312 case 6: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 ); break;
313 default: break;
314 }
315
316 setCurlOption( CURLOPT_HEADERFUNCTION, &nwr_headerCallback );
317 setCurlOption( CURLOPT_HEADERDATA, this );
318
322 setCurlOption( CURLOPT_CONNECTTIMEOUT, locSet.connectTimeout() );
323 // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
324 // just in case curl does not trigger its progress callback frequently
325 // enough.
326 if ( locSet.timeout() )
327 {
328 setCurlOption( CURLOPT_TIMEOUT, 3600L );
329 }
330
331 if ( urlScheme == "https" )
332 {
333#if CURLVERSION_AT_LEAST(7,19,4)
334 // restrict following of redirections from https to https only
335 if ( _url.getHost() == "download.opensuse.org" )
336 setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
337 else
338 setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
339#endif
340
341 if( locSet.verifyPeerEnabled() ||
342 locSet.verifyHostEnabled() )
343 {
344 setCurlOption(CURLOPT_CAPATH, locSet.certificateAuthoritiesPath().c_str());
345 }
346
347 if( ! locSet.clientCertificatePath().empty() )
348 {
349 setCurlOption(CURLOPT_SSLCERT, locSet.clientCertificatePath().c_str());
350 }
351 if( ! locSet.clientKeyPath().empty() )
352 {
353 setCurlOption(CURLOPT_SSLKEY, locSet.clientKeyPath().c_str());
354 }
355
356#ifdef CURLSSLOPT_ALLOW_BEAST
357 // see bnc#779177
358 setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
359#endif
360 setCurlOption(CURLOPT_SSL_VERIFYPEER, locSet.verifyPeerEnabled() ? 1L : 0L);
361 setCurlOption(CURLOPT_SSL_VERIFYHOST, locSet.verifyHostEnabled() ? 2L : 0L);
362 // bnc#903405 - POODLE: libzypp should only talk TLS
363 setCurlOption(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
364 }
365
366 // follow any Location: header that the server sends as part of
367 // an HTTP header (#113275)
368 setCurlOption( CURLOPT_FOLLOWLOCATION, 1L);
369 // 3 redirects seem to be too few in some cases (bnc #465532)
370 setCurlOption( CURLOPT_MAXREDIRS, 6L );
371
372 //set the user agent
373 setCurlOption(CURLOPT_USERAGENT, locSet.userAgentString().c_str() );
374
375
376 /*---------------------------------------------------------------*
377 CURLOPT_USERPWD: [user name]:[password]
378 Url::username/password -> CURLOPT_USERPWD
379 If not provided, anonymous FTP identification
380 *---------------------------------------------------------------*/
381 if ( locSet.userPassword().size() )
382 {
383 setCurlOption(CURLOPT_USERPWD, locSet.userPassword().c_str());
384 std::string use_auth = _settings.authType();
385 if (use_auth.empty())
386 use_auth = "digest,basic"; // our default
388 if( auth != CURLAUTH_NONE)
389 {
390 DBG << _easyHandle << " " << "Enabling HTTP authentication methods: " << use_auth
391 << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
392 setCurlOption(CURLOPT_HTTPAUTH, auth);
393 }
394 }
395
396 if ( locSet.proxyEnabled() && ! locSet.proxy().empty() )
397 {
398 DBG << _easyHandle << " " << "Proxy: '" << locSet.proxy() << "'" << std::endl;
399 setCurlOption(CURLOPT_PROXY, locSet.proxy().c_str());
400 setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
401
402 /*---------------------------------------------------------------*
403 * CURLOPT_PROXYUSERPWD: [user name]:[password]
404 *
405 * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
406 * If not provided, $HOME/.curlrc is evaluated
407 *---------------------------------------------------------------*/
408
409 std::string proxyuserpwd = locSet.proxyUserPassword();
410
411 if ( proxyuserpwd.empty() )
412 {
414 zypp::media::CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
415 if ( curlconf.proxyuserpwd.empty() )
416 DBG << _easyHandle << " " << "Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
417 else
418 {
419 proxyuserpwd = curlconf.proxyuserpwd;
420 DBG << _easyHandle << " " << "Proxy: using proxy-user from ~/.curlrc" << std::endl;
421 }
422 }
423 else
424 {
425 DBG << _easyHandle << " " << _easyHandle << " " << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << std::endl;
426 }
427
428 if ( ! proxyuserpwd.empty() )
429 {
430 setCurlOption(CURLOPT_PROXYUSERPWD, ::internal::curlUnEscape( proxyuserpwd ).c_str());
431 }
432 }
433#if CURLVERSION_AT_LEAST(7,19,4)
434 else if ( locSet.proxy() == EXPLICITLY_NO_PROXY )
435 {
436 // Explicitly disabled in URL (see fillSettingsFromUrl()).
437 // This should also prevent libcurl from looking into the environment.
438 DBG << _easyHandle << " " << "Proxy: explicitly NOPROXY" << std::endl;
439 setCurlOption(CURLOPT_NOPROXY, "*");
440 }
441
442#endif
443 else
444 {
445 DBG << _easyHandle << " " << "Proxy: not explicitly set" << std::endl;
446 DBG << _easyHandle << " " << "Proxy: libcurl may look into the environment" << std::endl;
447 }
448
450 if ( locSet.minDownloadSpeed() != 0 )
451 {
452 setCurlOption(CURLOPT_LOW_SPEED_LIMIT, locSet.minDownloadSpeed());
453 // default to 10 seconds at low speed
454 setCurlOption(CURLOPT_LOW_SPEED_TIME, 60L);
455 }
456
457#if CURLVERSION_AT_LEAST(7,15,5)
458 if ( locSet.maxDownloadSpeed() != 0 )
459 setCurlOption(CURLOPT_MAX_RECV_SPEED_LARGE, locSet.maxDownloadSpeed());
460#endif
461
462 if ( zypp::str::strToBool( _url.getQueryParam( "cookies" ), true ) )
463 setCurlOption( CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
464 else
465 MIL << _easyHandle << " " << "No cookies requested" << std::endl;
466 setCurlOption(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
467
468#if CURLVERSION_AT_LEAST(7,18,0)
469 // bnc #306272
470 setCurlOption(CURLOPT_PROXY_TRANSFER_MODE, 1L );
471#endif
472
473 // append settings custom headers to curl
474 for ( const auto &header : locSet.headers() ) {
475 if ( !z_func()->addRequestHeader( header.c_str() ) )
477 }
478
479 if ( _headers )
480 setCurlOption( CURLOPT_HTTPHEADER, _headers.get() );
481
482 return true;
483
484 } catch ( const zypp::Exception &excp ) {
485 ZYPP_CAUGHT(excp);
486 errBuf = excp.asString();
487 }
488 return false;
489 }
490
492 {
493 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
494 if ( !rmode ) {
495 DBG << _easyHandle << "Can only create output file in running mode" << std::endl;
496 return false;
497 }
498 // if we have no open file create or open it
499 if ( !rmode->_outFile ) {
500 std::string openMode = "w+b";
502 openMode = "r+b";
503
504 rmode->_outFile = fopen( _targetFile.asString().c_str() , openMode.c_str() );
505
506 //if the file does not exist create a new one
507 if ( !rmode->_outFile && _fMode == NetworkRequest::WriteShared ) {
508 rmode->_outFile = fopen( _targetFile.asString().c_str() , "w+b" );
509 }
510
511 if ( !rmode->_outFile ) {
513 ,zypp::str::Format("Unable to open target file (%1%). Errno: (%2%:%3%)") % _targetFile.asString() % errno % strerr_cxx() );
514 return false;
515 }
516 }
517
518 return true;
519 }
520
522 {
523 // We can recover from RangeFail errors if we have more batch sizes to try
524 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
525 if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail )
526 return ( rmode->_rangeAttemptIdx + 1 < sizeof( _rangeAttempt ) ) && hasMoreWork();
527 return false;
528 }
529
531 {
532 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
533
534 if ( hasMoreWork() ) {
535 // go to the next range batch level if we are restarted due to a failed range request
536 if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail ) {
537 if ( rmode->_rangeAttemptIdx + 1 >= sizeof( _rangeAttempt ) ) {
538 errBuf = "No more range batch sizes available";
539 return false;
540 }
541 rmode->_rangeAttemptIdx++;
542 }
543
544 _runningMode = prepareNextRangeBatch_t( std::move(std::get<running_t>( _runningMode )) );
545
546 // we reset the handle to default values. We do this to not run into
547 // "transfer closed with outstanding read data remaining" error CURL sometimes returns when
548 // we cancel a connection because of a range error to request a smaller batch.
549 // The error will still happen but much less frequently than without resetting the handle.
550 //
551 // Note: Even creating a new handle will NOT fix the issue
552 curl_easy_reset( _easyHandle );
553 if ( !setupHandle (errBuf) )
554 return false;
555 return true;
556 }
557 errBuf = "Request has no more work";
558 return false;
559
560 }
561
563 {
564 if ( _requestedRanges.size() == 0 ) {
565 errBuf = "Calling the prepareNextRangeBatch function without a range to download is not supported.";
566 return false;
567 }
568
569 std::string rangeDesc;
570 uint rangesAdded = 0;
571 if ( _requestedRanges.size() > 1 && _protocolMode != ProtocolMode::HTTP ) {
572 errBuf = "Using more than one range is not supported with protocols other than HTTP/HTTPS";
573 return false;
574 }
575
576 // check if we have one big range convering the whole file
577 if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 ) {
578 if ( !std::holds_alternative<pending_t>( _runningMode ) ) {
579 errBuf = zypp::str::Str() << "Unexpected state when calling prepareNextRangeBatch " << _runningMode.index ();
580 return false;
581 }
582
584 std::get<pending_t>( _runningMode )._requireStatusPartial = false;
585
586 } else {
587 std::sort( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &elem1, const auto &elem2 ){
588 return ( elem1.start < elem2.start );
589 });
590
591 if ( std::holds_alternative<pending_t>( _runningMode ) )
592 std::get<pending_t>( _runningMode )._requireStatusPartial = true;
593
594 auto maxRanges = _rangeAttempt[0];
595 if ( std::holds_alternative<prepareNextRangeBatch_t>( _runningMode ) )
596 maxRanges = _rangeAttempt[std::get<prepareNextRangeBatch_t>( _runningMode )._rangeAttemptIdx];
597
598 // helper function to build up the request string for the range
599 auto addRangeString = [ &rangeDesc, &rangesAdded ]( const std::pair<size_t, size_t> &range ) {
600 std::string rangeD = zypp::str::form("%llu-", static_cast<unsigned long long>( range.first ) );
601 if( range.second > 0 )
602 rangeD.append( zypp::str::form( "%llu", static_cast<unsigned long long>( range.second ) ) );
603
604 if ( rangeDesc.size() )
605 rangeDesc.append(",").append( rangeD );
606 else
607 rangeDesc = std::move( rangeD );
608
609 rangesAdded++;
610 };
611
612 std::optional<std::pair<size_t, size_t>> currentZippedRange;
613 bool closedRange = true;
614 for ( auto &range : _requestedRanges ) {
615
616 if ( range._rangeState != NetworkRequest::Pending )
617 continue;
618
619 //reset the download results
620 range.bytesWritten = 0;
621
622 //when we have a open range in the list of ranges we will get from start of range to end of file,
623 //all following ranges would never be marked as valid, so we have to fail early
624 if ( !closedRange ) {
625 errBuf = "It is not supported to request more ranges after a open range.";
626 return false;
627 }
628
629 const auto rangeEnd = range.len > 0 ? range.start + range.len - 1 : 0;
630 closedRange = (rangeEnd > 0);
631
632 // remember this range was already requested
633 range._rangeState = NetworkRequest::Running;
634 range.bytesWritten = 0;
635 if ( range._digest )
636 range._digest->reset();
637
638 // we try to compress the requested ranges into as big chunks as possible for the request,
639 // when receiving we still track the original ranges so we can collect and test their checksums
640 if ( !currentZippedRange ) {
641 currentZippedRange = std::make_pair( range.start, rangeEnd );
642 } else {
643 //range is directly consecutive to the previous range
644 if ( currentZippedRange->second + 1 == range.start ) {
645 currentZippedRange->second = rangeEnd;
646 } else {
647 //this range does not directly follow the previous one, we build the string and start a new one
648 addRangeString( *currentZippedRange );
649 currentZippedRange = std::make_pair( range.start, rangeEnd );
650 }
651 }
652
653 if ( rangesAdded >= maxRanges ) {
654 MIL << _easyHandle << " " << "Reached max nr of ranges (" << maxRanges << "), batching the request to not break the server" << std::endl;
655 break;
656 }
657 }
658
659 // add the last range too
660 if ( currentZippedRange )
661 addRangeString( *currentZippedRange );
662
663 MIL << _easyHandle << " " << "Requesting Ranges: " << rangeDesc << std::endl;
664
665 setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() );
666 }
667
668 return true;
669 }
670
672 {
673 // check if we have ranges that have never been requested
674 return std::any_of( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &range ){ return range._rangeState == NetworkRequest::Pending; });
675 }
676
678 {
679 bool isRangeContinuation = std::holds_alternative<prepareNextRangeBatch_t>( _runningMode );
680 if ( isRangeContinuation ) {
681 MIL << _easyHandle << " " << "Continuing a previously started range batch." << std::endl;
682 _runningMode = running_t( std::move(std::get<prepareNextRangeBatch_t>( _runningMode )) );
683 } else {
684 auto mode = running_t( std::move(std::get<pending_t>( _runningMode )) );
685 if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 )
686 mode._currentRange = 0;
687
688 _runningMode = std::move(mode);
689 }
690
691 auto &m = std::get<running_t>( _runningMode );
692
693 if ( m._activityTimer ) {
694 DBG_MEDIA << _easyHandle << " Setting activity timeout to: " << _settings.timeout() << std::endl;
695 m._activityTimer->connect( &Timer::sigExpired, *this, &NetworkRequestPrivate::onActivityTimeout );
696 m._activityTimer->start( static_cast<uint64_t>( _settings.timeout() * 1000 ) );
697 }
698
699 if ( !isRangeContinuation )
700 _sigStarted.emit( *z_func() );
701 }
702
704 {
705 if ( std::holds_alternative<running_t>(_runningMode) ) {
706 auto &rmode = std::get<running_t>( _runningMode );
707 // if we still have a current range set it valid by checking the checksum
708 if ( rmode._currentRange >= 0 ) {
709 auto &currR = _requestedRanges[rmode._currentRange];
710 rmode._currentRange = -1;
711 validateRange( currR );
712 }
713 }
714 }
715
717 {
718
719 finished_t resState;
720 resState._result = std::move(err);
721
722 if ( std::holds_alternative<running_t>(_runningMode) ) {
723
724 auto &rmode = std::get<running_t>( _runningMode );
725 rmode._outFile.reset();
726 resState._downloaded = rmode._downloaded;
727 resState._contentLenght = rmode._contentLenght;
728
730 //we have a successful download lets see if we got everything we needed
731 for ( const auto &r : _requestedRanges ) {
732 if ( r._rangeState != NetworkRequest::Finished ) {
733 if ( r.len > 0 && r.bytesWritten != r.len )
734 resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::MissingData, (zypp::str::Format("Did not receive all requested data from the server ( off: %1%, req: %2%, recv: %3% ).") % r.start % r.len % r.bytesWritten ) );
735 else if ( r._digest && r._checksum.size() && ! checkIfRangeChkSumIsValid(r) ) {
736 resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::InvalidChecksum, (zypp::str::Format("Invalid checksum %1%, expected checksum %2%") % r._digest->digest() % zypp::Digest::digestVectorToString( r._checksum ) ) );
737 } else {
739 }
740 //we only report the first error
741 break;
742 }
743 }
744 }
745 }
746
747 _runningMode = std::move( resState );
748 _sigFinished.emit( *z_func(), std::get<finished_t>(_runningMode)._result );
749 }
750
752 {
754 _headers.reset( nullptr );
755 _errorBuf.fill( 0 );
757 std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
758 range._rangeState = NetworkRequest::Pending;
759 });
760 }
761
763 {
764 auto &m = std::get<running_t>( _runningMode );
765
766 MIL_MEDIA << _easyHandle << " Request timeout interval: " << t.interval()<< " remaining: " << t.remaining() << std::endl;
767 std::map<std::string, boost::any> extraInfo;
768 extraInfo.insert( {"requestUrl", _url } );
769 extraInfo.insert( {"filepath", _targetFile } );
770 _dispatcher->cancel( *z_func(), NetworkRequestErrorPrivate::customError( NetworkRequestError::Timeout, "Download timed out", std::move(extraInfo) ) );
771 }
772
774 {
775 if ( rng._digest && rng._checksum.size() ) {
776 auto bytesHashed = rng._digest->bytesHashed ();
777 if ( rng._chksumPad && *rng._chksumPad > bytesHashed ) {
778 MIL_MEDIA << _easyHandle << " " << "Padding the digest to required block size" << std::endl;
779 zypp::ByteArray padding( *rng._chksumPad - bytesHashed, '\0' );
780 rng._digest->update( padding.data(), padding.size() );
781 }
782 auto digVec = rng._digest->digestVector();
783 if ( rng._relevantDigestLen ) {
784 digVec.resize( *rng._relevantDigestLen );
785 }
786 return ( digVec == rng._checksum );
787 }
788
789 // no checksum required
790 return true;
791 }
792
794 {
795 if ( rng._digest && rng._checksum.size() ) {
796 if ( ( rng.len == 0 || rng.bytesWritten == rng.len ) && checkIfRangeChkSumIsValid(rng) )
798 else
800 } else {
801 if ( rng.len == 0 ? true : rng.bytesWritten == rng.len )
803 else
805 }
806 }
807
808 bool NetworkRequestPrivate::parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len )
809 { //content-range: bytes 10485760-19147879/19147880
810 static const zypp::str::regex regex("^Content-Range:[[:space:]]+bytes[[:space:]]+([0-9]+)-([0-9]+)\\/([0-9]+)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
811
813 if( !zypp::str::regex_match( std::string(line), what, regex ) || what.size() != 4 ) {
814 DBG << _easyHandle << " " << "Invalid Content-Range Header format: '" << std::string(line) << std::endl;
815 return false;
816 }
817
818 size_t s = zypp::str::strtonum<size_t>( what[1]);
819 size_t e = zypp::str::strtonum<size_t>( what[2]);
820 start = std::move(s);
821 len = ( e - s ) + 1;
822 return true;
823 }
824
825 bool NetworkRequestPrivate::parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
826 {
827 static const zypp::str::regex regex("^Content-Type:[[:space:]]+multipart\\/byteranges;[[:space:]]+boundary=(.*)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
828
830 if( zypp::str::regex_match( std::string(line), what, regex ) ) {
831 if ( what.size() >= 2 ) {
832 boundary = what[1];
833 return true;
834 }
835 }
836 return false;
837 }
838
840 {
841 return std::string( _errorBuf.data() );
842 }
843
845 {
846 if ( std::holds_alternative<running_t>( _runningMode ) ){
847 auto &rmode = std::get<running_t>( _runningMode );
848 if ( rmode._activityTimer && rmode._activityTimer->isRunning() )
849 rmode._activityTimer->start();
850 }
851 }
852
853 int NetworkRequestPrivate::curlProgressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
854 {
855 if ( !clientp )
856 return CURLE_OK;
857 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( clientp );
858
859 if ( !std::holds_alternative<running_t>(that->_runningMode) ){
860 DBG << that->_easyHandle << " " << "Curl progress callback was called in invalid state "<< that->z_func()->state() << std::endl;
861 return -1;
862 }
863
864 auto &rmode = std::get<running_t>( that->_runningMode );
865
866 //reset the timer
867 that->resetActivityTimer();
868
869 rmode._isInCallback = true;
870 if ( rmode._lastProgressNow != dlnow ) {
871 rmode._lastProgressNow = dlnow;
872 that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
873 }
874 rmode._isInCallback = false;
875
876 return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
877 }
878
879 size_t NetworkRequestPrivate::headerCallback(char *ptr, size_t size, size_t nmemb)
880 {
881 //it is valid to call this function with no data to write, just return OK
882 if ( size * nmemb == 0)
883 return 0;
884
886
888
889 std::string_view hdr( ptr, size*nmemb );
890
891 hdr.remove_prefix( std::min( hdr.find_first_not_of(" \t\r\n"), hdr.size() ) );
892 const auto lastNonWhitespace = hdr.find_last_not_of(" \t\r\n");
893 if ( lastNonWhitespace != hdr.npos )
894 hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
895 else
896 hdr = std::string_view();
897
898 DBG_MEDIA << _easyHandle << " " << "Received header: " << hdr << std::endl;
899
900 auto &rmode = std::get<running_t>( _runningMode );
901 if ( !hdr.size() ) {
902 return ( size * nmemb );
903 }
904 if ( zypp::strv::hasPrefixCI( hdr, "HTTP/" ) ) {
905
906 long statuscode = 0;
907 (void)curl_easy_getinfo( _easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
908
909 const auto &doRangeFail = [&](){
910 WAR << _easyHandle << " " << "Range FAIL, trying with a smaller batch" << std::endl;
911 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::RangeFail, "Expected range status code 206, but got none." );
912
913 // reset all ranges we requested to pending, we never got the data for them
914 std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
915 if ( range._rangeState == NetworkRequest::Running )
916 range._rangeState = NetworkRequest::Pending;
917 });
918 return 0;
919 };
920
921 // if we have a status 204 we need to create a empty file
922 if( statuscode == 204 && !( _options & NetworkRequest::ConnectionTest ) && !( _options & NetworkRequest::HeadRequest ) )
924
925 if ( rmode._requireStatusPartial ) {
926 // ignore other status codes, maybe we are redirected etc.
927 if ( ( statuscode >= 200 && statuscode <= 299 && statuscode != 206 )
928 || statuscode == 416 ) {
929 return doRangeFail();
930 }
931 }
932
933 } else if ( zypp::strv::hasPrefixCI( hdr, "Location:" ) ) {
934 _lastRedirect = hdr.substr( 9 );
935 DBG << _easyHandle << " " << "redirecting to " << _lastRedirect << std::endl;
936
937 } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Type:") ) {
938 std::string sep;
939 if ( parseContentTypeMultiRangeHeader( hdr, sep ) ) {
940 rmode._gotMultiRangeHeader = true;
941 rmode._seperatorString = "--"+sep;
942 }
943 } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Range:") ) {
945 if ( !parseContentRangeHeader( hdr, r.start, r.len) ) {
946 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
947 return 0;
948 }
949 DBG << _easyHandle << " " << "Got content range :" << r.start << " len " << r.len << std::endl;
950 rmode._gotContentRangeHeader = true;
951 rmode._currentSrvRange = r;
952
953 } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Length:") ) {
954 auto lenStr = str::trim( hdr.substr( 15 ), zypp::str::TRIM );
955 auto str = std::string ( lenStr.data(), lenStr.length() );
956 auto len = zypp::str::strtonum<typename zypp::ByteCount::SizeType>( str.data() );
957 if ( len > 0 ) {
958 DBG << _easyHandle << " " << "Got Content-Length Header: " << len << std::endl;
959 rmode._contentLenght = zypp::ByteCount(len, zypp::ByteCount::B);
960 }
961 }
962 }
963
964 return ( size * nmemb );
965 }
966
967 size_t NetworkRequestPrivate::writeCallback(char *ptr, size_t size, size_t nmemb)
968 {
969 const auto max = ( size * nmemb );
970
972
973 //it is valid to call this function with no data to write, just return OK
974 if ( max == 0)
975 return 0;
976
977 //in case of a HEAD request, we do not write anything
979 return ( size * nmemb );
980 }
981
982 auto &rmode = std::get<running_t>( _runningMode );
983
984 auto writeDataToFile = [ this, &rmode ]( off_t offset, const char *data, size_t len ) -> off_t {
985
986 if ( rmode._currentRange < 0 ) {
987 DBG << _easyHandle << " " << "Current range is zero in write request" << std::endl;
988 return 0;
989 }
990
991 // if we have no open file create or open it
992 if ( !assertOutputFile() )
993 return 0;
994
995 // seek to the given offset
996 if ( offset >= 0 ) {
997 if ( fseek( rmode._outFile, offset, SEEK_SET ) != 0 ) {
999 "Unable to set output file pointer." );
1000 return 0;
1001 }
1002 }
1003
1004 auto &rng = _requestedRanges[ rmode._currentRange ];
1005 const auto bytesToWrite = rng.len > 0 ? std::min( rng.len - rng.bytesWritten, len ) : len;
1006
1007 //make sure we do not write after the expected file size
1008 if ( _expectedFileSize && _expectedFileSize <= static_cast<zypp::ByteCount::SizeType>(rng.start + rng.bytesWritten + bytesToWrite) ) {
1009 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Downloaded data exceeds expected length." );
1010 return 0;
1011 }
1012
1013 auto written = fwrite( data, 1, bytesToWrite, rmode._outFile );
1014 if ( written == 0 )
1015 return 0;
1016
1017 if ( rng._digest && rng._checksum.size() ) {
1018 if ( !rng._digest->update( data, written ) )
1019 return 0;
1020 }
1021
1022 rng.bytesWritten += written;
1023 if ( rmode._currentSrvRange ) rmode._currentSrvRange->bytesWritten += written;
1024
1025 if ( rng.len > 0 && rng.bytesWritten >= rng.len ) {
1026 rmode._currentRange = -1;
1027 validateRange( rng );
1028 }
1029
1030 if ( rmode._currentSrvRange && rmode._currentSrvRange->len > 0 && rmode._currentSrvRange->bytesWritten >= rmode._currentSrvRange->len ) {
1031 rmode._currentSrvRange.reset();
1032 // we ran out of data in the current chunk, reset the target range as well because next data will be
1033 // a chunk header again
1034 rmode._currentRange = -1;
1035 }
1036
1037 // count the number of real bytes we have downloaded so far
1038 rmode._downloaded += written;
1039 _sigBytesDownloaded.emit( *z_func(), rmode._downloaded );
1040
1041 return written;
1042 };
1043
1044 // we are currenty writing a range, continue until we hit the end of the requested chunk, or if we hit end of data
1045 size_t bytesWrittenSoFar = 0;
1046
1047 while ( bytesWrittenSoFar != max ) {
1048
1049 off_t seekTo = -1;
1050
1051 // this is called after all headers have been processed
1052 if ( !rmode._allHeadersReceived ) {
1053 rmode._allHeadersReceived = true;
1054
1055 // no ranges at all, must be a normal download
1056 if ( !rmode._gotMultiRangeHeader && !rmode._gotContentRangeHeader ) {
1057
1058 if ( rmode._requireStatusPartial ) {
1059 //we got a invalid response, the status code pointed to being partial but we got no range definition
1061 "Invalid data from server, range respone was announced but there was no range definiton." );
1062 return 0;
1063 }
1064
1065 //we always download a range even if it is not explicitly requested
1066 if ( _requestedRanges.empty() ) {
1069 }
1070
1071 rmode._currentRange = 0;
1072 seekTo = _requestedRanges[0].start;
1073 }
1074 }
1075
1076 if ( rmode._currentSrvRange && rmode._currentRange == -1 ) {
1077 //if we enter this branch, we just have finished writing a requested chunk but
1078 //are still inside a chunk that was sent by the server, due to the std the server can coalesce requested ranges
1079 //to optimize downloads we need to find the best match ( because the current offset might not even be in our requested ranges )
1080 //Or we just parsed a Content-Lenght header and start a new block
1081
1082 std::optional<uint> foundRange;
1083 const size_t beginRange = rmode._currentSrvRange->start + rmode._currentSrvRange->bytesWritten;
1084 const size_t endRange = beginRange + (rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten);
1085 auto currDist = ULONG_MAX;
1086 for ( uint i = 0; i < _requestedRanges.size(); i++ ) {
1087 const auto &currR = _requestedRanges[i];
1088
1089 // do not allow double ranges
1090 if ( currR._rangeState == NetworkRequest::Finished || currR._rangeState == NetworkRequest::Error )
1091 continue;
1092
1093 // check if the range was already written
1094 if ( currR.len == currR.bytesWritten )
1095 continue;
1096
1097 const auto currRBegin = currR.start + currR.bytesWritten;
1098 if ( !( beginRange <= currRBegin && endRange >= currRBegin ) )
1099 continue;
1100
1101 // calculate the distance of the current ranges offset+data written to the range we got back from the server
1102 const auto newDist = currRBegin - beginRange;
1103
1104 if ( !foundRange ) {
1105 foundRange = i;
1106 currDist = newDist;
1107 } else {
1108 //pick the range with the closest distance
1109 if ( newDist < currDist ) {
1110 foundRange = i;
1111 currDist = newDist;
1112 }
1113 }
1114 }
1115 if ( !foundRange ) {
1117 , "Unable to find a matching range for data returned by the server." );
1118 return 0;
1119 }
1120
1121 //set the found range as the current one
1122 rmode._currentRange = *foundRange;
1123
1124 //continue writing where we stopped
1125 seekTo = _requestedRanges[*foundRange].start + _requestedRanges[*foundRange].bytesWritten;
1126
1127 //if we skip bytes we need to advance our written bytecount
1128 const auto skipBytes = seekTo - beginRange;
1129 bytesWrittenSoFar += skipBytes;
1130 rmode._currentSrvRange->bytesWritten += skipBytes;
1131 }
1132
1133 if ( rmode._currentRange >= 0 ) {
1134 auto availableData = max - bytesWrittenSoFar;
1135 if ( rmode._currentSrvRange ) {
1136 availableData = std::min( availableData, rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten );
1137 }
1138 auto bw = writeDataToFile( seekTo, ptr + bytesWrittenSoFar, availableData );
1139 if ( bw <= 0 )
1140 return 0;
1141
1142 bytesWrittenSoFar += bw;
1143 }
1144
1145 if ( bytesWrittenSoFar == max )
1146 return max;
1147
1148 if ( rmode._currentRange == -1 ) {
1149
1150 // we still are inside the current range from the server
1151 if ( rmode._currentSrvRange )
1152 continue;
1153
1154 std::string_view incoming( ptr + bytesWrittenSoFar, max - bytesWrittenSoFar );
1155 auto hdrEnd = incoming.find("\r\n\r\n");
1156 if ( hdrEnd == incoming.npos ) {
1157 //no header end in the data yet, push to buffer and return
1158 rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.end() );
1159 return max;
1160 }
1161
1162 //append the data of the current header to the buffer and parse it
1163 rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.begin() + ( hdrEnd + 4 ) );
1164 bytesWrittenSoFar += ( hdrEnd + 4 ); //header data plus header end
1165
1166 std::string_view data( rmode._rangePrefaceBuffer.data(), rmode._rangePrefaceBuffer.size() );
1167 auto sepStrIndex = data.find( rmode._seperatorString );
1168 if ( sepStrIndex == data.npos ) {
1170 "Invalid multirange header format, seperator string missing." );
1171 return 0;
1172 }
1173
1174 auto startOfHeader = sepStrIndex + rmode._seperatorString.length();
1175 std::vector<std::string_view> lines;
1176 zypp::strv::split( data.substr( startOfHeader ), "\r\n", zypp::strv::Trim::trim, [&]( std::string_view strv ) { lines.push_back(strv); } );
1177 for ( const auto &hdrLine : lines ) {
1178 if ( zypp::strv::hasPrefixCI(hdrLine, "Content-Range:") ) {
1180 //if we can not parse the header the message must be broken
1181 if(! parseContentRangeHeader( hdrLine, r.start, r.len ) ) {
1182 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
1183 return 0;
1184 }
1185 rmode._currentSrvRange = r;
1186 break;
1187 }
1188 }
1189 //clear the buffer again
1190 rmode._rangePrefaceBuffer.clear();
1191 }
1192 }
1193 return bytesWrittenSoFar;
1194 }
1195
1197
1198 NetworkRequest::NetworkRequest(zyppng::Url url, zypp::filesystem::Pathname targetFile, zyppng::NetworkRequest::FileMode fMode)
1199 : Base ( *new NetworkRequestPrivate( std::move(url), std::move(targetFile), std::move(fMode), *this ) )
1200 {
1201 }
1202
1204 {
1205 Z_D();
1206
1207 if ( d->_dispatcher )
1208 d->_dispatcher->cancel( *this, "Request destroyed while still running" );
1209 }
1210
1212 {
1213 d_func()->_expectedFileSize = std::move( expectedFileSize );
1214 }
1215
1216 void NetworkRequest::setPriority( NetworkRequest::Priority prio, bool triggerReschedule )
1217 {
1218 Z_D();
1219 d->_priority = prio;
1220 if ( state() == Pending && triggerReschedule && d->_dispatcher )
1221 d->_dispatcher->reschedule();
1222 }
1223
1225 {
1226 return d_func()->_priority;
1227 }
1228
1229 void NetworkRequest::setOptions( Options opt )
1230 {
1231 d_func()->_options = opt;
1232 }
1233
1234 NetworkRequest::Options NetworkRequest::options() const
1235 {
1236 return d_func()->_options;
1237 }
1238
1239 void NetworkRequest::addRequestRange( size_t start, size_t len, DigestPtr digest, CheckSumBytes expectedChkSum , std::any userData, std::optional<size_t> digestCompareLen, std::optional<size_t> chksumpad )
1240 {
1241 Z_D();
1242 if ( state() == Running )
1243 return;
1244
1245 d->_requestedRanges.push_back( Range::make( start, len, std::move(digest), std::move( expectedChkSum ), std::move( userData ), digestCompareLen, chksumpad ) );
1246 }
1247
1249 {
1250 Z_D();
1251 if ( state() == Running )
1252 return;
1253
1254 d->_requestedRanges.push_back( range );
1255 auto &rng = d->_requestedRanges.back();
1256 rng._rangeState = NetworkRequest::Pending;
1257 rng.bytesWritten = 0;
1258 if ( rng._digest )
1259 rng._digest->reset();
1260 }
1261
1263 {
1264 Z_D();
1265 if ( state() == Running )
1266 return;
1267 d->_requestedRanges.clear();
1268 }
1269
1270 std::vector<NetworkRequest::Range> NetworkRequest::failedRanges() const
1271 {
1272 const auto mystate = state();
1273 if ( mystate != Finished && mystate != Error )
1274 return {};
1275
1276 Z_D();
1277
1278 std::vector<Range> failed;
1279 for ( const auto &r : d->_requestedRanges ) {
1280 if ( r._rangeState != NetworkRequest::Finished )
1281 failed.push_back( r );
1282 }
1283 return failed;
1284 }
1285
1286 const std::vector<NetworkRequest::Range> &NetworkRequest::requestedRanges() const
1287 {
1288 return d_func()->_requestedRanges;
1289 }
1290
1291 const std::string &NetworkRequest::lastRedirectInfo() const
1292 {
1293 return d_func()->_lastRedirect;
1294 }
1295
1297 {
1298 return d_func()->_easyHandle;
1299 }
1300
1301 std::optional<zyppng::NetworkRequest::Timings> NetworkRequest::timings() const
1302 {
1303 const auto myerr = error();
1304 const auto mystate = state();
1305 if ( mystate != Finished )
1306 return {};
1307
1308 Timings t;
1309
1310 auto getMeasurement = [ this ]( const CURLINFO info, std::chrono::microseconds &target ){
1311 using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
1312 double val = 0;
1313 const auto res = curl_easy_getinfo( d_func()->_easyHandle, info, &val );
1314 if ( CURLE_OK == res ) {
1315 target = std::chrono::duration_cast<std::chrono::microseconds>( FPSeconds(val) );
1316 }
1317 };
1318
1319 getMeasurement( CURLINFO_NAMELOOKUP_TIME, t.namelookup );
1320 getMeasurement( CURLINFO_CONNECT_TIME, t.connect);
1321 getMeasurement( CURLINFO_APPCONNECT_TIME, t.appconnect);
1322 getMeasurement( CURLINFO_PRETRANSFER_TIME , t.pretransfer);
1323 getMeasurement( CURLINFO_TOTAL_TIME, t.total);
1324 getMeasurement( CURLINFO_REDIRECT_TIME, t.redirect);
1325
1326 return t;
1327 }
1328
1329 std::vector<char> NetworkRequest::peekData( off_t offset, size_t count ) const
1330 {
1331 Z_D();
1332
1333 if ( !std::holds_alternative<NetworkRequestPrivate::running_t>( d->_runningMode) )
1334 return {};
1335
1336 const auto &rmode = std::get<NetworkRequestPrivate::running_t>( d->_runningMode );
1337 return peek_data_fd( rmode._outFile, offset, count );
1338 }
1339
1341 {
1342 return d_func()->_url;
1343 }
1344
1345 void NetworkRequest::setUrl(const Url &url)
1346 {
1347 Z_D();
1348 if ( state() == NetworkRequest::Running )
1349 return;
1350
1351 d->_url = url;
1352 }
1353
1355 {
1356 return d_func()->_targetFile;
1357 }
1358
1360 {
1361 Z_D();
1362 if ( state() == NetworkRequest::Running )
1363 return;
1364 d->_targetFile = path;
1365 }
1366
1368 {
1369 return d_func()->_fMode;
1370 }
1371
1373 {
1374 Z_D();
1375 if ( state() == NetworkRequest::Running )
1376 return;
1377 d->_fMode = std::move( mode );
1378 }
1379
1381 {
1382 char *ptr = NULL;
1383 if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
1384 return std::string(ptr);
1385 return std::string();
1386 }
1387
1389 {
1390 return std::visit([](auto& arg) -> zypp::ByteCount {
1391 using T = std::decay_t<decltype(arg)>;
1392 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1393 return zypp::ByteCount(0);
1394 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1395 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1396 return arg._contentLenght;
1397 else
1398 static_assert(always_false<T>::value, "Unhandled state type");
1399 }, d_func()->_runningMode);
1400 }
1401
1403 {
1404 return std::visit([](auto& arg) -> zypp::ByteCount {
1405 using T = std::decay_t<decltype(arg)>;
1406 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1407 return zypp::ByteCount();
1408 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1409 || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
1410 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1411 return arg._downloaded;
1412 else
1413 static_assert(always_false<T>::value, "Unhandled state type");
1414 }, d_func()->_runningMode);
1415 }
1416
1418 {
1419 return d_func()->_settings;
1420 }
1421
1423 {
1424 return std::visit([this](auto& arg) {
1425 using T = std::decay_t<decltype(arg)>;
1426 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1427 return Pending;
1428 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1429 return Running;
1430 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
1431 if ( std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode )._result.isError() )
1432 return Error;
1433 else
1434 return Finished;
1435 }
1436 else
1437 static_assert(always_false<T>::value, "Unhandled state type");
1438 }, d_func()->_runningMode);
1439 }
1440
1442 {
1443 const auto s = state();
1444 if ( s != Error && s != Finished )
1445 return NetworkRequestError();
1446 return std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode)._result;
1447 }
1448
1450 {
1451 if ( !hasError() )
1452 return std::string();
1453
1454 return error().nativeErrorString();
1455 }
1456
1458 {
1459 return error().isError();
1460 }
1461
1462 bool NetworkRequest::addRequestHeader( const std::string &header )
1463 {
1464 Z_D();
1465
1466 curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() : nullptr, header.c_str() );
1467 if ( !res )
1468 return false;
1469
1470 if ( !d->_headers )
1471 d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
1472
1473 return true;
1474 }
1475
1477 {
1478 return d_func()->_sigStarted;
1479 }
1480
1482 {
1483 return d_func()->_sigBytesDownloaded;
1484 }
1485
1486 SignalProxy<void (NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> NetworkRequest::sigProgress()
1487 {
1488 return d_func()->_sigProgress;
1489 }
1490
1492 {
1493 return d_func()->_sigFinished;
1494 }
1495
1496}
ZYppCommitResult & _result
Definition: TargetImpl.cc:1564
Store and operate with byte count.
Definition: ByteCount.h:31
static const Unit B
1 Byte
Definition: ByteCount.h:42
Unit::ValueType SizeType
Definition: ByteCount.h:37
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Definition: Digest.cc:184
Base class for Exception.
Definition: Exception.h:146
std::string asString() const
Error message provided by dumpOn as string.
Definition: Exception.cc:75
static MediaConfig & instance()
Definition: mediaconfig.cc:43
long download_transfer_timeout() const
Definition: mediaconfig.cc:112
const char * c_str() const
String representation.
Definition: Pathname.h:110
const std::string & asString() const
String representation.
Definition: Pathname.h:91
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
Holds transfer setting.
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
long connectTimeout() const
connection timeout
const std::string & authType() const
get the allowed authentication types
long timeout() const
transfer timeout
const Pathname & clientCertificatePath() const
SSL client certificate file.
std::string userPassword() const
returns the user and password as a user:pass string
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
const Headers & headers() const
returns a list of all added headers
const std::string & proxy() const
proxy host
const Pathname & clientKeyPath() const
SSL client key file.
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3"
void setConnectTimeout(long t)
set the connect timeout
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar"
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & userAgentString() const
user agent string
bool headRequestsAllowed() const
whether HEAD requests are allowed
bool proxyEnabled() const
proxy is enabled
const std::string & proxyUsername() const
proxy auth username
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
void setTimeout(long t)
set the transfer timeout
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
Regular expression.
Definition: Regex.h:95
@ icase
Do not differentiate case.
Definition: Regex.h:99
@ rxdefault
These are enforced even if you don't pass them as flag argument.
Definition: Regex.h:103
Regular expression match result.
Definition: Regex.h:168
unsigned size() const
Definition: Regex.cc:106
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
The NetworkRequestError class Represents a error that occured in.
Type type() const
type Returns the type of the error
std::string nativeErrorString() const
bool isError() const
isError Will return true if this is a actual error
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
const std::string _currentCookieFile
Definition: request_p.h:127
zypp::Pathname _targetFile
Definition: request_p.h:116
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
Definition: request.cc:825
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
Definition: request_p.h:134
NetworkRequestDispatcher * _dispatcher
Definition: request_p.h:130
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
Definition: request_p.h:120
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Definition: request.cc:853
std::string errorMessage() const
Definition: request.cc:839
Signal< void(NetworkRequest &req)> _sigStarted
Definition: request_p.h:133
NetworkRequest::FileMode _fMode
Definition: request_p.h:122
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode
Definition: request_p.h:212
bool initialize(std::string &errBuf)
Definition: request.cc:203
void validateRange(NetworkRequest::Range &rng)
Definition: request.cc:793
void onActivityTimeout(Timer &)
Definition: request.cc:762
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
Definition: request_p.h:135
size_t writeCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:967
std::string _lastRedirect
to log/report redirections
Definition: request_p.h:126
NetworkRequest::Options _options
Definition: request_p.h:118
static constexpr int _rangeAttempt[]
Definition: request_p.h:149
bool prepareToContinue(std::string &errBuf)
Definition: request.cc:530
bool prepareNextRangeBatch(std::string &errBuf)
Definition: request.cc:562
size_t headerCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:879
void setResult(NetworkRequestError &&err)
Definition: request.cc:716
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
Definition: request_p.h:104
bool setupHandle(std::string &errBuf)
Definition: request.cc:215
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
Definition: request.cc:185
TransferSettings _settings
Definition: request_p.h:117
void setCurlOption(CURLoption opt, T data)
Definition: request_p.h:107
zypp::ByteCount _expectedFileSize
Definition: request_p.h:119
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
Definition: request_p.h:136
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
Definition: request_p.h:142
bool checkIfRangeChkSumIsValid(const NetworkRequest::Range &rng)
Definition: request.cc:773
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len)
Definition: request.cc:808
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size,...
Definition: request.cc:1388
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
Definition: request.cc:1354
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
Definition: request.cc:1402
void setUrl(const Url &url)
This will change the URL of the request.
Definition: request.cc:1345
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
Definition: request.cc:1211
virtual ~NetworkRequest()
Definition: request.cc:1203
void setPriority(Priority prio, bool triggerReschedule=true)
Definition: request.cc:1216
std::vector< char > peekData(off_t offset, size_t count) const
Definition: request.cc:1329
std::string contentType() const
Returns the content type as reported from the server.
Definition: request.cc:1380
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
Definition: request.cc:1372
std::shared_ptr< zypp::Digest > DigestPtr
Definition: request.h:46
bool addRequestHeader(const std::string &header)
Definition: request.cc:1462
void setOptions(Options opt)
Definition: request.cc:1229
FileMode fileOpenMode() const
Returns the currently configured file open mode.
Definition: request.cc:1367
bool hasError() const
Checks if there was a error with the request.
Definition: request.cc:1457
State state() const
Returns the current state the HttpDownloadRequest is in.
Definition: request.cc:1422
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition: request.cc:1491
UByteArray CheckSumBytes
Definition: request.h:47
Options options() const
Definition: request.cc:1234
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
Definition: request.cc:1481
void addRequestRange(size_t start, size_t len=0, DigestPtr digest=nullptr, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
Definition: request.cc:1239
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
Definition: request.cc:1301
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition: request.cc:1449
Priority priority() const
Definition: request.cc:1224
NetworkRequestError error() const
Returns the last set Error.
Definition: request.cc:1441
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
Definition: request.cc:1359
void * nativeHandle() const
Definition: request.cc:1296
std::vector< Range > failedRanges() const
Definition: request.cc:1270
const std::vector< Range > & requestedRanges() const
Definition: request.cc:1286
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
Definition: request.cc:1476
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> sigProgress()
Signals if there was data read from the download.
Definition: request.cc:1486
TransferSettings & transferSettings()
Definition: request.cc:1417
const std::string & lastRedirectInfo() const
Definition: request.cc:1291
#define CONNECT_TIMEOUT
Definition: curlhelper_p.h:21
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:25
#define MIL_MEDIA
Definition: mediadebug_p.h:29
#define DBG_MEDIA
Definition: mediadebug_p.h:28
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:325
int log_curl(CURL *, curl_infotype info, char *ptr, size_t len, void *max_lvl)
Definition: curlhelper.cc:68
Definition: Arch.h:352
String related utilities and Regular expression matching.
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:48
@ TRIM
Definition: String.h:500
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
\relates regex \ingroup ZYPP_STR_REGEX \relates regex \ingroup ZYPP_STR_REGEX
Definition: Regex.h:70
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
void dump(const char *text, FILE *stream, unsigned char *ptr, size_t size)
Definition: request.cc:36
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
Definition: request.cc:135
ZYPP_IMPL_PRIVATE(Provide)
int log_curl_all(CURL *handle, curl_infotype type, char *data, size_t size, void *userp)
Definition: request.cc:67
Structure holding values of curlrc options.
Definition: curlconfig.h:27
std::string proxyuserpwd
Definition: curlconfig.h:49
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
Convenient building of std::string with boost::format.
Definition: String.h:253
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
Definition: String.h:212
running_t(pending_t &&prevState)
Definition: request.cc:174
CheckSumBytes _checksum
Enables automated checking of downloaded contents against a checksum.
Definition: request.h:86
static Range make(size_t start, size_t len=0, DigestPtr &&digest=nullptr, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
Definition: request.cc:153
std::optional< size_t > _relevantDigestLen
Definition: request.h:87
std::optional< size_t > _chksumPad
Definition: request.h:88
std::chrono::microseconds appconnect
Definition: request.h:99
std::chrono::microseconds redirect
Definition: request.h:102
std::chrono::microseconds pretransfer
Definition: request.h:100
std::chrono::microseconds total
Definition: request.h:101
std::chrono::microseconds namelookup
Definition: request.h:97
std::chrono::microseconds connect
Definition: request.h:98
#define nullptr
Definition: Easy.h:55
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:436
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
#define DBG
Definition: Logger.h:95
#define MIL
Definition: Logger.h:96
#define WAR
Definition: Logger.h:97