24 inline bool preloadEnabled()
27 if ( indeterminate(envstate) ) {
28 #if APIConfig(LIBZYPP_CONFIG_USE_SERIAL_PACKAGE_DOWNLOAD_BY_DEFAULT) 34 return bool(envstate);
38 if ( pck.isKind<Package>() ) {
40 }
else if ( pck.isKind<SrcPackage>() ) {
87 MIL <<
"No more jobs pending, exiting worker" << std::endl;
107 ERR <<
"Failed to create target dir for file: " <<
_targetPath << std::endl;
122 if ( !pathInfo.
isFile() ) {
123 if ( pathInfo.
isDir () )
145 _req = std::make_shared<zyppng::NetworkRequest>( url,
_tmpFile );
150 _req->resetRequestRanges();
158 _req->transferSettings() = settings;
171 const auto &pi =
_job;
220 std::vector<RepoUrl>::iterator curr = repoDlInfo._baseUrls.end();
221 int currentSmallestRef = INT_MAX;
223 for (
auto i = repoDlInfo._baseUrls.begin(); i != repoDlInfo._baseUrls.end(); i++ ) {
224 auto mirrorPtr = &(*i);
226 if ( skip == mirrorPtr )
234 if ( ( i->refs + i->miss ) < currentSmallestRef ) {
235 currentSmallestRef = ( i->refs + i->miss );
240 if ( curr == repoDlInfo._baseUrls.end() )
246 MIL <<
"Request for " << req.
url() <<
" started" << std::endl;
254 userData.
set(
"Url",
_req->url() );
269 MIL <<
"Request for " << req.
url() <<
" finished. (" << err.
toString() <<
")" << std::endl;
281 const auto &error = req.
error();
282 switch ( error.type() ) {
299 MIL <<
"Download from mirror failed for file " << req.
url () <<
" trying to taint mirror and move on" << std::endl;
311 MIL <<
"Found new mirror: " << url <<
" recovering, retry count: " <<
_notFoundRetry << std::endl;
314 _req->transferSettings () = settings;
327 std::string authHint = error.extraInfoValue(
"authHint", std::string());
346 DBG <<
"BUG: Download error flag is set , but Error code is NoError" << std::endl;
357 userData.
set(
"Url", *url );
359 userData.
set(
"description", *errorMessage );
378 if ( loc.medianr() > 1 )
386 std::string header { el.first };
389 MIL <<
"Added custom header -> " << header << std::endl;
394 resultSet = settings;
400 zyppng::NetworkRequestRef
_req;
424 if ( !preloadEnabled() ) {
425 MIL <<
"CommitPackagePreloader disabled" << std::endl;
431 MIL <<
"No receiver for the CommitPreloadReport, skipping preload phase" << std::endl;
436 _dispatcher = std::make_shared<zyppng::NetworkRequestDispatcher>();
443 _pTracker = std::make_shared<internal::ProgressTracker>();
454 for (
const auto &step : steps ) {
455 switch ( step.stepType() )
474 if ( pi->lookupLocation().checksum().empty() )
478 if( !pckCachedLocation(pi).empty() )
481 auto repoDlsIter =
_dlRepoInfo.find( pi.repository().id() );
486 ERR <<
"Failed to create predownload cache for repo " << pi.repoInfo().alias() << std::endl;
491 std::vector<RepoUrl> repoUrls;
492 const auto bu = pi.repoInfo().effectiveBaseUrls();
493 std::for_each( bu.begin(), bu.end(), [&](
const zypp::Url &u ) {
503 if ( rewriteUrl.isValid () )
507 MIL <<
"Adding Url: " << url <<
" to the mirror set" << std::endl;
511 .headers = std::move(custom_headers)
516 if( repoUrls.empty() ) {
517 MIL <<
"Skipping predownload for " << step.satSolvable() <<
" no downloading URL" << std::endl;
523 if ( repoUrls.begin()->baseUrl.getHost() ==
"download.opensuse.org" ){
528 pi.repository().id(),
546 const auto &workerDone = [&,
this](){
547 if ( std::all_of(
_workers.begin(),
_workers.end(), [](
const auto &w ) {
return w->finished();} ) )
563 auto worker = std::make_shared<PreloadWorker>(*this);
564 worker->sigWorkerFinished().connect(workerDone);
566 _workers.push_back( std::move(worker) );
570 MIL <<
"Running preload event loop!" << std::endl;
574 MIL <<
"Preloading done, mirror stats: " << std::endl;
576 std::for_each ( elem.second._baseUrls.begin (), elem.second._baseUrls.end(), [](
const RepoUrl &repoUrl ){
577 MIL <<
"url: " << repoUrl.baseUrl <<
" misses: " << repoUrl.miss << std::endl;
580 MIL <<
"Preloading done, mirror stats end" << std::endl;
585 if ( !preloadEnabled() ) {
586 MIL <<
"CommitPackagePreloader disabled" << std::endl;
602 const auto now = clock::now();
603 bool canUpdate =
false;
605 const auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now - *
_lastProgressUpdate);
606 canUpdate = (duration >= std::chrono::milliseconds(500));
618 userData.
set(
"dbps_avg" , static_cast<double>(
_pTracker->_drateTotal ) );
619 userData.
set(
"dbps_current", static_cast<double>(
_pTracker->_drateLast ) );
620 userData.
set(
"bytesReceived", static_cast<double>(
_pTracker->_dnlNow ) );
621 userData.
set(
"bytesRequired", static_cast<double>(
_pTracker->_dnlTotal ) );
std::string asString(const Patch::Category &obj)
RepoUrl * findUsableMirror(RepoUrl *skip=nullptr, bool allowTainted=true)
Tries to find a usable mirror
Repository::IdType _currentRepoId
boost::logic::tribool TriBool
3-state boolean logic (true, false and indeterminate).
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Pathname cachedLocation(const OnMediaLocation &loc_r, const RepoInfo &repo_r)
void onRequestStarted(zyppng::NetworkRequest &req)
[M] Install(multiversion) item (
static ZConfig & instance()
Singleton ctor.
int clean_dir(const Pathname &path)
Like 'rm -r DIR/ *'.
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
zyppng::NetworkRequestRef _req
void appendPathName(const Pathname &path_r, EEncoding eflag_r=zypp::url::E_DECODED)
Extend the path name.
void finishCurrentJob(const zypp::Pathname &localPath, const std::optional< zypp::Url > &url, media::CommitPreloadReport::Error e, const std::optional< std::string > &errorMessage, bool fatal)
void reset()
Reset to default Ctor values.
Store and operate with byte count.
static const RepoIdType noRepoId(0)
Id to denote Repo::noRepository.
std::optional< clock::time_point > _lastProgressUpdate
CommitPackagePreloader & _parent
zyppng::Signal< void()> _sigFinished
std::set< RepoUrl * > _taintedMirrors
std::string toString() const
toString Returns a string representation of the error
String related utilities and Regular expression matching.
media::UrlResolverPlugin::HeaderList headers
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
media::UrlResolverPlugin::HeaderList headers
zyppng::NetworkRequestDispatcherRef _dispatcher
std::string basename() const
Return the last component of this path.
TransferSettings & transferSettings()
callback::SendReport< media::CommitPreloadReport > _report
void preloadTransaction(const std::vector< sat::Transaction::Step > &steps)
OnMediaLocation lookupLocation() const
bool hasError() const
Checks if there was a error with the request.
void prepareSettingsAndUrl(zypp::Url &url_r, zypp::media::TransferSettings &s)
bool isKind(const Resolvable::constPtr &p)
Test whether a Resolvable::Ptr is of a certain Kind.
void onRequestFinished(zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)
bool taintCurrentMirror()
Taints the current mirror, returns true if a alternative was found
void makeJobUrl(zypp::Url &resultUrl, media::TransferSettings &resultSet)
TriBool getenvBool(const C_Str &var_r)
If the environment variable var_r is set to a legal true or false string return bool, else indeterminate.
zypp::Pathname _targetPath
ByteCount _downloadedBytes
bool set(const std::string &key_r, AnyType val_r)
Set the value for key (nonconst version always returns true).
zyppng::Ref< internal::ProgressTracker > _pTracker
bool isExist() const
Return whether valid stat info exists.
std::vector< zyppng::Ref< PreloadWorker > > _workers
RepoInfo repoInfo() const
Pathname dirname() const
Return all but the last component od this path.
void onRequestProgress(zyppng::NetworkRequest &req, zypp::ByteCount count)
RepoInfo info() const
Return any associated RepoInfo.
The NetworkRequestError class Represents a error that occured in.
NetworkRequestError error() const
Returns the last set Error.
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
std::deque< PoolItem > _requiredDls
sat::detail::RepoIdType IdType
std::map< Repository::IdType, RepoDownloadData > _dlRepoInfo
std::vector< RepoUrl > _baseUrls
int unlink(const Pathname &path)
Like 'unlink'.
void resetDispose()
Set no dispose function.
Pathname predownloadPath() const
Path where this repo packages are predownloaded.
zyppng::SignalProxy< void()> sigWorkerFinished()
bool any_of(const Container &c, Fnc &&cb)
Typesafe passing of user data via callbacks.
int chmodApplyUmask(const Pathname &path, mode_t mode)
Similar to 'chmod', but mode is modified by the process's umask in the usual way. ...
Wrapper class for ::stat/::lstat.
void reportBytesDownloaded(ByteCount newBytes)
Combining sat::Solvable and ResStatus.
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
IdType id() const
Expert backdoor.
Easy-to use interface to the ZYPP dependency resolver.
int rmdir(const Pathname &path)
Like 'rmdir'.
Repository repository() const
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
PreloadWorker(CommitPackagePreloader &parent)
static ManagedFile asManagedFile()
Create a temporary file and convert it to a automatically cleaned up ManagedFile. ...