21 #include <zypp-core/base/DefaultIntegral> 23 #include <zypp-media/MediaException> 27 #include <zypp-core/base/UserRequestException> 31 #undef ZYPP_BASE_LOGGER_LOGGROUP 32 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp::fetcher" 68 if ( lhs->location.medianr() == rhs->location.medianr() )
69 return lhs->location.filename() < rhs->location.filename();
71 return lhs->location.medianr() < rhs->location.medianr();
118 return str << obj->location;
145 return a._pathName <
b._pathName;
150 Fetcher::Options
options()
const;
171 static shared_ptr<Impl> _nullimpl(
new Impl );
213 void validate(
const Pathname & localfile_r,
const std::list<FileChecker> & checkers_r );
219 const Pathname &dest_dir, FetcherJob::Flags flags );
234 friend Impl * rwcowClone<Impl>(
const Impl * rhs );
237 {
return new Impl( *
this ); }
240 std::set<FetcherIndex_Ptr,SameFetcherIndex>
_indexes;
282 job->checkers.push_back(checker);
287 _resources.push_back(job);
297 job->checkers.push_back(checker);
303 _resources.push_back(job);
309 if ( _mediaSetAccess )
310 _mediaSetAccess->precacheFiles( {resource} );
315 job->checkers.push_back(checker);
316 _resources.push_back(job);
321 MIL <<
"adding index " << resource << endl;
336 _mediaSetAccess = &media;
346 DBG <<
"Adding fetcher cache: '" << cache_dir <<
"'." << endl;
347 _caches.insert( { cache_dir,
options } );
352 ERR <<
"Not adding cache: '" << cache_dir <<
"'. Not a directory." << endl;
357 ERR <<
"Not adding cache '" << cache_dir <<
"'. Path does not exists." << endl;
371 return cacheLocation;
374 MIL <<
"start fetcher with " << _caches.size() <<
" cache directories." << endl;
375 for(
const CacheInfo & cacheInfo : _caches ) {
379 MIL <<
"file " << resource_r.
filename() <<
" found in cache " << cacheInfo._pathName << endl;
382 return cacheLocation;
393 MIL <<
"Checking job [" << localfile_r <<
"] (" << checkers_r.size() <<
" checkers )" << endl;
398 chkfnc( localfile_r );
400 ERR <<
"Invalid checker for '" << localfile_r <<
"'" << endl;
423 auto fnc_addIfInContent( [&](
const std::string & index_r ) ->
bool 432 downloadAndReadIndexList( media, dest_dir );
438 fnc_addIfInContent(
"CHECKSUMS" ) || fnc_addIfInContent(
"SHA1SUMS" );
442 fnc_addIfInContent(
"content" );
451 != _dircontent.end() )
455 std::copy(filled.begin(), filled.end(), std::back_inserter(content));
464 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
471 const Pathname &dest_dir, FetcherJob::Flags flags )
475 MIL <<
"Adding directory " << resource.
filename() << endl;
478 getDirectoryContent(media, resource, content);
483 WAR <<
"Skipping subtree hidden at " << resource.
filename() << endl;
489 autoaddIndexes(content, media, resource, dest_dir);
491 for ( filesystem::DirContent::const_iterator it = content.begin();
507 if ( _checksums.find(filename.
asString()) != _checksums.end() )
510 chksm = _checksums[filename.
asString()];
514 WAR <<
"Resource " << filename <<
" has no checksum in the index either." << endl;
524 addDirJobs(media, filename, dest_dir, flags);
539 scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard;
542 ManagedFile managedTmpFile = locateInCache( resource, destDir_r );
545 if ( tmpFile.
empty() )
547 MIL <<
"Not found in cache, retrieving..." << endl;
556 if ( tmpFile == destFullPath )
560 validate( tmpFile, jobp_r->checkers );
563 if ( tmpFile == destFullPath )
564 destFullPath.resetDispose();
567 if (
assert_dir( destFullPath->dirname() ) != 0 )
579 WAR <<
"optional resource " << resource <<
" could not be transferred." << endl;
608 readChecksumsIndex(index, basedir);
609 else if ( index.
basename() ==
"content" )
610 readContentFileIndex(index, basedir);
612 WAR << index <<
": index file format not known" << endl;
620 MIL << index <<
" contains " << reader.
_repoindex->mediaFileChecksums.size() <<
" checksums." << endl;
624 _checksums[(basedir + it->first).
asString()] = it->second;
631 std::ifstream in( index.
c_str() );
635 while (
getline( in, buffer ) )
638 if ( buffer[0] ==
'#' )
645 if ( buffer.empty() )
647 WAR <<
"Missing filename in CHECKSUMS file: " << index.
asString() <<
" (" <<
checksum <<
")" << endl;
660 MIL <<
"downloading index " << resource << endl;
686 fetcher.
start( dest_dir, media );
695 fetcher.
start( dest_dir, media );
700 getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.
filename()),
false);
702 WAR <<
"No public key specified by user for index '" << keyloc.
filename() <<
"'"<< endl;
706 fetcher.
start( dest_dir, media );
717 if ( _indexes.empty() )
719 MIL <<
"No indexes to read." << endl;
723 for_( it_idx, _indexes.begin(), _indexes.end() )
725 if ( (*it_idx)->read )
727 DBG <<
"Already read index " <<
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
732 downloadIndex( media, (*it_idx)->location, dest_dir );
734 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().
dirname() );
736 MIL <<
"Remember read index " <<
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
737 (*it_idx)->read =
true;
740 MIL <<
"done reading indexes" << endl;
746 if ( !_mediaSetAccess )
748 start( dest_dir, *_mediaSetAccess, progress );
757 _mediaSetAccess =
nullptr;
760 progress.
sendTo(progress_receiver);
762 downloadAndReadIndexList(media, dest_dir);
769 addDirJobs(media, location, dest_dir, jobp->flags);
783 getDirectoryContent(media, jobp->location.filename().dirname(), content);
786 MIL <<
"Autodiscovering signed indexes on '" 787 << jobp->location.filename().dirname() <<
"' for '" 788 << jobp->location.filename() <<
"'" << endl;
790 autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
794 getDirectoryContent(media,
Pathname(
"/"), content);
797 MIL <<
"Autodiscovering signed indexes on '" 799 << jobp->location.filename() <<
"'" << endl;
801 autoaddIndexes(content, media,
Pathname(
"/"), dest_dir);
806 if ( jobp->location.checksum().empty() )
808 if ( _checksums.find(jobp->location.filename().asString())
809 != _checksums.end() )
811 CheckSum chksm = _checksums[jobp->location.filename().asString()];
813 jobp->checkers.push_back(digest_check);
823 jobp->checkers.push_back(digest_check);
831 jobp->checkers.push_back(digest_check);
836 provideToDest( media, dest_dir, jobp );
838 if ( ! progress.incr() )
846 for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj.
_resources.begin(); it_res != obj.
_resources.end(); ++it_res )
std::string asString(const Patch::Category &obj)
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
std::set< CacheInfo > _caches
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
void setOptions(Options options)
Set the Fetcher options.
Listentry returned by readdir.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Checks for the validity of a signature.
FetcherJob & operator=(const FetcherJob &)=default
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
parser::susetags::RepoIndex_Ptr _repoindex
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
const char * c_str() const
String representation.
String related utilities and Regular expression matching.
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
std::string getline(std::istream &str)
Read one line from stream.
std::ostream & operator<<(std::ostream &str, const SerialNumber &obj)
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
If a content file is found, it is downloaded and read.
std::string basename() const
Return the last component of this path.
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
OnMediaLocation location
Index localtion.
zypp::RepoManager::RefreshServiceOptions _options
std::list< DirEntry > DirContent
Returned by readdir.
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
std::list< FileChecker > checkers
shared_ptr< FetcherJob > FetcherJob_Ptr
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
static shared_ptr< Impl > nullimpl()
Offer default Impl.
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
void remember(const Exception &old_r)
Store an other Exception as history.
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Impl & operator=(const Impl &)=delete
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
bool empty() const
Test for an empty path.
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
void addCachePath(const Pathname &cache_dir, Fetcher::CacheOptions options)
FetcherJob(const FetcherJob &)=default
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Fetcher::Options _options
const std::string & asString() const
String representation.
void setMediaSetAccess(MediaSetAccess &media)
bool isExist() const
Return whether valid stat info exists.
friend bool operator<(const CacheInfo &a, const CacheInfo &b)
void reset()
Reset the transfer (jobs) list.
MediaSetAccess * _mediaSetAccess
Pathname dirname() const
Return all but the last component od this path.
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Parse repoindex part from a content file.
FetcherJob(const OnMediaLocation &loc)
std::set ordering (less semantic)
Fetcher::Options options() const
Maintain [min,max] and counter (value) for progress counting.
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
DefaultIntegral< bool, false > read
Whether we read this index.
std::map< std::string, filesystem::DirContent > _dircontent
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
std::list< FetcherJob_Ptr > _resources
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress)
int unlink(const Pathname &path)
Like 'unlink'.
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
ZYPP_DECLARE_FLAGS(Flags, Flag)
shared_ptr< FetcherIndex > FetcherIndex_Ptr
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
If a CHECKSUMS file is found, it is downloaded and read.
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Class to encapsulate the OnMediaLocation object and the FileChecker together.
void setOptions(Fetcher::Options options)
FetcherIndex(const OnMediaLocation &loc)
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Base class for Exception.
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
std::map< std::string, CheckSum > _checksums
Wrapper class for ::stat/::lstat.
ManagedFile locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory)...
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Options options() const
Get current options.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Easy-to use interface to the ZYPP dependency resolver.
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
Fetcher::CacheOptions _options
void addIndex(const OnMediaLocation &resource)
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Impl * clone() const
clone for RWCOW_pointer