Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:andriinikitin:branches:zypp:Head
libzypp
0001-Add-predownload-plugin.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 0001-Add-predownload-plugin.patch of Package libzypp
From 09a799a2e9c33570bf3cc30d024d2f4fea5e79d8 Mon Sep 17 00:00:00 2001 From: Andrii Nikitin <anikitin@suse.de> Date: Fri, 27 Sep 2024 07:46:29 +0200 Subject: [PATCH] Add predownload plugin --- zypp/Fetcher.cc | 48 +++++++-- zypp/PluginExecutor.cc | 6 ++ zypp/PluginExecutor.h | 3 + zypp/PluginScript.cc | 22 +++++ zypp/PluginScript.h | 10 ++ zypp/target/CommitPackageCache.cc | 3 + zypp/target/CommitPackageCache.h | 1 + zypp/target/CommitPackageCacheImpl.h | 5 + zypp/target/TargetImpl.cc | 141 +++++++++++++++++++++++++++ 9 files changed, 231 insertions(+), 8 deletions(-) diff --git a/zypp/Fetcher.cc b/zypp/Fetcher.cc index df1a5c016..1dae441c2 100644 --- a/zypp/Fetcher.cc +++ b/zypp/Fetcher.cc @@ -196,7 +196,7 @@ namespace zypp * the cache (matching checksum is mandatory). Returns the * location of the cached file or an empty \ref Pathname. */ - Pathname locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r ); + Pathname locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r, const std::string & suffix_in_cache = "" ); /** * Validates the provided file against its checkers. * \throws Exception @@ -350,7 +350,7 @@ namespace zypp } - Pathname Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r ) + Pathname Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r, const std::string & suffix_in_cache ) { Pathname ret; // No checksum - no match @@ -359,6 +359,10 @@ namespace zypp // first check in the destination directory Pathname cacheLocation = destDir_r / resource_r.filename(); + + if ( !suffix_in_cache.empty() ) + cacheLocation = cacheLocation.extend( suffix_in_cache.c_str() ); + if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) ) { swap( ret, cacheLocation ); @@ -369,11 +373,19 @@ namespace zypp for( const Pathname & cacheDir : _caches ) { cacheLocation = cacheDir / resource_r.filename(); - if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) ) + std::string detail = cacheDir.asString(); + if ( !suffix_in_cache.empty() ) + detail += "with suffix " + suffix_in_cache; + + if ( PathInfo(cacheLocation).isExist() ) { - MIL << "file " << resource_r.filename() << " found in cache " << cacheDir << endl; - swap( ret, cacheLocation ); - return ret; + MIL << "file " << resource_r.filename() << " found in cache " << detail << endl; + if (!is_checksum( cacheLocation, resource_r.checksum() ) ) + MIL << "file " << resource_r.filename() << " failed checksum verification in cache " << detail << endl; + else { + swap( ret, cacheLocation ); + return ret; + } } } @@ -534,8 +546,28 @@ namespace zypp // get cached file (by checksum) or provide from media Pathname tmpFile = locateInCache( resource, destDir_r ); - if ( tmpFile.empty() ) - { + if ( tmpFile.empty() ) { + const std::string suffix(".unverified"); + Pathname tmpFileUnverified = locateInCache( resource, destDir_r, suffix ); + if (!tmpFileUnverified.empty()) { + try { + validate( tmpFileUnverified, jobp_r->checkers ); + if (tmpFileUnverified.extension() == suffix ) { + std::string_view dest = tmpFileUnverified.asString().c_str(); + dest.remove_suffix( suffix.length() ); + + rename( tmpFileUnverified.asString().c_str(), std::string(dest).c_str() ); + tmpFile = locateInCache( resource, destDir_r ); + } + } catch (...) { + // ignore all exceptions and proceed normally + }; + } + } + + if ( !tmpFile.empty() ) { + MIL << "Found in cache, verifying..." << endl; + } else { MIL << "Not found in cache, retrieving..." << endl; tmpFile = media_r.provideFile( resource, resource.optional() ? MediaSetAccess::PROVIDE_NON_INTERACTIVE : MediaSetAccess::PROVIDE_DEFAULT ); releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block diff --git a/zypp/PluginExecutor.cc b/zypp/PluginExecutor.cc index 826a915cf..3cefba596 100644 --- a/zypp/PluginExecutor.cc +++ b/zypp/PluginExecutor.cc @@ -101,6 +101,9 @@ namespace zypp const std::list<PluginScript> scripts() const { return _scripts; } + PluginScript first() + { return *_scripts.begin(); } + private: /** Launch a plugin sending PLUGINSTART message. */ void doLoad( const PathInfo & pi_r ) @@ -177,6 +180,9 @@ namespace zypp void PluginExecutor::send( const PluginFrame & frame_r ) { _pimpl->send( frame_r ); } + PluginScript PluginExecutor::first() + { return _pimpl->first(); } + std::ostream & operator<<( std::ostream & str, const PluginExecutor & obj ) { return str << obj._pimpl->scripts(); } diff --git a/zypp/PluginExecutor.h b/zypp/PluginExecutor.h index 275112734..51cafd39e 100644 --- a/zypp/PluginExecutor.h +++ b/zypp/PluginExecutor.h @@ -61,6 +61,9 @@ namespace zypp /** Number of open plugins */ size_t size() const; + /** First plugin */ + PluginScript first(); + public: /** Find and launch plugins sending \c PLUGINBEGIN. * diff --git a/zypp/PluginScript.cc b/zypp/PluginScript.cc index 89fcf0796..94e1bd09b 100644 --- a/zypp/PluginScript.cc +++ b/zypp/PluginScript.cc @@ -175,6 +175,8 @@ namespace zypp void send( const PluginFrame & frame_r ) const; + Progress progress() const; + PluginFrame receive() const; private: @@ -274,6 +276,23 @@ namespace zypp return _lastReturn; } + PluginScript::Progress PluginScript::Impl::progress() const + { + const PluginFrame frame_r = PluginFrame( "PLUGIN_PROGRESS" ); + PluginFrame resp; + Progress ret(-1,-1); + this->send( frame_r ); + resp = this->receive(); + if(resp.empty()) + return ret; + + ret.first = atoi( resp.getHeaderNT("PLUGIN_PROGRESS_CURRENT", "-1").c_str() ); + ret.second = atoi( resp.getHeaderNT("PLUGIN_PROGRESS_MAX", "-1").c_str() ); + + return ret; + } + + void PluginScript::Impl::send( const PluginFrame & frame_r ) const { if ( !_cmd ) @@ -519,6 +538,9 @@ namespace zypp void PluginScript::send( const PluginFrame & frame_r ) const { _pimpl->send( frame_r ); } + PluginScript::Progress PluginScript::progress() const + { return _pimpl->progress(); } + PluginFrame PluginScript::receive() const { return _pimpl->receive(); } diff --git a/zypp/PluginScript.h b/zypp/PluginScript.h index fc6410cca..881bc7626 100644 --- a/zypp/PluginScript.h +++ b/zypp/PluginScript.h @@ -66,6 +66,7 @@ namespace zypp public: /** Commandline arguments passed to a script on \ref open. */ using Arguments = std::vector<std::string>; + using Progress = std::pair<int, int>; /** \c pid_t(-1) constant indicating no connection. */ static const pid_t NotConnected; @@ -171,6 +172,15 @@ namespace zypp * */ void send( const PluginFrame & frame_r ) const; + + /** Send PLUGIN_PROGRESS frame and return /ref PluginScript::Progress + * \throw PluginScriptNotConnected + * \throw PluginScriptSendTimeout + * \throw PluginScriptDiedUnexpectedly (does not \ref close) + * \throw PluginScriptException on error + * + */ + Progress progress() const; /** Receive a \ref PluginFrame. * \throw PluginScriptNotConnected diff --git a/zypp/target/CommitPackageCache.cc b/zypp/target/CommitPackageCache.cc index 8fea5b63a..7e6414489 100644 --- a/zypp/target/CommitPackageCache.cc +++ b/zypp/target/CommitPackageCache.cc @@ -146,6 +146,9 @@ namespace zypp ManagedFile CommitPackageCache::get( const PoolItem & citem_r ) { return _pimpl->get( citem_r ); } + ManagedFile CommitPackageCache::get_from_cache( const PoolItem & citem_r ) + { return _pimpl->get_from_cache( citem_r ); } + bool CommitPackageCache::preloaded() const { return _pimpl->preloaded(); } diff --git a/zypp/target/CommitPackageCache.h b/zypp/target/CommitPackageCache.h index bf50c92dd..4be69eb14 100644 --- a/zypp/target/CommitPackageCache.h +++ b/zypp/target/CommitPackageCache.h @@ -84,6 +84,7 @@ namespace zypp /** Provide a package. */ ManagedFile get( const PoolItem & citem_r ); + ManagedFile get_from_cache( const PoolItem & citem_r ); /** \overload */ ManagedFile get( sat::Solvable citem_r ) { return get( PoolItem(citem_r) ); } diff --git a/zypp/target/CommitPackageCacheImpl.h b/zypp/target/CommitPackageCacheImpl.h index 95ac84936..5b5a9284d 100644 --- a/zypp/target/CommitPackageCacheImpl.h +++ b/zypp/target/CommitPackageCacheImpl.h @@ -59,6 +59,11 @@ namespace zypp return sourceProvidePackage( citem_r ); } + virtual ManagedFile get_from_cache( const PoolItem & citem_r ) + { + return sourceProvideCachedPackage( citem_r ); + } + void setCommitList( std::vector<sat::Solvable> commitList_r ) { _commitList = std::move(commitList_r); } diff --git a/zypp/target/TargetImpl.cc b/zypp/target/TargetImpl.cc index 1960c8a3a..6741b9f2c 100644 --- a/zypp/target/TargetImpl.cc +++ b/zypp/target/TargetImpl.cc @@ -1301,6 +1301,8 @@ namespace zypp MIL << "Target loaded: " << system.solvablesSize() << " resolvables" << endl; } + void predownloadPluginsHook( CommitPackageCache & packageCache, const ZYppCommitResult::TransactionStepList & steps ); + /////////////////////////////////////////////////////////////////// // // COMMIT @@ -1464,6 +1466,9 @@ namespace zypp bool miss = false; if ( policy_r.downloadMode() != DownloadAsNeeded ) { + + predownloadPluginsHook(packageCache, steps); + // Preload the cache. Until now this means pre-loading all packages. // Once DownloadInHeaps is fully implemented, this will change and // we may actually have more than one heap. @@ -3046,6 +3051,142 @@ namespace zypp repo::SrcPackageProvider prov( access_r ); return prov.provideSrcPackage( srcPackage_r ); } + + + void predownloadPluginsHook( CommitPackageCache & packageCache, const ZYppCommitResult::TransactionStepList & steps ) + { + PluginExecutor plugins; + plugins.load( ZConfig::instance().pluginsPath()/"predownload" ); + if ( ! plugins ) + return; + MIL << "TargetImpl::predownloadPluginsHook() start" << endl; + + // first build items for download, skipping those which are already in the cache + ZYppCommitResult::TransactionStepList stepsNotCached; + for_( it, steps.begin(), steps.end() ) + { + switch ( it->stepType() ) + { + case sat::Transaction::TRANSACTION_INSTALL: + case sat::Transaction::TRANSACTION_MULTIINSTALL: + // proceed: only install actionas may require download. + break; + + default: + // next: no download for or non-packages and delete actions. + continue; + break; + } + + PoolItem pi( *it ); + if ( pi->isKind<Package>() || pi->isKind<SrcPackage>() ) + { + ManagedFile localfile; + localfile = packageCache.get_from_cache( pi ); + if (!localfile->empty()) + localfile.resetDispose(); // keep the package file in the cache + else { + stepsNotCached.push_back(*it); + } + } + } + using namespace std; + if (stepsNotCached.empty()) { + JobReport::info( "Nothing to download" ); + return; + } + JobReport::info( "Preparing predownload plugins..." ); + + string message = ( ZConfig::instance().repoCachePath()/"packages" ).asString(); + plugins.send( PluginFrame( "PREDOWNLOAD_DEST", message ) ); + if ( plugins.empty() ) { + JobReport::warning( "Predownload plugins have died" ); + return; + } + + // collect info about required downloads + map<string, Url> repoUrls; // repo alias => baseurl + map<string, list<pair<int, string> > > repoFiles; // repo alias => [ ( file_size, file_location ) ] + for_(it, stepsNotCached.begin(), stepsNotCached.end()) { + if ( sat::Solvable solv = it->satSolvable() ) { + Repository repo = solv.repository(); + string alias = repo.alias(); + Url url = repo.info().url(); + if (!url.isValid()) + continue; + PoolItem pi( solv ); + + Package::constPtr p = pi->asKind<Package>(); + string loc = p->location().filename().asString(); + + if (loc.size() < 2) + continue; + // let's trim the leading './' + if (loc[0] == '.' && loc[1] == '/') + loc = loc.substr(2); + + int sz = p->location().downloadSize(); + repoUrls[alias] = url; + repoFiles[alias].push_back(make_pair(sz, loc)); + } + } + + // send a message to plugin: + // baseurl file1 file2 .. fileN + for_(it, repoUrls.begin(), repoUrls.end()) { + message = it->first + string(" ") + it->second.asString(); + list<pair<int, string> >& files = repoFiles[it->first]; + if (files.empty()) + continue; + files.sort(); + // reverse loop because after sort() the bigger files are at the end, we want the download start with them + for_(i, files.rbegin(), files.rend()) { + message += " " + i->second; + } + message += " "; // make sure there is a delimiter after last package + plugins.send( PluginFrame( "PREDOWNLOAD_FROM_REPO", message ) ); + } + if ( plugins.empty() ) { + JobReport::warning( "Predownload plugins have died" ); + return; + } + // only the first survived plugin will be called for now + PluginScript plugin = plugins.first(); + JobReport::info( "Starting predownload plugin..." ); + + plugin.send( PluginFrame( "PREDOWNLOAD_START" ) ); + plugin.receive(); + typedef PluginScript::Progress Progress; + Progress last(0,0); + bool allgood = 0; + + Progress pr = plugin.progress(); + while(1) { + if (pr.second < 1) { + JobReport::warning( "Predownload plugin have failed to report progress, continue without plugin" ); + break; + } + JobReport::info( str::Format(_("Waiting predownload... Progress: %1%/%2%") ) % pr.first % pr.second ); + last = pr; + + if (pr.first < pr.second) + sleep(1); + else { + if (pr.first > 0) + allgood = 1; + break; + } + pr = plugin.progress(); + } + const string & lastError = plugin.lastExecError(); + if (!lastError.empty()) { + JobReport::warning( "Predownload plugin error: " + lastError ); + } else if (allgood) { + JobReport::info( "Predownload plugin finished without errors" ); + } + + MIL << "TargetImpl::predownloadPluginsHook() end" << endl; + } //////////////////////////////////////////////////////////////// } // namespace target /////////////////////////////////////////////////////////////////// -- 2.46.1
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
