openSUSE Build Service

Overview Repositories Revisions Requests Users Attributes Meta

File test.patch of Package libzypp (Revision d8fa6db4efe037bd3fe1ae65507f6188)

Currently displaying revision d8fa6db4efe037bd3fe1ae65507f6188 , Show latest

commit fb7a073381a64309218bce7e5339d36df2e8a519
Author: Bernhard M. Wiedemann <bwiedemann@suse.de>
Date:   Tue Sep 27 04:53:46 2022 +0000

Don't do 2 HTTP requests when one is enough
    
    TODO: if the file does not exist, we need to handle it
    
    improves time of zypper ref from 10.7 to 9.4s

diff --git a/zypp/MediaSetAccess.cc b/zypp/MediaSetAccess.cc
index b7faf2505..80e3be169 100644
--- a/zypp/MediaSetAccess.cc
+++ b/zypp/MediaSetAccess.cc
@@ -183,11 +183,12 @@ IMPL_PTR_TYPE(MediaSetAccess);
   {
     try
     {
-      if ( doesFileExist( file, media_nr ) )
+      //if ( doesFileExist( file, media_nr ) )
+      // handle not-found condition via MediaFileNotFoundException
         return provideFile( OnMediaLocation( file, media_nr ), PROVIDE_NON_INTERACTIVE );
     }
     catch ( const media::MediaFileNotFoundException & excpt_r )
-    { ZYPP_CAUGHT( excpt_r ); }
+    { } // ignore not found
     catch ( const media::MediaForbiddenException & excpt_r )
     { ZYPP_CAUGHT( excpt_r ); }
     catch ( const media::MediaNotAFileException & excpt_r )

commit c55053fe13d99f3d549516de0b170e086b363b5d
Author: Benjamin Zeller <bzeller@suse.de>
Date:   Tue Sep 27 10:51:35 2022 +0200

Use a dynamic fallback for BLKSIZE in downloads
    
    When not receiving a blocklist via metalink file from the server
    MediaMultiCurl used to fallback to a fixed, relatively small BLKSIZE.
    This patch changes the fallback into a dynamic value based on the
    filesize using a similar metric as the MirrorCache implementation on the
    server side.

diff --git a/zypp/media/MediaMultiCurl.cc b/zypp/media/MediaMultiCurl.cc
index f5986441f..72cecf229 100644
--- a/zypp/media/MediaMultiCurl.cc
+++ b/zypp/media/MediaMultiCurl.cc
@@ -121,6 +121,9 @@ public:
   void run(std::vector<Url> &urllist);
 
 protected:
+
+  static size_t makeBlksize ( size_t filesize );
+
   friend class multifetchworker;
 
   const MediaMultiCurl *_context;
@@ -139,6 +142,7 @@ protected:
   bool _havenewjob;
 
   size_t _blkno;
+  size_t _defaultBlksize = 0; //< The blocksize to use if the metalink file does not specify one
   off_t _blkoff;
   size_t _activeworkers;
   size_t _lookupworkers;
@@ -164,7 +168,6 @@ public:
 };
 
 constexpr auto MIN_REQ_MIRRS = 4;
-constexpr auto BLKSIZE       = 131072;
 constexpr auto MAXURLS       = 10;
 
 //////////////////////////////////////////////////////////////////////
@@ -707,18 +710,19 @@ multifetchworker::nextjob()
   MediaBlockList *blklist = _request->_blklist;
   if (!blklist)
     {
-      _blksize = BLKSIZE;
+      _blksize = _request->_defaultBlksize;
       if (_request->_filesize != off_t(-1))
-        {
-          if (_request->_blkoff >= _request->_filesize)
-            {
-              stealjob();
-              return;
-            }
-          _blksize = _request->_filesize - _request->_blkoff;
-          if (_blksize > BLKSIZE)
-            _blksize = BLKSIZE;
-        }
+      {
+        if (_request->_blkoff >= _request->_filesize)
+          {
+            stealjob();
+            return;
+          }
+        _blksize = _request->_filesize - _request->_blkoff;
+        if (_blksize > _request->_defaultBlksize)
+          _blksize = _request->_defaultBlksize;
+      }
+      DBG << "No BLOCKLIST falling back to chunk size: " <<  _request->_defaultBlksize << std::endl;
     }
   else
     {
@@ -734,8 +738,10 @@ multifetchworker::nextjob()
           _request->_blkoff = blk.off;
         }
       _blksize = blk.off + blk.size - _request->_blkoff;
-      if (_blksize > BLKSIZE && !blklist->haveChecksum(_request->_blkno))
-        _blksize = BLKSIZE;
+      if (_blksize > _request->_defaultBlksize && !blklist->haveChecksum(_request->_blkno)) {
+        DBG << "Block: "<< _request->_blkno << " has no checksum falling back to default blocksize: " <<  _request->_defaultBlksize << std::endl;
+        _blksize = _request->_defaultBlksize;
+      }
     }
   _blkno = _request->_blkno;
   _blkstart = _request->_blkoff;
@@ -791,6 +797,7 @@ multifetchrequest::multifetchrequest(const MediaMultiCurl *context, const Pathna
   _report = report;
   _blklist = blklist;
   _filesize = filesize;
+  _defaultBlksize = makeBlksize( filesize );
   _multi = multi;
   _stealing = false;
   _havenewjob = false;
@@ -1156,6 +1163,15 @@ multifetchrequest::run(std::vector<Url> &urllist)
     }
 }
 
+inline size_t multifetchrequest::makeBlksize ( size_t filesize )
+{
+  // this case should never happen because we never start a multi download if we do not know the filesize beforehand
+  if ( filesize == 0 )  return 2 * 1024 * 1024;
+  else if ( filesize < 2*256*1024 ) return filesize;
+  else if ( filesize < 8*1024*1024 ) return 256*1024;
+  else if ( filesize < 256*1024*1024 ) return 1024*1024;
+  return 4*1024*1024;
+}
 
 //////////////////////////////////////////////////////////////////////
 
@@ -1610,4 +1626,3 @@ void MediaMultiCurl::toEasyPool(const std::string &host, CURL *easy) const
 
   } // namespace media
 } // namespace zypp
-

commit cf5026534730351953cbd442b197b1bb96b0d229
Author: Bernhard M. Wiedemann <bwiedemann@suse.de>
Date:   Tue Sep 27 04:42:14 2022 +0000

Drop fetch of media.1
    
    changes time for zypper ref from 13.4 to 10.7 seconds

diff --git a/zypp/repo/yum/Downloader.cc b/zypp/repo/yum/Downloader.cc
index 4b495a8c3..1a1158d6e 100644
--- a/zypp/repo/yum/Downloader.cc
+++ b/zypp/repo/yum/Downloader.cc
@@ -201,9 +201,11 @@ namespace yum
   RepoStatus Downloader::status( MediaSetAccess & media_r )
   {
     RepoStatus ret { media_r.provideOptionalFile( repoInfo().path() / "/repodata/repomd.xml" ) };
+    /*
     if ( !ret.empty() )	// else: mandatory master index is missing
       ret = ret && RepoStatus( media_r.provideOptionalFile( "/media.1/media" ) );
     // else: mandatory master index is missing -> stay empty
+    */
     return ret;
   }
 } // namespace yum

commit d58ffe449e8af09fff7efdf9500241b5d7e0e553
Author: Bernhard M. Wiedemann <bwiedemann@suse.de>
Date:   Sat Oct 1 19:46:25 2022 +0200

Add ZYPP_NOCHUNK

diff --git a/zypp/media/MediaMultiCurl.cc b/zypp/media/MediaMultiCurl.cc
index 72cecf229..f532ca8e1 100644
--- a/zypp/media/MediaMultiCurl.cc
+++ b/zypp/media/MediaMultiCurl.cc
@@ -700,6 +700,7 @@ multifetchworker::disableCompetition()
 void
 multifetchworker::nextjob()
 {
+  const char *nochunk = getenv("ZYPP_NOCHUNK");
   _noendrange = false;
   if (_request->_stealing)
     {
@@ -708,11 +709,12 @@ multifetchworker::nextjob()
     }
 
   MediaBlockList *blklist = _request->_blklist;
-  if (!blklist)
+  if (nochunk || !blklist)
     {
       _blksize = _request->_defaultBlksize;
       if (_request->_filesize != off_t(-1))
       {
+        if (nochunk) _blksize = _request->_filesize;
         if (_request->_blkoff >= _request->_filesize)
           {
             stealjob();

Places

File test.patch of Package libzypp (Revision d8fa6db4efe037bd3fe1ae65507f6188)

Places