File 0001-update-mtime-to-avoid-mirror-failures.patch of Package product-builder

Overview Repositories Revisions Requests Users Attributes Meta

File 0001-update-mtime-to-avoid-mirror-failures.patch of Package product-builder (Revision b6ec8dceada37e9a5b30491c4b7949dd)

Currently displaying revision b6ec8dceada37e9a5b30491c4b7949dd , Show latest

From 263b611c421970c2747bb586cbd0d583fb90eefe Mon Sep 17 00:00:00 2001
From: Jan Zerebecki <jan.suse@zerebecki.de>
Date: Wed, 17 May 2023 16:28:11 +0200
Subject: [PATCH] update mtime to avoid mirror failures

Mirrors may use rsync with the skip on same mtime feature, which would
skip files that are different in content but have the same mtime. This
results in an inconsistent mirror.

Avoid this by touching files of extracted content so the mtime is updated.

To make rsync skip by mtime more efficient, if an old copy of the build
result exists, use the old mtime for files that are the same.

Updating mtime is only needed for ouput that is a directory which
happens on REPO_ONLY but not when output is a single file like a disk
image or archive.

When the rpm macro %clamp_mtime_to_source_date_epoch is set to Y to
enable reproducible builds, the mtime of files in the rpm will be set to
the date of the last changes entry, but build dependencies that affect
the content may be newer. This is relevant when extracting such an rpm
in REPO_ONLY output for the installer to use. Some mirrors may fail to
sync to the newest content as they skipped them. This would make an
installer using that mirror fail.

Fixes: https://bugzilla.opensuse.org/show_bug.cgi?id=1148824
---
 modules/KIWICollect.pm |  8 ++---
 modules/KIWIUtil.pm    | 69 +++++++++++++++++++++++++++++++++++++++---
 2 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/modules/KIWICollect.pm b/modules/KIWICollect.pm
index 34e24d4..e7ec249 100644
--- a/modules/KIWICollect.pm
+++ b/modules/KIWICollect.pm
@@ -1627,7 +1627,7 @@ sub unpackMetapackages {
 
                     $this->logMsg('I', "unpack $packPointer->{localfile} ");
                     $this->{m_util}->unpac_package(
-                        $packPointer->{localfile}, $tmp
+                        $packPointer->{localfile}, $tmp, $this->{m_proddata}->getVar("REPO_ONLY", 'false') eq "true"
                     );
                     # copy content of CD1 ... CD<i> subdirs if exists:
                     for (1..10) {
@@ -2179,7 +2179,7 @@ sub collectProducts {
         }
         $this->logMsg('I',
             "Unpacking product release package $i in file $file ".$tmp);
-        $this->{m_util}->unpac_package($file, $tmp);
+        $this->{m_util}->unpac_package($file, $tmp, $this->{m_proddata}->getVar("REPO_ONLY", 'false') eq "true");
 
         # get all .prod files
         local *D;
@@ -2372,7 +2372,7 @@ sub unpackModules {
             my $pack_file = $this->getBestPackFromRepos($module, $arch)
                     ->{'localfile'};
             $this->logMsg('I', "Unpacking $pack_file to $arch_tmp_dir/");
-            $this->{m_util}->unpac_package($pack_file, "$arch_tmp_dir");
+            $this->{m_util}->unpac_package($pack_file, "$arch_tmp_dir", $this->{m_proddata}->getVar("REPO_ONLY", 'false') eq "true");
         }
     }
     # copy modules from temp dir to targets
@@ -2447,7 +2447,7 @@ sub unpackInstSys {
             my $pack_file =
                 $this->getBestPackFromRepos($module, $arch)->{'localfile'};
             $this->logMsg('I', "Unpacking $pack_file to $arch_tmp_dir");
-            $this->{m_util}->unpac_package($pack_file, "$arch_tmp_dir");
+            $this->{m_util}->unpac_package($pack_file, "$arch_tmp_dir", $this->{m_proddata}->getVar("REPO_ONLY", 'false') eq "true");
         }
     }
     # copy inst_sys_packages from temp dir to targets
diff --git a/modules/KIWIUtil.pm b/modules/KIWIUtil.pm
index 9285fe6..ae5ed2f 100644
--- a/modules/KIWIUtil.pm
+++ b/modules/KIWIUtil.pm
@@ -19,9 +19,12 @@ package KIWIUtil;
 #------------------------------------------
 use strict;
 use warnings;
+use File::Compare;
 use File::Glob ':glob';
 use File::Find;
 use File::Path;
+use Time::HiRes qw(stat);
+use Time::HiRes qw(utime);
 
 #==========================================
 # KIWI Modules
@@ -483,10 +486,12 @@ sub unpac_package {
     #   $this - class name; always call as member
     #   $p_uri - uri of the rpm file to unpack
     #   $dir - target dir for the unpacking, created if necessary
+    #   $touch - do not restore mtime on extracted files, reproducible builds can make changes without newer mtimes, use this when creating directories instead of images like REPO_ONLY to makes sure that rsync skip by mtime still works
     # ---
     my $this   = shift;
     my $p_uri  = shift;
     my $dir    = shift;
+    my $touch  = shift;
     my $retval = 0;
     if(!($this and $p_uri and $dir)) {
         $retval = 1;
@@ -500,26 +505,80 @@ sub unpac_package {
             goto up_failed;
         }
     }
+
+    my $odir = $dir;
+    my $tmpdir;
+    if($touch) {
+        my $tmpdir = KIWIQX::qxx ("mktemp -qdt kiwiunpac.XXXXXX"); chomp $tmpdir;
+        $dir = $tmpdir;
+    }
+
     if($p_uri =~ m{(.*\.tgz|.*\.tar\.gz|.*\.taz|.*\.tar\.Z)}) {
         my $out = qx(cd $dir && tar -zxvfp $p_uri);
         my $status = $?>>8;
         if($status != 0) {
-            my $msg = "[E] command cp $dir && tar xvzfp $p_uri failed!\n";
+            my $msg = "[E] command cd $dir && tar xvzfp $p_uri failed!\n";
             $this->{m_collect}->logMsg("E", $msg);
             $this->{m_collect}->logMsg("E", "\t$out\n");
             $retval = 5;
             goto up_failed;
-        } else {
-            my $msg = "[I] unpacked $p_uri in directory $dir\n";
-            $this->{m_collect}->logMsg("I", $msg);
         }
     } elsif($p_uri =~ m{.*\.rpm}i) {
-        my $out = qx(cd $dir && unrpm -q $p_uri);
+       my $out = qx(cd $dir && unrpm -q $p_uri);
+       if($? != 0) {
+           $this->{m_collect}->logMsg("E", "[E] command cd $dir && unrpm -q $p_uri failed, output follows!\n");
+           $this->{m_collect}->logMsg("E", "\t$out\n");
+           $retval = 5;
+           goto up_failed;
+       }
     } else {
         $this->{m_collect}->logMsg("E", "[E] cannot process file $p_uri\n");
         $retval = 4;
         goto up_failed;
     }
+    my $msg = "[I] unpacked $p_uri in directory $dir\n";
+    $this->{m_collect}->logMsg("I", $msg);
+
+    if($touch) {
+        # if an old copy of the build result exists use the old mtime for files that are the same so rsync skip by mtime is more efficient
+        my $olddir;
+        my $exists = 0;
+        if($ENV{BUILD_ROOT}) {
+            $olddir = "$ENV{BUILD_ROOT}/.build.oldpackages";
+            $exists = -d($olddir);
+            $this->{m_collect}->logMsg("I", "[I] \$BUILD_ROOT is $ENV{BUILD_ROOT} and existence of \$BUILD_ROOT/.build.oldpackages is $exists\n");
+
+            #TODO debug remove
+            my $res = qx(find $ENV{BUILD_ROOT}/.build.oldpackages);
+            my $m = "[I] status: $? output: find $ENV{BUILD_ROOT}/.build.oldpackages\n$res\n";
+            $this->{m_collect}->logMsg("I", $m);
+
+        } else {
+            $this->{m_collect}->logMsg("I", "[I] \$BUILD_ROOT is not set\n");
+        }
+
+        my $now = time();
+        find({ wanted => sub {
+            my $old = $olddir . substr($_, length($tmpdir));
+            my $time;
+            if ($exists && compare($old, $_)) {
+                # index 9 is mtime
+                $time = (stat($old))[9];
+            } else {
+                $time = $now;
+            }
+            # this is the equivalent of the touch command
+            utime($time, $time, $_);
+        } , no_chdir => 1 }, $tmpdir);
+        my $out = qx(find $tmpdir -mindepth 1 -maxdepth 1 -exec cp -a -t $odir '{}' '+' && rm -rf $tmpdir);
+        if($? != 0) {
+           $this->{m_collect}->logMsg("E", "[E] command find $tmpdir -mindepth 1 -maxdepth 1 -exec cp -a -t $odir '{}' '+' && rm -rf $tmpdir failed, output follows!\n");
+           $this->{m_collect}->logMsg("E", "\t$out\n");
+           $retval = 6;
+           goto up_failed;
+        }
+    }
+
     up_failed:
     return $retval;
 }
-- 
2.30.2

Places

File 0001-update-mtime-to-avoid-mirror-failures.patch of Package product-builder (Revision b6ec8dceada37e9a5b30491c4b7949dd)

Places