From 3bae6e4ffad8ba76df8333d9e43ec49ff2607eed Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Wed, 24 Jan 2024 17:19:18 +0100 Subject: [PATCH] pbuild: Add support for the productcompose build type --- PBuild/BuildResult.pm | 6 +- PBuild/Checker.pm | 186 ++++++++++++++++++++++++++++++++++++------ PBuild/Container.pm | 2 +- PBuild/Expand.pm | 9 ++ PBuild/Job.pm | 11 ++- PBuild/LocalRepo.pm | 8 ++ PBuild/OBS.pm | 126 ++++++++++++++++++++++++++++ PBuild/RemoteRepo.pm | 110 ++++++++++++++++++++++++- PBuild/RepoMgr.pm | 70 ++++++++++++++-- PBuild/Util.pm | 11 +++ 10 files changed, 502 insertions(+), 37 deletions(-) diff --git a/PBuild/BuildResult.pm b/PBuild/BuildResult.pm index 79e1666ea..08c05bcf8 100644 --- a/PBuild/BuildResult.pm +++ b/PBuild/BuildResult.pm @@ -114,7 +114,7 @@ sub integrate_build_result { next if $file eq '_meta' || $file eq '_meta.success' || $file eq '_meta.fail'; next if $file eq '_log' || $file eq '_log.success'; next if $file eq '_repository'; - unlink("$dst/$file"); + unlink("$dst/$file") || PBuild::Util::rm_rf("$dst/$file"); } # copy new stuff over for my $file (sort keys %$result) { @@ -126,6 +126,10 @@ sub integrate_build_result { my $obsbinlnk = PBuild::Container::containerinfo2obsbinlnk($1, $2, $p->{'pkg'}); PBuild::Util::store("$dst/$prefix.obsbinlnk", undef, $obsbinlnk) if $obsbinlnk; } + if ($p->{'buildtype'} eq 'productcompose' && -d $result->{$file}) { + PBuild::Util::cp_a($result->{$file}, "$dst/$file"); + next; + } PBuild::Util::cp($result->{$file}, "$dst/$file"); } # create new bininfo diff --git a/PBuild/Checker.pm b/PBuild/Checker.pm index e4ee6ea9d..107619336 100644 --- a/PBuild/Checker.pm +++ b/PBuild/Checker.pm @@ -79,6 +79,7 @@ sub prepare { $ctx->{'dep2pkg'} = $dep2pkg; $ctx->{'subpacks'} = \%subpacks; PBuild::Meta::setgenmetaalgo($ctx->{'genmetaalgo'}); + $ctx->{'repos'} = $repos; $ctx->{'dep2pkg_host'} = PBuild::Expand::configure_repos($ctx->{'bconf_host'}, $hostrepos) if $ctx->{'bconf_host'}; } @@ -301,6 +302,34 @@ sub genmeta_image { return \@new_meta; } +# +# Generate the dependency tracking data for a image/container +# +sub genmeta_product { + my ($ctx, $p, $edeps) = @_; + my @new_meta; + for my $bin (@$edeps) { + die("bad binary in genmeta_product (not a hash)\n") unless ref($bin) eq 'HASH'; + my $name = $bin->{'filename'}; + my $package = $bin->{'packid'}; + if (!$package) { + die("bad binary in genmeta_product (bad location)\n") unless $bin->{'location'} =~ /\/([^\/]+)\/([^\/]+)$/; + $package = $1; + $name = $2; + } + if ($bin->{'hdrmd5'}) { + push @new_meta, "$bin->{'hdrmd5'} $package/$name"; + } elsif ($bin->{'md5sum'}) { + push @new_meta, "$bin->{'md5sum'} $package/$name"; + } else { + die("bad binary in genmeta_product (no hrdmd5/md5sum)\n"); + } + } + @new_meta = sort {substr($a, 34) cmp substr($b, 34) || $a cmp $b} @new_meta; + unshift @new_meta, ($p->{'verifymd5'} || $p->{'srcmd5'})." $p->{'pkg'}"; + return \@new_meta; +} + # # Generate the dependency tracking data for a package # @@ -308,6 +337,7 @@ sub genmeta { my ($ctx, $p, $edeps, $hdeps) = @_; my $buildtype = $p->{'buildtype'}; return genmeta_image($ctx, $p, $edeps) if $buildtype eq 'kiwi' || $buildtype eq 'docker' || $buildtype eq 'preinstallimage'; + return genmeta_product($ctx, $p, $edeps) if $buildtype eq 'productcompose'; my $dep2pkg = $p->{'native'} ? $ctx->{'dep2pkg_host'} : $ctx->{'dep2pkg'}; my $metacache = $ctx->{'metacache'}; my @new_meta; @@ -345,21 +375,8 @@ sub genmeta { return \@new_meta; } -# -# Check the status of a single image/container -# -sub check_image { - my ($ctx, $p) = @_; - my $edeps = $p->{'dep_expanded'} || []; - my $notready = $ctx->{'notready'}; - my $dep2src = $ctx->{'dep2src'}; - my @blocked = grep {$notready->{$dep2src->{$_}}} @$edeps; - @blocked = () if $ctx->{'block'} && $ctx->{'block'} eq 'never'; - if (@blocked) { - splice(@blocked, 10, scalar(@blocked), '...') if @blocked > 10; - return ('blocked', join(', ', @blocked)); - } - my $new_meta = genmeta($ctx, $p, $edeps); +sub check_meta { + my ($ctx, $p, $new_meta, @data) = @_; my $packid = $p->{'pkg'}; my $dst = "$ctx->{'builddir'}/$packid"; my @meta; @@ -369,21 +386,137 @@ sub check_image { close $mfp; chomp @meta; } - return ('scheduled', [ { 'explain' => 'new build' } ]) if !@meta; - return ('scheduled', [ { 'explain' => 'source change', 'oldsource' => substr($meta[0], 0, 32) } ]) if $meta[0] ne $new_meta->[0]; - return ('scheduled', [ { 'explain' => 'forced rebuild' } ]) if $p->{'force_rebuild'}; + return ('scheduled', [ { 'explain' => 'new build' }, @data ]) if !@meta; + return ('scheduled', [ { 'explain' => 'source change', 'oldsource' => substr($meta[0], 0, 32) }, @data ]) if $meta[0] ne $new_meta->[0]; + return ('scheduled', [ { 'explain' => 'forced rebuild' }, @data ]) if $p->{'force_rebuild'}; my $rebuildmethod = $ctx->{'rebuild'} || 'transitive'; if ($rebuildmethod eq 'local') { - return ('scheduled', [ { 'explain' => 'rebuild counter sync' } ]) if $ctx->{'relsynctrigger'}->{$packid}; + return ('scheduled', [ { 'explain' => 'rebuild counter sync' }, @data ]) if $ctx->{'relsynctrigger'}->{$packid}; return ('done'); } if (@meta == @$new_meta && join('\n', @meta) eq join('\n', @$new_meta)) { - return ('scheduled', [ { 'explain' => 'rebuild counter sync' } ]) if $ctx->{'relsynctrigger'}->{$packid}; + return ('scheduled', [ { 'explain' => 'rebuild counter sync' }, @data ]) if $ctx->{'relsynctrigger'}->{$packid}; return ('done'); } my @diff = PBuild::Meta::diffsortedmd5(\@meta, $new_meta); my $reason = PBuild::Meta::sortedmd5toreason(@diff); - return ('scheduled', [ { 'explain' => 'meta change', 'packagechange' => $reason } ] ); + return ('scheduled', [ { 'explain' => 'meta change', 'packagechange' => $reason }, @data ] ); +} + +# +# Check the status of a single image/container +# +sub check_image { + my ($ctx, $p) = @_; + my $edeps = $p->{'dep_expanded'} || []; + my $notready = $ctx->{'notready'}; + my $dep2src = $ctx->{'dep2src'}; + my @blocked = grep {$notready->{$dep2src->{$_}}} @$edeps; + @blocked = () if $ctx->{'block'} && $ctx->{'block'} eq 'never'; + if (@blocked) { + splice(@blocked, 10, scalar(@blocked), '...') if @blocked > 10; + return ('blocked', join(', ', @blocked)); + } + my $new_meta = genmeta($ctx, $p, $edeps); + return check_meta($ctx, $p, $new_meta); +} + +sub check_product { + my ($ctx, $p) = @_; + my $notready = $ctx->{'notready'}; + my $dep2src = $ctx->{'dep2src'}; + my @deps = @{$p->{'dep'} || []}; + my %deps = map {$_ => 1} @deps; + my $versioned_deps; + for (grep {/[<=>]/} @deps) { + next unless /^(.*?)\s*([<=>].*)$/; + $deps{$1} = $2; + delete $deps{$_}; + $versioned_deps = 1; + } + delete $deps{''}; + delete $deps{"-$_"} for grep {!/^-/} keys %deps; + my $allpacks = $deps{'*'} ? 1 : 0; + my $nodbgpkgs = $p->{'nodbgpkgs'}; + my $nosrcpkgs = $p->{'nosrcpkgs'}; + my %unneeded_na; + + my @rpms; + for my $repo (@{$ctx->{'repos'}}) { + my %seen_fn; # resolve file conflicts in this prp + my $gbininfo; + my @next_unneeded_na; + $gbininfo = $ctx->{'repomgr'}->get_gbininfo($repo); + my @apackids = sort keys %$gbininfo; + for my $apackid (@apackids) { + next if $apackid eq '_volatile'; + my $bininfo = $gbininfo->{$apackid}; + next unless $bininfo; + my $needit; + for my $fn (keys %$bininfo) { + next unless $fn =~ /^(?:::import::.*::)?(.+)-(?:[^-]+)-(?:[^-]+)\.([a-zA-Z][^\.\-]*)\.rpm$/; + my ($bn, $ba) = ($1, $2); + next if $ba eq 'src' || $ba eq 'nosrc'; # always unneeded + my $na = "$bn.$ba"; + next if $unneeded_na{$na}; + if ($fn =~ /-(?:debuginfo|debugsource)-/) { + if ($nodbgpkgs || !$deps{$bn}) { + $unneeded_na{$na} = 1; + next; + } + } + next if $seen_fn{$fn}; + if ($fn =~ /^::import::(.*?)::(.*)$/) { + next if $seen_fn{$2}; + } + my $d = $deps{$bn}; + if (!($d || ($allpacks && !$deps{"-$bn"}))) { + $unneeded_na{$na} = 1; # cache unneeded + next; + } + if ($d && $d ne '1') { + my $bi = $bininfo->{$fn}; + my $evr = "$bi->{'version'}-$bi->{'release'}"; + $evr = "$bi->{'epoch'}:$evr" if $bi->{'epoch'}; + next unless Build::matchsingledep("$bn=$evr", "$bn$d", 'rpm'); + } + $needit = 1; + last; + } + next unless $needit; + # we need the package, add all artifacts + my @bi = sort(keys %$bininfo); + my @ibi = grep {/^::import::/} @bi; + if (@ibi) { + @bi = grep {!/^::import::/} @bi; + push @bi, @ibi; + } + for my $fn (@bi) { + next unless $fn =~ /^(?:::import::.*::)?(.+)-(?:[^-]+)-(?:[^-]+)\.([a-zA-Z][^\.\-]*)\.rpm$/; + my ($bn, $ba) = ($1, $2); + next if $nosrcpkgs && ($ba eq 'src' || $ba eq 'nosrc'); + next if $nodbgpkgs && $fn =~ /-(?:debuginfo|debugsource)-/; + my $na = "$bn.$ba"; + # ignore if we already have this file + next if $seen_fn{$fn}; + next if $fn =~ /^::import::(.*?)::(.*)$/ && $seen_fn{$2}; + my $b = $bininfo->{$fn}; + push @rpms, { %{$bininfo->{$fn}}, 'package' => $apackid }; + $seen_fn{$fn} = 1; + push @next_unneeded_na, $na unless $ba eq 'src' || $ba eq 'nosrc'; + } + for my $fn (@bi) { + next unless ($fn =~ /[-.]appdata\.xml$/) || $fn eq '_modulemd.yaml' || $fn eq 'updateinfo.xml'; + next if $seen_fn{$fn}; + push @rpms, { %{$bininfo->{$fn}}, 'package' => $apackid }; + $seen_fn{$fn} = 1 unless $fn eq 'updateinfo.xml' || $fn eq '_modulemd.yaml'; # we expect those to be renamed + } + } + @next_unneeded_na = () if $deps{'--use-newest-package'}; + $unneeded_na{$_} = 1 for @next_unneeded_na; + } + my $new_meta = genmeta($ctx, $p, \@rpms); + return check_meta($ctx, $p, $new_meta, undef, \@rpms); } # @@ -394,6 +527,7 @@ sub check { my $buildtype = $p->{'buildtype'}; return check_image($ctx, $p) if $buildtype eq 'kiwi' || $buildtype eq 'docker' || $buildtype eq 'preinstallimage'; + return check_product($ctx, $p) if $buildtype eq 'productcompose'; my $packid = $p->{'pkg'}; my $notready = $ctx->{'notready'}; @@ -585,6 +719,7 @@ sub dep2bins { my ($ctx, @deps) = @_; my $dep2pkg = $ctx->{'dep2pkg'}; for (@deps) { + next if ref($_) eq 'HASH'; # already a binary reference (used for product builds) my $q = $dep2pkg->{$_}; die("unknown binary $_\n") unless $q; $_ = $q; @@ -621,8 +756,9 @@ sub build { my $bconf = $bconf_host || $ctx->{'bconf'}; my $buildtype = $p->{'buildtype'}; $buildtype = 'kiwi-image' if $buildtype eq 'kiwi'; + $edeps = $data->[2] if $buildtype eq 'productcompose'; my $kiwimode; - $kiwimode = $buildtype if $buildtype eq 'kiwi-image' || $buildtype eq 'kiwi-product' || $buildtype eq 'docker' || $buildtype eq 'fissile'; + $kiwimode = $buildtype if $buildtype eq 'kiwi-image' || $buildtype eq 'kiwi-product' || $buildtype eq 'docker' || $buildtype eq 'fissile' || $buildtype eq 'productcompose'; if ($p->{'buildtimeservice'}) { for my $service (@{$p->{'buildtimeservice'} || []}) { @@ -699,7 +835,11 @@ sub build { return ('recheck', 'assets changed') if $p->{'srcmd5'} ne $oldsrcmd5; return ('broken', $p->{'error'}) if $p->{'error'}; # missing assets my $bins; - if ($kiwimode && $bconf_host) { + if ($kiwimode && $kiwimode eq 'productcompose') { + $bins = dep2bins_host($ctx, PBuild::Util::unify(@pdeps, @vmdeps, @sysdeps)); + push @$bins, @{dep2bins($ctx, PBuild::Util::unify(@$tdeps))} if $tdeps; + $ctx->{'repomgr'}->getremoteproductbinaries(\@bdeps); + } elsif ($kiwimode && $bconf_host) { $bins = dep2bins_host($ctx, PBuild::Util::unify(@pdeps, @vmdeps, @sysdeps)); push @$bins, @{dep2bins($ctx, PBuild::Util::unify(@bdeps))}; } else { diff --git a/PBuild/Container.pm b/PBuild/Container.pm index e63cf61d0..796e89212 100644 --- a/PBuild/Container.pm +++ b/PBuild/Container.pm @@ -52,7 +52,7 @@ sub containerinfo2obsbinlnk { for my $tag (@{$d->{tags}}) { push @{$lnk->{'provides'}}, "container:$tag" unless "container:$tag" eq $lnk->{'name'}; } - eval { PBuild::Verify::verify_nevraquery($lnk) }; + eval { PBuild::Verify::verify_nevraquery($lnk); PBuild::Verify::verify_filename($d->{'file'}) }; return undef if $@; local *F; if ($d->{'tar_md5sum'}) { diff --git a/PBuild/Expand.pm b/PBuild/Expand.pm index fa4d4f1e7..20dba709c 100644 --- a/PBuild/Expand.pm +++ b/PBuild/Expand.pm @@ -78,6 +78,14 @@ sub configure_repos { return \%packs; } +# +# expand dependencies of a single package (product case) +# +sub expand_deps_product { + my ($p, $bconf, $subpacks, $cross) = @_; + $p->{'dep_expanded'} = []; # we don't expand anything for products +} + # # expand dependencies of a single package (image case) # @@ -106,6 +114,7 @@ sub expand_deps_image { sub expand_deps { my ($p, $bconf, $subpacks, $cross) = @_; my $buildtype = $p->{'buildtype'} || ''; + return expand_deps_product($p, $bconf, $subpacks, $cross) if $buildtype eq 'productcompose'; return expand_deps_image($p, $bconf, $subpacks, $cross) if $buildtype eq 'kiwi' || $buildtype eq 'docker' || $buildtype eq 'fissile' || $buildtype eq 'preinstallimage'; delete $p->{'dep_experror'}; if ($p->{'error'}) { diff --git a/PBuild/Job.pm b/PBuild/Job.pm index 5a99a9452..0b0b9f833 100644 --- a/PBuild/Job.pm +++ b/PBuild/Job.pm @@ -130,12 +130,17 @@ sub collect_result { @d = ('DOCKER') if $p->{'recipe'} =~ /Dockerfile(\.|$)/; @d = ('FISSILE') if $p->{'recipe'} =~ /fissile\.yml$/; @d = ('HELM') if $p->{'recipe'} =~ /Chart\.yaml$/; + @d = ('PRODUCT') if $p->{'recipe'} =~ /\.productcompose$/; push @d, 'OTHER'; my @send; for my $d ('.', @d) { my @files = sort(PBuild::Util::ls("$buildroot/.build.packages/$d")); @files = grep {$_ ne 'same_result_marker' && $_ ne '.kiwitree'} @files; - @files = grep {! -l "$buildroot/.build.packages/$d/$_" && -f _} @files; + if ($p->{'buildtype'} eq 'productcompose' && $d eq 'PRODUCT') { + @files = grep {! -l "$buildroot/.build.packages/$d/$_" && (-d _ || -f _)} @files; + } else { + @files = grep {! -l "$buildroot/.build.packages/$d/$_" && -f _} @files; + } push @send, map {"$buildroot/.build.packages/$d/$_"} @files; } my %send = map {(split('/', $_))[-1] => $_} @send; @@ -207,7 +212,7 @@ sub createjob { my %sysdeps = map {$_ => 1} @$sysdeps; my @alldeps; - my $kiwimode = $p->{'buildtype'} eq 'kiwi' || $p->{'buildtype'} eq 'docker' || $p->{'buildtype'} eq 'fissile' ? $p->{'buildtype'} : undef; + my $kiwimode = $p->{'buildtype'} eq 'kiwi' || $p->{'buildtype'} eq 'docker' || $p->{'buildtype'} eq 'fissile' || $p->{'buildtype'} eq 'productcompose' ? $p->{'buildtype'} : undef; if ($kiwimode) { @alldeps = PBuild::Util::unify(@$pdeps, @$vmdeps, @$sysdeps); } else { @@ -338,7 +343,7 @@ sub createjob { if ($kiwimode) { # now setup the repos/containers directories $ctx->{'repomgr'}->copyimagebinaries($ctx->dep2bins(@$bdeps), $srcdir); - # tell kiwi how to use them + # and tell kiwi how to use them if ($p->{'buildtype'} eq 'kiwi') { my @kiwiargs; push @kiwiargs, '--ignore-repos'; diff --git a/PBuild/LocalRepo.pm b/PBuild/LocalRepo.pm index 312499d2d..76bf62f94 100644 --- a/PBuild/LocalRepo.pm +++ b/PBuild/LocalRepo.pm @@ -189,4 +189,12 @@ sub cleanup_builddir { } } +# +# Return the artifact data of all built packages +# +sub get_gbininfo { + my ($builddir) = @_; + return PBuild::Util::retrieve("$builddir/.pbuild/_bininfo"); +} + 1; diff --git a/PBuild/OBS.pm b/PBuild/OBS.pm index 96555f5e2..871b7ce5c 100644 --- a/PBuild/OBS.pm +++ b/PBuild/OBS.pm @@ -23,8 +23,10 @@ package PBuild::OBS; use strict; use Build::Download; +use Build::Rpm; use PBuild::Util; +use PBuild::Verify; use PBuild::Cpio; use PBuild::Structured; use PBuild::SigAuth; @@ -79,6 +81,26 @@ my $dtd_proj = [ [ $dtd_repo ], ]; +my $dtd_packagebinaryversionlist = [ + 'packagebinaryversionlist' => + [[ 'binaryversionlist' => + 'package', + 'code', + [[ 'binary' => + 'name', + 'sizek', + 'error', + 'hdrmd5', + 'metamd5', + 'leadsigmd5', + 'md5sum', + 'evr', + 'arch', + ]], + ]], +]; + + # # set the cookie jar for the user agent # @@ -349,4 +371,108 @@ sub fetch_repodata { return \@bins; } +sub fetch_gbininfo { + my ($url, $arch, $opts) = @_; + die("bad obs: reference\n") unless $url =~ /^obs:\/{1,3}([^\/]+\/[^\/]+)(?:\/([^\/]*))?$/; + my $prp = $1; + $arch = $2 if $2; + die("please specify the build service url with the --obs option\n") unless $opts->{'obs'}; + my $baseurl = $opts->{'obs'}; + $baseurl .= '/' unless $baseurl =~ /\/$/; + my $requrl .= "${baseurl}build/$prp/$arch?view=binaryversions"; + print "fetching package artifact data for OBS repo $prp\n"; + my $ua = create_ua(); + my ($data) = Build::Download::fetch($requrl, 'ua' => $ua, 'retry' => 3); + my $packagebinaryversionlist = PBuild::Structured::fromxml($data, $dtd_packagebinaryversionlist, 0, 1); + my $gbininfo = {}; + for my $binaryversionlist (@{$packagebinaryversionlist->{'binaryversionlist'} || []}) { + my $location = "${baseurl}build/$prp/$arch/$binaryversionlist->{'package'}/"; + my %bins; + for my $binary (@{$binaryversionlist->{'binary'} || []}) { + my $filename = $binary->{'name'}; + my $bin; + # XXX: should not rely on the filename here! + if ($filename =~ /^(?:::import::.*::)?(.+)-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.rpm$/) { + $bin = {'name' => $1, 'arch' => $2}; + } elsif ($filename =~ /^([^\/]+)_[^\/]*_([^\/]*)\.deb$/) { + $bin = {'name' => $1, 'arch' => $2}; + } elsif ($filename =~ /^([^\/]+)-[^-]+-[^-]+-([a-zA-Z][^\/\.\-]*)\.pkg\.tar\.(?:gz|xz|zst)$/) { + $bin = {'name' => $1, 'arch' => $2}; + } elsif ($filename eq '.nouseforbuild') { + $bin = {}; + } else { + $bin = {}; + } + $bin->{'hdrmd5'} = $binary->{'hdrmd5'} if $binary->{'hdrmd5'}; + $bin->{'leadsigmd5'} = $binary->{'leadsigmd5'} if $binary->{'leadsigmd5'}; + $bin->{'md5sum'} = $binary->{'md5sum'} if $binary->{'md5sum'}; + $bin->{'location'} = $location . $filename; + $bins{$filename} = $bin; + } + $gbininfo->{$binaryversionlist->{'package'}} = \%bins; + } + return $gbininfo; +} + +sub fetch_productbinaries_cpioextract { + my ($ent, $xfile, $repodir, $packid, $files) = @_; + return undef unless $ent->{'cpiotype'} == 8; + my $name = $ent->{'name'}; + PBuild::Verify::verify_filename("$packid-$name"); + my $tmpname = "$repodir/.$$.$packid-$name"; + if (!defined($xfile)) { + return undef unless $files->{$name}; + return $tmpname; # ok, extract this one! + } + my $bin = $files->{$name}; + die unless $bin; + if ($bin->{'md5sum'}) { + Build::Download::checkfiledigest($tmpname, "md5:$bin->{'md5sum'}"); + } elsif ($bin->{'hdrmd5'} || $bin->{'leadsigmd5'}) { + my $leadsigmd5; + my $hdrmd5 = Build::Rpm::queryhdrmd5($tmpname, \$leadsigmd5); + die("downloaded binary does not match hdrmd5\n") if $bin->{'hdrmd5'} && $bin->{'hdrmd5'} ne ($hdrmd5 || ''); + die("downloaded binary does not match leadsigmd5\n") if $bin->{'leadsigmd5'} && $bin->{'leadsigmd5'} ne ($leadsigmd5 || ''); + } + rename($tmpname, "$repodir/_gbins/$packid-$name") || die("rename $tmpname $repodir/_gbins/$packid-$name: $!\n"); + die unless ($bin->{'package'} || '') eq $packid; + $bin->{'filename'} = "$packid-$name"; + return undef; # continue extracting +} + +sub fetch_productbinaries { + my ($url, $repodir, $bins) = @_; + # group by package + my %packages; + my $location; + for my $bin (@$bins) { + my $l = $bin->{'location'}; + die("fetch_productbinaries: missing location\n") unless $l; + die("fetch_productbinaries: bad location $l\n") unless $l =~ /^(.+)\/([^\/]+)\/([^\/]+)$/; + die("fetch_productbinaries: package mismatch$l\n") unless $2 eq $bin->{'package'}; + $location = $1 unless defined $location; + die("fetch_productbinaries: location conflict\n") unless $1 eq $location; + push @{$packages{$2}}, [ $3, $bin ]; + } + PBuild::Util::mkdir_p("$repodir/_gbins"); + my $ua = create_ua(); + for my $packid (sort keys %packages) { + #print "downloading ".scalar(@{$packages{$packid}}). " artifacts from $packid\n"; + my $tmpcpio = "$repodir/.$$.binaries.cpio"; + my %files; + my $requrl .= "$location/$packid?view=cpio"; + for (@{$packages{$packid}}) { + my $fn = $_->[0]; + $fn =~ s/([\000-\037<>;\"#\?&\+=%[\177-\377])/sprintf("%%%02X",ord($1))/sge; + $fn =~ tr/ /+/; + $requrl .= "&binary=$fn"; + $files{$_->[0]} = $_->[1]; + } + die unless %files; + Build::Download::download($requrl, $tmpcpio, undef, 'ua' => $ua, 'retry' => 3); + PBuild::Cpio::cpio_extract($tmpcpio, sub {fetch_productbinaries_cpioextract($_[0], $_[1], $repodir, $packid, \%files)}); + unlink($tmpcpio); + } +} + 1; diff --git a/PBuild/RemoteRepo.pm b/PBuild/RemoteRepo.pm index 71ed6a5a8..6a0a5fbf0 100644 --- a/PBuild/RemoteRepo.pm +++ b/PBuild/RemoteRepo.pm @@ -246,11 +246,17 @@ sub calc_binname { # Replace already downloaded entries in the metadata # sub replace_with_local { - my ($repodir, $bins) = @_; + my ($repodir, $bins, $oldbins) = @_; + my %oldbins; + for (@{$oldbins || []}) { + $oldbins{$_->{'filename'}} = $_ if $_->{'filename'}; + } my $bad; my %files = map {$_ => 1} PBuild::Util::ls($repodir); delete $files{'_metadata'}; delete $files{'.tmp'}; + delete $files{'_gbins'}; + delete $files{'_gbininfo'}; for my $bin (@$bins) { next if $bin->{'name'} eq 'moduleinfo:'; my $file = $bin->{'filename'}; @@ -269,6 +275,16 @@ sub replace_with_local { $bin->{'filename'} = $file; next; } + my $oldbin = $oldbins{$file}; + if ($oldbin) { + if ($bin->{'hdrmd5'} && $bin->{'hdrmd5'} eq ($oldbin->{'hdrmd5'} || '')) { + if (!$bin->{'leadsigmd5'} || $bin->{'leadsigmd5'} eq ($oldbin->{'leadsigmd5'} || '')) { + %$bin = %$oldbin; + $files{$file} = 2; + next; + } + } + } eval { my $q = querybinary($repodir, $file); %$bin = %$q; @@ -276,7 +292,7 @@ sub replace_with_local { }; if ($@) { warn($@); - unlink($file); + unlink("$repodir/$file"); } } for my $file (grep {$files{$_} == 1} sort keys %files) { @@ -364,7 +380,7 @@ sub fetchrepo { } $repo = { 'bins' => $repo } if $repo && ref($repo) ne 'HASH'; die unless $repo && $repo->{'bins'}; - replace_with_local($repodir, $repo->{'bins'}) unless $repo == $oldrepo; + replace_with_local($repodir, $repo->{'bins'}, $oldrepo ? $oldrepo->{'bins'} : undef) unless $oldrepo && $repo == $oldrepo; PBuild::Util::store("$repodir/._metadata.$$", $repofile, $repo); return $repo->{'bins'}; } @@ -494,4 +510,92 @@ sub fetchbinaries { PBuild::Util::store("$repodir/._metadata.$$", "$repodir/_metadata", $repo->{'bins'}); } +sub fetchproductbinaries { + my ($repo, $bins) = @_; + my $repodir = $repo->{'dir'}; + my $url = $repo->{'url'}; + die("bad repo\n") unless $url; + print "fetching ".PBuild::Util::plural(scalar(@$bins), 'product binary')." from $url\n"; + die("unsupported url $url\n") unless $url =~ /^obs:/; + PBuild::OBS::fetch_productbinaries($url, $repodir, $bins); +} + +sub replace_with_local_gbininfo { + my ($repodir, $gbininfo, $oldgbininfo) = @_; + my %files = map {$_ => 1} PBuild::Util::ls("$repodir/_gbins"); + for my $packid (sort keys %$gbininfo) { + my $bins = $gbininfo->{$packid}; + my $oldbins = $oldgbininfo ? $oldgbininfo->{$packid} : undef; + for my $name (sort keys %$bins) { + my $bin = $bins->{$name}; + my $file = $bin->{'filename'}; + if (defined $file) { + if ($files{$file}) { + $files{$file} = 2; + } else { + delete $bin->{'filename'}; + } + next; + } + $file = "$packid-$name"; + next unless $files{$file}; + my $oldbin = $oldbins ? $oldbins->{$name} : undef; + if ($oldbin && $oldbin->{'filename'} eq $file) { + if ($bin->{'md5sum'} && $bin->{'md5sum'} eq ($oldbin->{'md5sum'} || '')) { + %$bin = %$oldbin; + $files{$file} = 2; + next; + } + if ($bin->{'hdrmd5'} && $bin->{'hdrmd5'} eq ($oldbin->{'hdrmd5'} || '')) { + if (!$bin->{'leadsigmd5'} || $bin->{'leadsigmd5'} eq ($oldbin->{'leadsigmd5'} || '')) { + %$bin = %$oldbin; + $files{$file} = 2; + next; + } + } + } + if ($bin->{'md5sum'}) { + eval { Build::Download::checkfiledigest("$repodir/_gbins/$file", "md5:$bin->{'md5sum'}") }; + if (!$@) { + $bin->{'filename'} = $file; + $files{$file} = 2; + next; + } + } elsif ($bin->{'hdrmd5'}) { + my ($hdrmd5, $leadsigmd5); + eval { $hdrmd5 = Build::queryhdrmd5("$repodir/_gbins/$file", \$leadsigmd5) }; + if ($@) { + warn($@); + unlink("$repodir/_gbins/$file"); + next; + } + if ($bin->{'hdrmd5'} eq ($hdrmd5 || '')) { + if (!$bin->{'leadsigmd5'} || $bin->{'leadsigmd5'} eq ($leadsigmd5 || '')) { + $bin->{'filename'} = $file; + $files{$file} = 2; + next; + } + } + } + } + } + for my $file (grep {$files{$_} == 1} sort keys %files) { + unlink("$repodir/_gbins/$file"); + } +} + + +sub get_gbininfo { + my ($repo) = @_; + my $url = $repo->{'url'}; + die("bad repo\n") unless $url; + die("get_gbininfo is not supported for $url\n") unless $url =~ /^obs:/; + my $repodir = $repo->{'dir'}; + my $oldgbininfo = PBuild::Util::retrieve("$repodir/_gbininfo", 1); + my $gbininfo = PBuild::OBS::fetch_gbininfo($url, $repo->{'arch'}, { 'obs' => $repo->{'obs'} }); + replace_with_local_gbininfo($repodir, $gbininfo, $oldgbininfo); + PBuild::Util::store("$repodir/._gbininfo.$$", "$repodir/_gbininfo", $gbininfo); + return $gbininfo; +} + 1; diff --git a/PBuild/RepoMgr.pm b/PBuild/RepoMgr.pm index 0048b75d1..36b7cb1e7 100644 --- a/PBuild/RepoMgr.pm +++ b/PBuild/RepoMgr.pm @@ -46,6 +46,7 @@ sub addremoterepo { my $bins = PBuild::RemoteRepo::fetchrepo($bconf, $myarch, $repodir, $repourl, $buildtype, $opts); $_->{'repoid'} = $id for @$bins; my $repo = { 'dir' => $repodir, 'bins' => $bins, 'url' => $repourl, 'arch' => $myarch, 'type' => 'repo', 'repoid' => $id }; + $repo->{'obs'} = $opts->{'obs'} if $repourl =~ /^obs:/; $repos->{$id} = $repo; return $repo; } @@ -82,7 +83,7 @@ sub addlocalrepo { } # -# Add an emptt repository to the manager +# Add an empty repository to the manager # sub addemptyrepo { my ($repos) = @_; @@ -123,7 +124,25 @@ sub getremotebinaries { } elsif ($repo->{'type'} eq 'registry') { PBuild::RemoteRegistry::fetchbinaries($repo, $tofetch{$repoid}); } else { - die("unknown repo type $repo->{'type'}\n"); + die("unsupported repo type $repo->{'type'}\n"); + } + $_->{'repoid'} = $repoid for @{$tofetch{$repoid}}; + } +} + +sub getremoteproductbinaries { + my ($repos, $bins) = @_; + my %tofetch; + for my $q (@$bins) { + push @{$tofetch{$q->{'repoid'}}}, $q unless $q->{'filename'}; + } + for my $repoid (sort {$a cmp $b} keys %tofetch) { + my $repo = $repos->{$repoid}; + die("bad repoid $repoid\n") unless $repo; + if ($repo->{'type'} eq 'repo') { + PBuild::RemoteRepo::fetchproductbinaries($repo, $tofetch{$repoid}); + } else { + die("unsupported repo type $repo->{'type'}\n"); } $_->{'repoid'} = $repoid for @{$tofetch{$repoid}}; } @@ -135,6 +154,7 @@ sub getremotebinaries { sub copyimagebinaries { my ($repos, $bins, $dstdir) = @_; PBuild::Util::mkdir_p("$dstdir/repos/pbuild/pbuild"); + my %provenance; for my $q (@$bins) { my $repo = $repos->{$q->{'repoid'}}; die("package $q->{'name'} has no repo\n") unless $repo; @@ -155,13 +175,29 @@ sub copyimagebinaries { PBuild::RemoteRegistry::construct_containertar($repo->{'dir'}, $q, $to); next; } - die("package $q->{'name'} is not available\n") unless $q->{'filename'}; - PBuild::Verify::verify_filename($q->{'filename'}); - my $from = "$repo->{'dir'}/$q->{'filename'}"; - $from = "$repo->{'dir'}/$q->{'packid'}/$q->{'filename'}" if $q->{'packid'}; + my $filename = $q->{'filename'}; + die("package $q->{'name'} is not available\n") unless $filename; + if (!$q->{'packid'} && $q->{'package'}) { + PBuild::Verify::verify_filename($filename); + my $from = "$repo->{'dir'}/_gbins/$filename"; + $filename =~ s/^\Q$q->{'package'}-\E// if $filename =~ /\.rpm$/ || $filename =~ /\.slsa_provenance\.json$/; + $to = "$dstdir/repos/pbuild/pbuild/$filename"; + $provenance{"$1.slsa_provenance.json"} = "$dstdir/repos/pbuild/pbuild/$q->{'package'}-_slsa_provenance.json" if $to =~ /(.*)\.rpm$/; + PBuild::Util::cp($from, $to); + next; + } + PBuild::Verify::verify_filename($filename); + my $from = "$repo->{'dir'}/$filename"; + $from = "$repo->{'dir'}/$q->{'packid'}/$filename" if $q->{'packid'}; $from = "$repo->{'dir'}/$q->{'packid'}/$q->{'lnk'}" if $q->{'packid'} && $q->{'lnk'}; # obsbinlnk PBuild::Util::cp($from, $to); } + # create provenance links + for my $p (sort keys %provenance) { + if (-e $provenance{$p} && ! -e "$p") { + link($provenance{$p}, $p) || die("link $provenance{$p} $p: $!\n"); + } + } } # @@ -183,4 +219,26 @@ sub getbinarylocations { return \%locations; } +# +# Return gbininfo data for a repository +# +sub get_gbininfo { + my ($repos, $repo) = @_; + return $repo->{'gbininfo'} if $repo->{'gbininfo'}; + my $gbininfo; + if ($repo->{'type'} eq 'local') { + $gbininfo = PBuild::LocalRepo::get_gbininfo($repo->{'dir'}); + } elsif ($repo->{'type'} eq 'repo') { + $gbininfo = PBuild::RemoteRepo::get_gbininfo($repo); + } else { + die("get_gbininfo: unsupported repo type '$repo->{'type'}'\n"); + } + my $id = $repo->{'repoid'}; + for my $p (values %$gbininfo) { + $_->{'repoid'} = $id for values %$p; + } + $repo->{'gbininfo'} = $gbininfo; + return $gbininfo; +} + 1; diff --git a/PBuild/Util.pm b/PBuild/Util.pm index 83a55b083..38f5f3b5f 100644 --- a/PBuild/Util.pm +++ b/PBuild/Util.pm @@ -194,6 +194,17 @@ sub retrieve { return $dd; } +sub fromstorable { + my $nonfatal = $_[1]; + return Storable::thaw(substr($_[0], 4)) unless $nonfatal; + my $d = eval { Storable::thaw(substr($_[0], 4)) }; + if ($@) { + warn($@) if $nonfatal == 2; + return undef; + } + return $d; +} + sub identical { my ($d1, $d2, $except, $subexcept) = @_;