Skip to content

Commit

Permalink
pbuild: Add support for the productcompose build type
Browse files Browse the repository at this point in the history
  • Loading branch information
@mlschroe
mlschroe committed Jan 24, 2024
1 parent f6bd18c commit 3bae6e4
Show file tree
Hide file tree
Showing 10 changed files with 502 additions and 37 deletions.
6 changes: 5 additions & 1 deletion PBuild/BuildResult.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ sub integrate_build_result {
next if $file eq '_meta' || $file eq '_meta.success' || $file eq '_meta.fail';
next if $file eq '_log' || $file eq '_log.success';
next if $file eq '_repository';
unlink("$dst/$file");
unlink("$dst/$file") || PBuild::Util::rm_rf("$dst/$file");
}
# copy new stuff over
for my $file (sort keys %$result) {
Expand All @@ -126,6 +126,10 @@ sub integrate_build_result {
my $obsbinlnk = PBuild::Container::containerinfo2obsbinlnk($1, $2, $p->{'pkg'});
PBuild::Util::store("$dst/$prefix.obsbinlnk", undef, $obsbinlnk) if $obsbinlnk;
}
if ($p->{'buildtype'} eq 'productcompose' && -d $result->{$file}) {
PBuild::Util::cp_a($result->{$file}, "$dst/$file");
next;
}
PBuild::Util::cp($result->{$file}, "$dst/$file");
}
# create new bininfo
Expand Down
186 changes: 163 additions & 23 deletions PBuild/Checker.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ sub prepare {
$ctx->{'dep2pkg'} = $dep2pkg;
$ctx->{'subpacks'} = \%subpacks;
PBuild::Meta::setgenmetaalgo($ctx->{'genmetaalgo'});
$ctx->{'repos'} = $repos;
$ctx->{'dep2pkg_host'} = PBuild::Expand::configure_repos($ctx->{'bconf_host'}, $hostrepos) if $ctx->{'bconf_host'};
}

Expand Down Expand Up @@ -301,13 +302,42 @@ sub genmeta_image {
return \@new_meta;
}

#
# Generate the dependency tracking data for a image/container
#
sub genmeta_product {
my ($ctx, $p, $edeps) = @_;
my @new_meta;
for my $bin (@$edeps) {
die("bad binary in genmeta_product (not a hash)\n") unless ref($bin) eq 'HASH';
my $name = $bin->{'filename'};
my $package = $bin->{'packid'};
if (!$package) {
die("bad binary in genmeta_product (bad location)\n") unless $bin->{'location'} =~ /\/([^\/]+)\/([^\/]+)$/;
$package = $1;
$name = $2;
}
if ($bin->{'hdrmd5'}) {
push @new_meta, "$bin->{'hdrmd5'} $package/$name";
} elsif ($bin->{'md5sum'}) {
push @new_meta, "$bin->{'md5sum'} $package/$name";
} else {
die("bad binary in genmeta_product (no hrdmd5/md5sum)\n");
}
}
@new_meta = sort {substr($a, 34) cmp substr($b, 34) || $a cmp $b} @new_meta;
unshift @new_meta, ($p->{'verifymd5'} || $p->{'srcmd5'})." $p->{'pkg'}";
return \@new_meta;
}

#
# Generate the dependency tracking data for a package
#
sub genmeta {
my ($ctx, $p, $edeps, $hdeps) = @_;
my $buildtype = $p->{'buildtype'};
return genmeta_image($ctx, $p, $edeps) if $buildtype eq 'kiwi' || $buildtype eq 'docker' || $buildtype eq 'preinstallimage';
return genmeta_product($ctx, $p, $edeps) if $buildtype eq 'productcompose';
my $dep2pkg = $p->{'native'} ? $ctx->{'dep2pkg_host'} : $ctx->{'dep2pkg'};
my $metacache = $ctx->{'metacache'};
my @new_meta;
Expand Down Expand Up @@ -345,21 +375,8 @@ sub genmeta {
return \@new_meta;
}

#
# Check the status of a single image/container
#
sub check_image {
my ($ctx, $p) = @_;
my $edeps = $p->{'dep_expanded'} || [];
my $notready = $ctx->{'notready'};
my $dep2src = $ctx->{'dep2src'};
my @blocked = grep {$notready->{$dep2src->{$_}}} @$edeps;
@blocked = () if $ctx->{'block'} && $ctx->{'block'} eq 'never';
if (@blocked) {
splice(@blocked, 10, scalar(@blocked), '...') if @blocked > 10;
return ('blocked', join(', ', @blocked));
}
my $new_meta = genmeta($ctx, $p, $edeps);
sub check_meta {
my ($ctx, $p, $new_meta, @data) = @_;
my $packid = $p->{'pkg'};
my $dst = "$ctx->{'builddir'}/$packid";
my @meta;
Expand All @@ -369,21 +386,137 @@ sub check_image {
close $mfp;
chomp @meta;
}
return ('scheduled', [ { 'explain' => 'new build' } ]) if !@meta;
return ('scheduled', [ { 'explain' => 'source change', 'oldsource' => substr($meta[0], 0, 32) } ]) if $meta[0] ne $new_meta->[0];
return ('scheduled', [ { 'explain' => 'forced rebuild' } ]) if $p->{'force_rebuild'};
return ('scheduled', [ { 'explain' => 'new build' }, @data ]) if !@meta;
return ('scheduled', [ { 'explain' => 'source change', 'oldsource' => substr($meta[0], 0, 32) }, @data ]) if $meta[0] ne $new_meta->[0];
return ('scheduled', [ { 'explain' => 'forced rebuild' }, @data ]) if $p->{'force_rebuild'};
my $rebuildmethod = $ctx->{'rebuild'} || 'transitive';
if ($rebuildmethod eq 'local') {
return ('scheduled', [ { 'explain' => 'rebuild counter sync' } ]) if $ctx->{'relsynctrigger'}->{$packid};
return ('scheduled', [ { 'explain' => 'rebuild counter sync' }, @data ]) if $ctx->{'relsynctrigger'}->{$packid};
return ('done');
}
if (@meta == @$new_meta && join('\n', @meta) eq join('\n', @$new_meta)) {
return ('scheduled', [ { 'explain' => 'rebuild counter sync' } ]) if $ctx->{'relsynctrigger'}->{$packid};
return ('scheduled', [ { 'explain' => 'rebuild counter sync' }, @data ]) if $ctx->{'relsynctrigger'}->{$packid};
return ('done');
}
my @diff = PBuild::Meta::diffsortedmd5(\@meta, $new_meta);
my $reason = PBuild::Meta::sortedmd5toreason(@diff);
return ('scheduled', [ { 'explain' => 'meta change', 'packagechange' => $reason } ] );
return ('scheduled', [ { 'explain' => 'meta change', 'packagechange' => $reason }, @data ] );
}

#
# Check the status of a single image/container
#
sub check_image {
my ($ctx, $p) = @_;
my $edeps = $p->{'dep_expanded'} || [];
my $notready = $ctx->{'notready'};
my $dep2src = $ctx->{'dep2src'};
my @blocked = grep {$notready->{$dep2src->{$_}}} @$edeps;
@blocked = () if $ctx->{'block'} && $ctx->{'block'} eq 'never';
if (@blocked) {
splice(@blocked, 10, scalar(@blocked), '...') if @blocked > 10;
return ('blocked', join(', ', @blocked));
}
my $new_meta = genmeta($ctx, $p, $edeps);
return check_meta($ctx, $p, $new_meta);
}

sub check_product {
my ($ctx, $p) = @_;
my $notready = $ctx->{'notready'};
my $dep2src = $ctx->{'dep2src'};
my @deps = @{$p->{'dep'} || []};
my %deps = map {$_ => 1} @deps;
my $versioned_deps;
for (grep {/[<=>]/} @deps) {
next unless /^(.*?)\s*([<=>].*)$/;
$deps{$1} = $2;
delete $deps{$_};
$versioned_deps = 1;
}
delete $deps{''};
delete $deps{"-$_"} for grep {!/^-/} keys %deps;
my $allpacks = $deps{'*'} ? 1 : 0;
my $nodbgpkgs = $p->{'nodbgpkgs'};
my $nosrcpkgs = $p->{'nosrcpkgs'};
my %unneeded_na;

my @rpms;
for my $repo (@{$ctx->{'repos'}}) {
my %seen_fn; # resolve file conflicts in this prp
my $gbininfo;
my @next_unneeded_na;
$gbininfo = $ctx->{'repomgr'}->get_gbininfo($repo);
my @apackids = sort keys %$gbininfo;
for my $apackid (@apackids) {
next if $apackid eq '_volatile';
my $bininfo = $gbininfo->{$apackid};
next unless $bininfo;
my $needit;
for my $fn (keys %$bininfo) {
next unless $fn =~ /^(?:::import::.*::)?(.+)-(?:[^-]+)-(?:[^-]+)\.([a-zA-Z][^\.\-]*)\.rpm$/;
my ($bn, $ba) = ($1, $2);
next if $ba eq 'src' || $ba eq 'nosrc'; # always unneeded
my $na = "$bn.$ba";
next if $unneeded_na{$na};
if ($fn =~ /-(?:debuginfo|debugsource)-/) {
if ($nodbgpkgs || !$deps{$bn}) {
$unneeded_na{$na} = 1;
next;
}
}
next if $seen_fn{$fn};
if ($fn =~ /^::import::(.*?)::(.*)$/) {
next if $seen_fn{$2};
}
my $d = $deps{$bn};
if (!($d || ($allpacks && !$deps{"-$bn"}))) {
$unneeded_na{$na} = 1; # cache unneeded
next;
}
if ($d && $d ne '1') {
my $bi = $bininfo->{$fn};
my $evr = "$bi->{'version'}-$bi->{'release'}";
$evr = "$bi->{'epoch'}:$evr" if $bi->{'epoch'};
next unless Build::matchsingledep("$bn=$evr", "$bn$d", 'rpm');
}
$needit = 1;
last;
}
next unless $needit;
# we need the package, add all artifacts
my @bi = sort(keys %$bininfo);
my @ibi = grep {/^::import::/} @bi;
if (@ibi) {
@bi = grep {!/^::import::/} @bi;
push @bi, @ibi;
}
for my $fn (@bi) {
next unless $fn =~ /^(?:::import::.*::)?(.+)-(?:[^-]+)-(?:[^-]+)\.([a-zA-Z][^\.\-]*)\.rpm$/;
my ($bn, $ba) = ($1, $2);
next if $nosrcpkgs && ($ba eq 'src' || $ba eq 'nosrc');
next if $nodbgpkgs && $fn =~ /-(?:debuginfo|debugsource)-/;
my $na = "$bn.$ba";
# ignore if we already have this file
next if $seen_fn{$fn};
next if $fn =~ /^::import::(.*?)::(.*)$/ && $seen_fn{$2};
my $b = $bininfo->{$fn};
push @rpms, { %{$bininfo->{$fn}}, 'package' => $apackid };
$seen_fn{$fn} = 1;
push @next_unneeded_na, $na unless $ba eq 'src' || $ba eq 'nosrc';
}
for my $fn (@bi) {
next unless ($fn =~ /[-.]appdata\.xml$/) || $fn eq '_modulemd.yaml' || $fn eq 'updateinfo.xml';
next if $seen_fn{$fn};
push @rpms, { %{$bininfo->{$fn}}, 'package' => $apackid };
$seen_fn{$fn} = 1 unless $fn eq 'updateinfo.xml' || $fn eq '_modulemd.yaml'; # we expect those to be renamed
}
}
@next_unneeded_na = () if $deps{'--use-newest-package'};
$unneeded_na{$_} = 1 for @next_unneeded_na;
}
my $new_meta = genmeta($ctx, $p, \@rpms);
return check_meta($ctx, $p, $new_meta, undef, \@rpms);
}

#
Expand All @@ -394,6 +527,7 @@ sub check {

my $buildtype = $p->{'buildtype'};
return check_image($ctx, $p) if $buildtype eq 'kiwi' || $buildtype eq 'docker' || $buildtype eq 'preinstallimage';
return check_product($ctx, $p) if $buildtype eq 'productcompose';

my $packid = $p->{'pkg'};
my $notready = $ctx->{'notready'};
Expand Down Expand Up @@ -585,6 +719,7 @@ sub dep2bins {
my ($ctx, @deps) = @_;
my $dep2pkg = $ctx->{'dep2pkg'};
for (@deps) {
next if ref($_) eq 'HASH'; # already a binary reference (used for product builds)
my $q = $dep2pkg->{$_};
die("unknown binary $_\n") unless $q;
$_ = $q;
Expand Down Expand Up @@ -621,8 +756,9 @@ sub build {
my $bconf = $bconf_host || $ctx->{'bconf'};
my $buildtype = $p->{'buildtype'};
$buildtype = 'kiwi-image' if $buildtype eq 'kiwi';
$edeps = $data->[2] if $buildtype eq 'productcompose';
my $kiwimode;
$kiwimode = $buildtype if $buildtype eq 'kiwi-image' || $buildtype eq 'kiwi-product' || $buildtype eq 'docker' || $buildtype eq 'fissile';
$kiwimode = $buildtype if $buildtype eq 'kiwi-image' || $buildtype eq 'kiwi-product' || $buildtype eq 'docker' || $buildtype eq 'fissile' || $buildtype eq 'productcompose';

if ($p->{'buildtimeservice'}) {
for my $service (@{$p->{'buildtimeservice'} || []}) {
Expand Down Expand Up @@ -699,7 +835,11 @@ sub build {
return ('recheck', 'assets changed') if $p->{'srcmd5'} ne $oldsrcmd5;
return ('broken', $p->{'error'}) if $p->{'error'}; # missing assets
my $bins;
if ($kiwimode && $bconf_host) {
if ($kiwimode && $kiwimode eq 'productcompose') {
$bins = dep2bins_host($ctx, PBuild::Util::unify(@pdeps, @vmdeps, @sysdeps));
push @$bins, @{dep2bins($ctx, PBuild::Util::unify(@$tdeps))} if $tdeps;
$ctx->{'repomgr'}->getremoteproductbinaries(\@bdeps);
} elsif ($kiwimode && $bconf_host) {
$bins = dep2bins_host($ctx, PBuild::Util::unify(@pdeps, @vmdeps, @sysdeps));
push @$bins, @{dep2bins($ctx, PBuild::Util::unify(@bdeps))};
} else {
Expand Down
2 changes: 1 addition & 1 deletion PBuild/Container.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ sub containerinfo2obsbinlnk {
for my $tag (@{$d->{tags}}) {
push @{$lnk->{'provides'}}, "container:$tag" unless "container:$tag" eq $lnk->{'name'};
}
eval { PBuild::Verify::verify_nevraquery($lnk) };
eval { PBuild::Verify::verify_nevraquery($lnk); PBuild::Verify::verify_filename($d->{'file'}) };
return undef if $@;
local *F;
if ($d->{'tar_md5sum'}) {
Expand Down
9 changes: 9 additions & 0 deletions PBuild/Expand.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,14 @@ sub configure_repos {
return \%packs;
}

#
# expand dependencies of a single package (product case)
#
sub expand_deps_product {
my ($p, $bconf, $subpacks, $cross) = @_;
$p->{'dep_expanded'} = []; # we don't expand anything for products
}

#
# expand dependencies of a single package (image case)
#
Expand Down Expand Up @@ -106,6 +114,7 @@ sub expand_deps_image {
sub expand_deps {
my ($p, $bconf, $subpacks, $cross) = @_;
my $buildtype = $p->{'buildtype'} || '';
return expand_deps_product($p, $bconf, $subpacks, $cross) if $buildtype eq 'productcompose';
return expand_deps_image($p, $bconf, $subpacks, $cross) if $buildtype eq 'kiwi' || $buildtype eq 'docker' || $buildtype eq 'fissile' || $buildtype eq 'preinstallimage';
delete $p->{'dep_experror'};
if ($p->{'error'}) {
Expand Down
11 changes: 8 additions & 3 deletions PBuild/Job.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,12 +130,17 @@ sub collect_result {
@d = ('DOCKER') if $p->{'recipe'} =~ /Dockerfile(\.|$)/;
@d = ('FISSILE') if $p->{'recipe'} =~ /fissile\.yml$/;
@d = ('HELM') if $p->{'recipe'} =~ /Chart\.yaml$/;
@d = ('PRODUCT') if $p->{'recipe'} =~ /\.productcompose$/;
push @d, 'OTHER';
my @send;
for my $d ('.', @d) {
my @files = sort(PBuild::Util::ls("$buildroot/.build.packages/$d"));
@files = grep {$_ ne 'same_result_marker' && $_ ne '.kiwitree'} @files;
@files = grep {! -l "$buildroot/.build.packages/$d/$_" && -f _} @files;
if ($p->{'buildtype'} eq 'productcompose' && $d eq 'PRODUCT') {
@files = grep {! -l "$buildroot/.build.packages/$d/$_" && (-d _ || -f _)} @files;
} else {
@files = grep {! -l "$buildroot/.build.packages/$d/$_" && -f _} @files;
}
push @send, map {"$buildroot/.build.packages/$d/$_"} @files;
}
my %send = map {(split('/', $_))[-1] => $_} @send;
Expand Down Expand Up @@ -207,7 +212,7 @@ sub createjob {
my %sysdeps = map {$_ => 1} @$sysdeps;

my @alldeps;
my $kiwimode = $p->{'buildtype'} eq 'kiwi' || $p->{'buildtype'} eq 'docker' || $p->{'buildtype'} eq 'fissile' ? $p->{'buildtype'} : undef;
my $kiwimode = $p->{'buildtype'} eq 'kiwi' || $p->{'buildtype'} eq 'docker' || $p->{'buildtype'} eq 'fissile' || $p->{'buildtype'} eq 'productcompose' ? $p->{'buildtype'} : undef;
if ($kiwimode) {
@alldeps = PBuild::Util::unify(@$pdeps, @$vmdeps, @$sysdeps);
} else {
Expand Down Expand Up @@ -338,7 +343,7 @@ sub createjob {
if ($kiwimode) {
# now setup the repos/containers directories
$ctx->{'repomgr'}->copyimagebinaries($ctx->dep2bins(@$bdeps), $srcdir);
# tell kiwi how to use them
# and tell kiwi how to use them
if ($p->{'buildtype'} eq 'kiwi') {
my @kiwiargs;
push @kiwiargs, '--ignore-repos';
Expand Down
8 changes: 8 additions & 0 deletions PBuild/LocalRepo.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,4 +189,12 @@ sub cleanup_builddir {
}
}

#
# Return the artifact data of all built packages
#
sub get_gbininfo {
my ($builddir) = @_;
return PBuild::Util::retrieve("$builddir/.pbuild/_bininfo");
}

1;
Loading

0 comments on commit 3bae6e4

Please sign in to comment.