diff --git a/infrastructure/bin/dpb b/infrastructure/bin/dpb index 2737da5ed61..30c88888eae 100755 --- a/infrastructure/bin/dpb +++ b/infrastructure/bin/dpb @@ -1,7 +1,7 @@ #! /usr/bin/perl # ex:ts=8 sw=4: -# $OpenBSD: dpb,v 1.25 2011/09/25 10:41:30 espie Exp $ +# $OpenBSD: dpb,v 1.26 2011/10/10 18:56:50 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -236,7 +236,7 @@ sub parse_build_file chomp; my ($pkgpath, $host, $time, $sz, @rest) = parse_build_line($_); next if (!defined $sz) || $sz =~ m/!$/; - my $o = DPB::PkgPath->new_hidden($pkgpath); + my $o = DPB::PkgPath->new($pkgpath); for my $c (@consumers) { $c->add_build_info($o, $host, $time, $sz); } @@ -288,7 +288,7 @@ use DPB::Locks; use DPB::Job; use DPB::Grabber; -my @subdirlist; +my $subdirlist = {}; my $state = DPB::State->new('dpb'); $state->handle_options; @@ -310,7 +310,7 @@ for my $arg (@ARGV) { } my $pkgpath = DPB::PkgPath->new($path); $state->heuristics->set_weight($pkgpath, $weight); - $pkgpath->add_to_subdirlist(\@subdirlist); + $pkgpath->add_to_subdirlist($subdirlist); } $state->{builder} = DPB::PortBuilder->new($state); @@ -375,12 +375,13 @@ if ($state->{all}) { my $list = $state->engine->find_best($state->logger->logfile("dependencies"), 10); # if we have them, list them before the full ports tree walk. if (@$list > 0) { - $state->grabber->grab_subdirs($core, $list); + my $actual = map {($_,1)} @$list; + $state->grabber->grab_subdirs($core, $actual); } } -if (@subdirlist > 0) { - $state->grabber->grab_subdirs($core, \@subdirlist); +if (keys %$subdirlist > 0) { + $state->grabber->grab_subdirs($core, $subdirlist); } $state->grabber->complete_subdirs($core); diff --git a/infrastructure/lib/DPB/Engine.pm b/infrastructure/lib/DPB/Engine.pm index 1991f75895d..6b0d2b8d088 100644 --- a/infrastructure/lib/DPB/Engine.pm +++ b/infrastructure/lib/DPB/Engine.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: Engine.pm,v 1.27 2011/09/28 09:49:29 espie Exp $ +# $OpenBSD: Engine.pm,v 1.28 2011/10/10 18:56:50 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -583,8 +583,8 @@ sub rebuild_info my ($self, $core) = @_; my @l = @{$self->{requeued}}; $self->{requeued} = []; - my @subdirs = map {$_->fullpkgpath} @l; - $self->{state}->grabber->grab_subdirs($core, \@subdirs); + my %subdirs = map {($_->pkgpath_and_flavors, 1)} @l; + $self->{state}->grabber->grab_subdirs($core, \%subdirs); $core->mark_ready; } diff --git a/infrastructure/lib/DPB/Grabber.pm b/infrastructure/lib/DPB/Grabber.pm index 2968ce09594..f4b877b059d 100644 --- a/infrastructure/lib/DPB/Grabber.pm +++ b/infrastructure/lib/DPB/Grabber.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: Grabber.pm,v 1.14 2011/09/25 10:40:25 espie Exp $ +# $OpenBSD: Grabber.pm,v 1.15 2011/10/10 18:56:50 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -53,7 +53,8 @@ sub finish delete $v->{info}; delete $v->{broken}; $self->{engine}->add_fatal($v); - } else { + } elsif ($v->{wantbuild}) { + delete $v->{wantbuild}; $self->{engine}->new_path($v); } } @@ -84,7 +85,11 @@ sub grab_subdirs DPB::Vars->grab_list($core, $self, $list, $self->{loglist}, $self->{dpb}, sub { - $self->finish(shift); + my $h = shift; + for my $v (values %$h) { + $v->{wantbuild} = 1; + } + $self->finish($h); }); } @@ -105,22 +110,32 @@ sub complete_subdirs my ($self, $core) = @_; # more passes if necessary while ($self->{keep_going}) { - my @subdirlist = (); + my $subdirlist = {}; for my $v (DPB::PkgPath->seen) { - next if defined $v->{info}; + if (defined $v->{info}) { + if (defined $v->{wantbuild}) { + delete $v->{wantbuild}; + $self->{engine}->new_path($v); + } + next; + } next if defined $v->{category}; if (defined $v->{tried}) { $self->{engine}->add_fatal($v) if !defined $v->{errored}; $v->{errored} = 1; - } else { - $v->add_to_subdirlist(\@subdirlist); + } elsif ($v->{wantinfo} || $v->{wantbuild}) { + $v->add_to_subdirlist($subdirlist); $v->{tried} = 1; } } - last if @subdirlist == 0; + last if (keys %$subdirlist) == 0; - $self->grab_subdirs($core, \@subdirlist); + DPB::Vars->grab_list($core, $self, $subdirlist, + $self->{loglist}, $self->{dpb}, + sub { + $self->finish(shift); + }); } } diff --git a/infrastructure/lib/DPB/Heuristics.pm b/infrastructure/lib/DPB/Heuristics.pm index 089930b4216..11be350dce6 100644 --- a/infrastructure/lib/DPB/Heuristics.pm +++ b/infrastructure/lib/DPB/Heuristics.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: Heuristics.pm,v 1.10 2011/09/13 09:46:53 espie Exp $ +# $OpenBSD: Heuristics.pm,v 1.11 2011/10/10 18:56:50 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -64,7 +64,19 @@ sub finished_parsing sub intrinsic_weight { my ($self, $v) = @_; - $weight{$v} //= $default; + $weight{$v} // $default; +} + +sub equates +{ + my ($class, $h) = @_; + for my $v (values %$h) { + next unless defined $weight{$v}; + for my $w (values %$h) { + $weight{$w} //= $weight{$v}; + } + return; + } } my $threshold; diff --git a/infrastructure/lib/DPB/Job/Port.pm b/infrastructure/lib/DPB/Job/Port.pm index 98e5c8c76ad..fd30234ef34 100644 --- a/infrastructure/lib/DPB/Job/Port.pm +++ b/infrastructure/lib/DPB/Job/Port.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: Port.pm,v 1.13 2011/09/25 10:41:30 espie Exp $ +# $OpenBSD: Port.pm,v 1.14 2011/10/10 18:56:50 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -429,6 +429,18 @@ sub add_build_info $logsize->{$pkgpath} = $sz; } +sub equates +{ + my ($class, $h) = @_; + for my $v (values %$h) { + next unless defined $logsize->{$v}; + for my $w (values %$h) { + $logsize->{$w} //= $logsize->{$v}; + } + return; + } +} + sub set_watch { my ($self, $logger, $v) = @_; diff --git a/infrastructure/lib/DPB/PkgPath.pm b/infrastructure/lib/DPB/PkgPath.pm index 4da54fceef8..023e06cbef7 100644 --- a/infrastructure/lib/DPB/PkgPath.pm +++ b/infrastructure/lib/DPB/PkgPath.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: PkgPath.pm,v 1.12 2011/10/03 08:56:40 espie Exp $ +# $OpenBSD: PkgPath.pm,v 1.13 2011/10/10 18:56:50 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -24,7 +24,6 @@ use warnings; package DPB::PkgPath; my $cache = {}; -my $seen = {}; sub create { @@ -65,8 +64,8 @@ sub normalize return $cache->{$fullpkgpath} //= $o; } -# actual user constructor that doesn't record into seen -sub new_hidden +# actual constructor +sub new { my ($class, $fullpkgpath) = @_; if (defined $cache->{$fullpkgpath}) { @@ -76,17 +75,9 @@ sub new_hidden } } -# actual user constructor that records into seen -sub new -{ - my ($class, $fullpkgpath) = @_; - my $o = $class->new_hidden($fullpkgpath); - $seen->{$o} //= $o; -} - sub seen { - return values %$seen; + return values %$cache; } sub basic_list @@ -113,7 +104,8 @@ sub fullpkgpath sub logname { - return shift->fullpkgpath; + my $self = shift; + return $self->fullpkgpath; } sub lockname @@ -150,7 +142,7 @@ sub pkgpath_and_flavors sub add_to_subdirlist { my ($self, $list) = @_; - push(@$list, $self->pkgpath_and_flavors); + $list->{$self->pkgpath_and_flavors} = 1; } sub copy_flavors @@ -170,7 +162,6 @@ sub compose my $o = $class->create($fullpkgpath); $o->{flavors} = $pseudo->copy_flavors; $o->{sawflavor} = $pseudo->{sawflavor}; - my $p = $o->normalize; return $o->normalize; } @@ -182,11 +173,46 @@ sub fullpkgname return (defined $self->{info}) ? $self->{info}->fullpkgname : undef; } +sub may_create +{ + my ($n, $o, $h, $state) = @_; + my $k = $n->fullpkgpath; + if (defined $cache->{$k}) { + $n = $cache->{$k}; + } else { + $cache->{$k} = $n; + } + $n->{has} //= $o->{has}; + $n->{new} //= $o->{new}; + $n->{info} //= $o->{info}; + $h->{$n} = 1; + return $n; +} + +sub simplifies_to +{ + my ($self, $simpler, $state) = @_; + open my $quicklog, '>>', $state->logger->logfile('equiv'); + print $quicklog $self->fullpkgpath, " -> ", $simpler->fullpkgpath, "\n"; +} + +sub handle_equivalences +{ + my ($class, $state, $todo) = @_; + my $h = {}; + for my $v (values %$todo) { + $h->{$v} = 1; + $v->handle_default_flavor($h, $state); + $v->handle_default_subpackage($h, $state); + } + DPB::Job::Port->equates($h); + DPB::Heuristics->equates($h); +} sub zap_default { my ($self, $subpackage) = @_; - return $self unless defined $subpackage; + return $self unless defined $subpackage and defined $self->{multi}; if ($subpackage->string eq $self->{multi}) { my $o = bless {pkgpath => $self->{pkgpath}, sawflavor => $self->{sawflavor}, @@ -197,15 +223,30 @@ sub zap_default } } -# default subpackage leads to pkgpath,-default = pkgpath -sub handle_default +sub handle_default_flavor { - my ($self, $h) = @_; + my ($self, $h, $state) = @_; + + if (!$self->{sawflavor}) { + my $m = bless { pkgpath => $self->{pkgpath}, + sawflavor => 1, + multi => $self->{multi}, + flavors => $self->{info}->{FLAVOR}}, ref($self); + $m = $m->may_create($self, $h, $state); + $m->simplifies_to($self, $state); + $m->handle_default_subpackage($h, $state); + } +} + +# default subpackage leads to pkgpath,-default = pkgpath +sub handle_default_subpackage +{ + my ($self, $h, $state) = @_; my $m = $self->zap_default($self->{info}->{SUBPACKAGE}); if ($m ne $self) { -# print $m->fullpkgpath, " vs. ", $self->fullpkgpath,"\n"; - $m->{info} = $self->{info}; - $h->{$m} = $m; + $m = $m->may_create($self, $h, $state); + $self->simplifies_to($m, $state); + $m->handle_default_flavor($h, $state); } } diff --git a/infrastructure/lib/DPB/PortInfo.pm b/infrastructure/lib/DPB/PortInfo.pm index 8475b482678..d1a392d5a39 100644 --- a/infrastructure/lib/DPB/PortInfo.pm +++ b/infrastructure/lib/DPB/PortInfo.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: PortInfo.pm,v 1.13 2011/10/03 08:53:49 espie Exp $ +# $OpenBSD: PortInfo.pm,v 1.14 2011/10/10 18:56:50 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -165,6 +165,7 @@ sub new } else { my $info = DPB::PkgPath->new($_); $info->{parent} //= $parent; + $info->{wantbuild} = 1; $r->{$info} = $info; } } @@ -198,6 +199,7 @@ sub add my ($class, $key, $self, $value, $parent) = @_; $self->{$key} //= bless {}, $class; my $path = DPB::PkgPath->new($value); + $path->{wantinfo} = 1; $path->{parent} //= $parent; $self->{$key}{$path} = $path; return $self; diff --git a/infrastructure/lib/DPB/Vars.pm b/infrastructure/lib/DPB/Vars.pm index f17dcf3859c..7c259dd46b3 100644 --- a/infrastructure/lib/DPB/Vars.pm +++ b/infrastructure/lib/DPB/Vars.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: Vars.pm,v 1.18 2011/09/13 09:46:53 espie Exp $ +# $OpenBSD: Vars.pm,v 1.19 2011/10/10 18:56:50 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -21,7 +21,7 @@ package DPB::GetThings; sub subdirlist { my ($class, $list) = @_; - return join(' ', sort @$list); + return join(' ', sort keys %$list); } sub run_command @@ -118,9 +118,7 @@ sub grab_list my $subdir; my $category; my $reset = sub { - for my $v (values %$h) { - $v->handle_default($h); - } + DPB::PkgPath->handle_equivalences($grabber->{state}, $h); $grabber->{fetch}->build_distinfo($h, $grabber->{state}->{fetch_only}); DPB::PkgPath->merge_depends($h); @@ -134,7 +132,7 @@ sub grab_list chomp; if (m/^\=\=\=\>\s*Exiting (.*) with an error$/) { undef $category; - my $dir = DPB::PkgPath->new_hidden($1); + my $dir = DPB::PkgPath->new($1); $dir->{broken} = 1; $h->{$dir} = $dir; open my $quicklog, '>>', @@ -147,7 +145,7 @@ sub grab_list @current = ("$_\n"); print $log $_, "\n"; $core->job->set_status(" at $1"); - $subdir = DPB::PkgPath->new_hidden($1); + $subdir = DPB::PkgPath->new($1); if (defined $category) { $category->{category} = 1; } @@ -171,7 +169,7 @@ sub grab_list $o->{broken} = 1; } } elsif (m/^\>\>\s*Broken dependency:\s*(.*?)\s*non existent/) { - my $dir = DPB::PkgPath->new_hidden($1); + my $dir = DPB::PkgPath->new($1); $dir->{broken} = 1; $h->{$dir} = $dir; print $log $_, "\n";