turn -J into an option taking a number.
always run show-prepare-results, and always serialize depends/prepare/show-prepare-results(/junk) on a single host.
This commit is contained in:
espie
parent
edd8b208f3
commit
abb8ddf57d
@ -1,7 +1,7 @@
|
||||
#! /usr/bin/perl
|
||||
|
||||
# ex:ts=8 sw=4:
|
||||
# $OpenBSD: dpb,v 1.35 2011/12/04 12:05:41 espie Exp $
|
||||
# $OpenBSD: dpb,v 1.36 2011/12/05 16:10:00 espie Exp $
|
||||
#
|
||||
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
|
||||
#
|
||||
@ -114,8 +114,8 @@ sub handle_options
|
||||
},
|
||||
};
|
||||
|
||||
$state->SUPER::handle_options('aceJqrRstuUvh:xA:C:f:F:I:j:m:P:b:L:S:',
|
||||
"[-aceJrRsuUvx] [-A arch] [-C plist] [-f N] [-F N] [-I plist] [-j N]",
|
||||
$state->SUPER::handle_options('aceqrRstuUvh:xA:C:f:F:I:j:J:m:P:b:L:S:',
|
||||
"[-acerRsuUvx] [-A arch] [-C plist] [-f N] [-F N] [-I plist] [-J p] [-j N]",
|
||||
"[-P plist] [-h hosts] [-L logdir] [-b log] [-t ctimeout] [-m threshold]",
|
||||
"[path ...]");
|
||||
$state->{fullrepo} = join("/", $state->{repo}, $state->arch, "all");
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# ex:ts=8 sw=4:
|
||||
# $OpenBSD: Port.pm,v 1.21 2011/12/04 12:05:41 espie Exp $
|
||||
# $OpenBSD: Port.pm,v 1.22 2011/12/05 16:10:01 espie Exp $
|
||||
#
|
||||
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
|
||||
#
|
||||
@ -51,26 +51,6 @@ sub finalize
|
||||
return $core->{status} == 0;
|
||||
}
|
||||
|
||||
sub junk_lock
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
my $builder = $core->job->{builder};
|
||||
return unless $builder->{junk};
|
||||
|
||||
while (!$builder->locker->lock($core)) {
|
||||
sleep 1;
|
||||
}
|
||||
}
|
||||
|
||||
sub junk_unlock
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
my $builder = $core->job->{builder};
|
||||
return unless $builder->{junk};
|
||||
|
||||
$builder->locker->unlock($core);
|
||||
}
|
||||
|
||||
sub handle_output
|
||||
{
|
||||
my ($self, $job) = @_;
|
||||
@ -88,6 +68,8 @@ sub run
|
||||
my $sudo = OpenBSD::Paths->sudo;
|
||||
my $shell = $core->{shell};
|
||||
$self->handle_output($job);
|
||||
close STDIN;
|
||||
open STDIN, '</dev/null';
|
||||
print ">>> Running $t in $fullpkgpath\n";
|
||||
my @args = ($t, "TRUST_PACKAGES=Yes",
|
||||
"FETCH_PACKAGES=No",
|
||||
@ -121,12 +103,45 @@ sub run
|
||||
|
||||
sub notime { 0 }
|
||||
|
||||
package DPB::Task::Port::NoTime;
|
||||
package DPB::Task::Port::Serialized;
|
||||
our @ISA = qw(DPB::Task::Port);
|
||||
sub notime { 1 }
|
||||
|
||||
sub junk_lock
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
my $job = $core->job;
|
||||
my $locker = $job->{builder}->locker;
|
||||
|
||||
while (1) {
|
||||
my $fh = $locker->lock($core);
|
||||
if ($fh) {
|
||||
print $fh "path=".$job->{v}->fullpkgpath, "\n";
|
||||
return;
|
||||
}
|
||||
sleep 1;
|
||||
}
|
||||
}
|
||||
|
||||
sub junk_unlock
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
|
||||
$core->job->{builder}->locker->unlock($core);
|
||||
}
|
||||
|
||||
sub finalize
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
if ($core->{status} != 0) {
|
||||
$self->junk_unlock($core);
|
||||
}
|
||||
$self->SUPER::finalize($core);
|
||||
}
|
||||
|
||||
package DPB::Task::Port::Signature;
|
||||
our @ISA =qw(DPB::Task::Port::NoTime);
|
||||
our @ISA =qw(DPB::Task::Port);
|
||||
|
||||
sub notime { 1 }
|
||||
|
||||
sub run
|
||||
{
|
||||
@ -169,14 +184,15 @@ sub run
|
||||
}
|
||||
|
||||
package DPB::Task::Port::Depends;
|
||||
our @ISA=qw(DPB::Task::Port::NoTime);
|
||||
our @ISA=qw(DPB::Task::Port::Serialized);
|
||||
|
||||
sub notime { 1 }
|
||||
|
||||
sub run
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
my $job = $core->job;
|
||||
my $dep = {};
|
||||
$self->junk_lock($core);
|
||||
my $v = $job->{v};
|
||||
if (exists $v->{info}{BDEPENDS}) {
|
||||
for my $d (values %{$v->{info}{BDEPENDS}}) {
|
||||
@ -207,6 +223,7 @@ sub run
|
||||
}
|
||||
print join(' ', @cmd, (sort keys %$dep)), "\n";
|
||||
my $path = $job->{builder}->{fullrepo}.'/';
|
||||
$self->junk_lock($core);
|
||||
if (defined $shell) {
|
||||
$shell->run(join(' ', "PKG_PATH=$path", $sudo, @cmd,
|
||||
(sort keys %$dep)));
|
||||
@ -220,13 +237,15 @@ sub run
|
||||
sub finalize
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
$self->SUPER::finalize($core);
|
||||
$core->{status} = 0;
|
||||
$self->SUPER::finalize($core);
|
||||
return 1;
|
||||
}
|
||||
|
||||
package DPB::Task::Port::Install;
|
||||
our @ISA=qw(DPB::Task::Port::NoTime);
|
||||
our @ISA=qw(DPB::Task::Port);
|
||||
|
||||
sub notime { 1 }
|
||||
|
||||
sub run
|
||||
{
|
||||
@ -253,34 +272,23 @@ sub run
|
||||
sub finalize
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
$self->SUPER::finalize($core);
|
||||
$core->{status} = 0;
|
||||
$self->SUPER::finalize($core);
|
||||
return 1;
|
||||
}
|
||||
|
||||
package DPB::Task::Port::Prepare;
|
||||
our @ISA = qw(DPB::Task::Port);
|
||||
|
||||
sub run
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
$self->SUPER::run($core);
|
||||
}
|
||||
|
||||
sub finalize
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
$self->SUPER::finalize($core);
|
||||
}
|
||||
our @ISA = qw(DPB::Task::Port::Serialized);
|
||||
|
||||
package DPB::Task::Port::PrepareResults;
|
||||
our @ISA = qw(DPB::Task::Port);
|
||||
our @ISA = qw(DPB::Task::Port::Serialized);
|
||||
|
||||
sub result_filename
|
||||
{
|
||||
my ($self, $job) = @_;
|
||||
return $job->{builder}->logger->log_pkgpath($job->{v}).".tmp";
|
||||
}
|
||||
|
||||
sub handle_output
|
||||
{
|
||||
my ($self, $job) = @_;
|
||||
@ -291,34 +299,41 @@ sub finalize
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
|
||||
$self->SUPER::finalize($core);
|
||||
$core->{status} = 0;
|
||||
|
||||
my $job = $core->{job};
|
||||
my $v = $job->{v};
|
||||
my $file = $self->result_filename($job);
|
||||
if (open my $fh, '<', $file) {
|
||||
my @l;
|
||||
my @r;
|
||||
while (<$fh>) {
|
||||
# zap headers
|
||||
next if m/^\>\>\>\s/ || m/^\=\=\=\>\s/;
|
||||
chomp;
|
||||
push(@l, $_);
|
||||
push(@r, $_);
|
||||
}
|
||||
print {$job->{lock}} "needed=", join(' ', sort @l), "\n";
|
||||
print {$job->{lock}} "needed=", join(' ', sort @r), "\n";
|
||||
close $fh;
|
||||
unlink($file);
|
||||
# full list for every lock
|
||||
my @d = $core->job->{builder}->locker->find_dependencies(
|
||||
$core->hostname);
|
||||
$job->{needed} = [sort @d];
|
||||
print {$job->{builder}{junk_log}} $core->hostname, "(",
|
||||
$job->{v}->fullpkgpath, "): ", join(' ', scalar(@d), @d),
|
||||
"\n";
|
||||
} else {
|
||||
$core->{status} = 1;
|
||||
}
|
||||
unlink($file);
|
||||
my $l = $job->{builder}{junk};
|
||||
# full list for every lock
|
||||
my @l = $core->job->{builder}->locker->find_dependencies($core->hostname);
|
||||
$job->{needed} = [sort @l];
|
||||
print $l $core->hostname, "(", $job->{v}->fullpkgpath, "): ",
|
||||
join(' ', scalar(@l), @l), "\n";
|
||||
return 1;
|
||||
# if we're not followed by uninstall, then we end the serialized part.
|
||||
if (!$job->{has_junk}) {
|
||||
$self->junk_unlock($core);
|
||||
}
|
||||
$self->SUPER::finalize($core);
|
||||
}
|
||||
|
||||
package DPB::Task::Port::Uninstall;
|
||||
our @ISA=qw(DPB::Task::Port::NoTime);
|
||||
our @ISA=qw(DPB::Task::Port::Serialized);
|
||||
|
||||
sub notime { 1 }
|
||||
|
||||
sub run
|
||||
{
|
||||
@ -342,9 +357,9 @@ sub run
|
||||
sub finalize
|
||||
{
|
||||
my ($self, $core) = @_;
|
||||
$self->SUPER::finalize($core);
|
||||
$self->junk_unlock($core);
|
||||
$core->{status} = 0;
|
||||
$self->junk_unlock($core);
|
||||
$self->SUPER::finalize($core);
|
||||
return 1;
|
||||
}
|
||||
|
||||
@ -402,7 +417,9 @@ sub finalize
|
||||
|
||||
|
||||
package DPB::Task::Port::Fetch;
|
||||
our @ISA = qw(DPB::Task::Port::NoTime);
|
||||
our @ISA = qw(DPB::Task::Port);
|
||||
|
||||
sub notime { 1 }
|
||||
|
||||
sub finalize
|
||||
{
|
||||
@ -418,7 +435,9 @@ sub finalize
|
||||
}
|
||||
|
||||
package DPB::Task::Port::Clean;
|
||||
our @ISA = qw(DPB::Task::Port::NoTime);
|
||||
our @ISA = qw(DPB::Task::Port);
|
||||
|
||||
sub notime { 1 }
|
||||
|
||||
sub finalize
|
||||
{
|
||||
@ -498,9 +517,13 @@ sub add_normal_tasks
|
||||
if ($builder->{clean}) {
|
||||
push @todo, "clean";
|
||||
}
|
||||
push(@todo, qw(depends prepare));
|
||||
push(@todo, qw(depends prepare show-prepare-results));
|
||||
if ($builder->{junk}) {
|
||||
push(@todo, qw(show-prepare-results junk));
|
||||
if ($builder->{junk_count}++ >= $builder->{junk}) {
|
||||
$builder->{junk_count} = 0;
|
||||
push(@todo, 'junk');
|
||||
$self->{has_junk} = 1;
|
||||
}
|
||||
}
|
||||
if ($builder->{fetch}) {
|
||||
push(@todo, qw(checksum));
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# ex:ts=8 sw=4:
|
||||
# $OpenBSD: PortBuilder.pm,v 1.17 2011/12/04 12:05:41 espie Exp $
|
||||
# $OpenBSD: PortBuilder.pm,v 1.18 2011/12/05 16:10:01 espie Exp $
|
||||
#
|
||||
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
|
||||
#
|
||||
@ -44,7 +44,8 @@ sub new
|
||||
$self->{forceupdate} = 1;
|
||||
}
|
||||
if ($state->opt('J')) {
|
||||
$self->{junk} = DPB::Util->make_hot($self->logger->open("needed"));
|
||||
$self->{junk_log} = DPB::Util->make_hot($self->logger->open("needed"));
|
||||
$self->{junk} = $state->opt('J');
|
||||
}
|
||||
$self->init;
|
||||
return $self;
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
.\" $OpenBSD: dpb.1,v 1.24 2011/12/03 18:02:39 espie Exp $
|
||||
.\" $OpenBSD: dpb.1,v 1.25 2011/12/05 16:10:00 espie Exp $
|
||||
.\"
|
||||
.\" Copyright (c) 2010 Marc Espie <espie@openbsd.org>
|
||||
.\"
|
||||
@ -14,7 +14,7 @@
|
||||
.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
.\"
|
||||
.Dd $Mdocdate: December 3 2011 $
|
||||
.Dd $Mdocdate: December 5 2011 $
|
||||
.Dt DPB 1
|
||||
.Os
|
||||
.Sh NAME
|
||||
@ -23,7 +23,7 @@
|
||||
.Sh SYNOPSIS
|
||||
.Nm dpb
|
||||
.Bk -words
|
||||
.Op Fl aceJqrRsuUx
|
||||
.Op Fl aceqrRsuUx
|
||||
.Op Fl A Ar arch
|
||||
.Op Fl b Ar logfile
|
||||
.Op Fl C Ar pathlist
|
||||
@ -31,6 +31,7 @@
|
||||
.Op Fl f Ar m
|
||||
.Op Fl h Ar hosts
|
||||
.Op Fl I Ar subdirlist
|
||||
.Op Fl J Ar p
|
||||
.Op Fl j Ar n
|
||||
.Op Fl L Ar logdir
|
||||
.Op Fl m Ar threshold
|
||||
@ -198,13 +199,16 @@ List of
|
||||
.Xr pkgpath 7
|
||||
to install, on the local box.
|
||||
This will also add them to the list of things to build.
|
||||
.It Fl J
|
||||
.It Fl J Ar p
|
||||
Delete unneeded installed packages during the build.
|
||||
Each
|
||||
.Ar prepare
|
||||
stage is followed by a
|
||||
.Ar show-prepare-results
|
||||
stage and a
|
||||
stage.
|
||||
Every
|
||||
.Ar p
|
||||
build, it will be followed by a
|
||||
.Ar junk
|
||||
stage which uses
|
||||
.Xr pkg_delete 1
|
||||
@ -608,28 +612,6 @@ You have to stop
|
||||
.Nm ,
|
||||
cleanup and restart.
|
||||
.Pp
|
||||
Mixing
|
||||
.Nm
|
||||
running with
|
||||
.Fl J
|
||||
and
|
||||
.Nm
|
||||
running without
|
||||
.Fl J
|
||||
will lead to spurious errors: a
|
||||
.Nm
|
||||
lacking the option will not lock the host, and thus, may install new
|
||||
packages during
|
||||
.Ar depends ,
|
||||
and then fail the
|
||||
.Ar prepare
|
||||
stage as another
|
||||
.Nm
|
||||
will have removed the extraneous packages in-between.
|
||||
Always locking would be a safer option, but the gathering of statistical
|
||||
information to determine if the resulting bottleneck has an actual
|
||||
negative impact is currently on-going.
|
||||
.Pp
|
||||
There's a bug in
|
||||
.Xr mfs 8
|
||||
that prevents it from proper use in bulk builds.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user