small change for people running multiple hosts:
options on the command line now define *defaults* that host files can override (for instance -j, stuck, -p, -J). Add -p /n to mean "take number of jobs, if >1, divide by n, round up to 2, and use that for parallel. Document -p. Make junk be 'by host' (and it's a prop, so you can tweak it). concurrent log that records how many jobs are running each time it changes. tag parallel builds *n in the time record.
This commit is contained in:
espie
parent
535f486bc2
commit
cdc017bcb0
@ -1,7 +1,7 @@
|
||||
#! /usr/bin/perl
|
||||
|
||||
# ex:ts=8 sw=4:
|
||||
# $OpenBSD: dpb,v 1.53 2012/09/23 18:13:32 espie Exp $
|
||||
# $OpenBSD: dpb,v 1.54 2012/10/08 12:41:03 espie Exp $
|
||||
#
|
||||
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
|
||||
#
|
||||
@ -163,9 +163,9 @@ sub handle_options
|
||||
},
|
||||
};
|
||||
|
||||
$state->SUPER::handle_options('acemqrRstuUvh:xA:C:f:F:I:j:J:M:p:P:b:L:S:',
|
||||
$state->SUPER::handle_options('acemqrRsuUvh:xA:C:f:F:I:j:J:M:p:P:b:L:S:',
|
||||
"[-acemrRsuUvx] [-A arch] [-C plist] [-f N] [-F N] [-I plist] [-J p] [-j N]",
|
||||
"[-P plist] [-h hosts] [-L logdir] [-b log] [-t ctimeout] [-M threshold]",
|
||||
"[-p parallel] [-P plist] [-h hosts] [-L logdir] [-b log] [-M threshold]",
|
||||
"[path ...]");
|
||||
$state->{fullrepo} = join("/", $state->{repo}, $state->arch, "all");
|
||||
$state->{logdir} //= $ENV{LOGDIR} // '%p/logs/%a';
|
||||
@ -180,6 +180,7 @@ sub handle_options
|
||||
}
|
||||
$state->{opt}{f} //= 2;
|
||||
$state->{opt}{J} //= 250;
|
||||
$state->{opt}{p} //= '/2';
|
||||
if (defined $state->opt('j')) {
|
||||
if ($state->localarch ne $state->arch) {
|
||||
$state->usage(
|
||||
@ -224,37 +225,42 @@ sub handle_options
|
||||
$state->{logger} = DPB::Logger->new($state->logdir, $state->opt('c'));
|
||||
$state->heuristics->set_logger($state->logger);
|
||||
$state->{display_timeout} =
|
||||
$state->{subst}->value('DISPLAY_TIMEOUT') // $state->opt('T') // 10;
|
||||
$state->{subst}->value('DISPLAY_TIMEOUT') // 10;
|
||||
$state->{connection_timeout} =
|
||||
$state->{subst}->value('CONNECTION_TIMEOUT') // $state->opt('t');
|
||||
$state->{subst}->value('CONNECTION_TIMEOUT');
|
||||
$state->{stuck_timeout} = $state->{subst}->value('STUCK_TIMEOUT');
|
||||
$state->{build_once} = $state->{all};
|
||||
if ($state->defines("DONT_BUILD_ONCE")) {
|
||||
$state->{build_once} = 0;
|
||||
}
|
||||
$state->{concurrent} = $state->{logger}->open("concurrent");
|
||||
}
|
||||
|
||||
sub start_cores
|
||||
{
|
||||
my $state = shift;
|
||||
|
||||
if ($state->{config}) {
|
||||
DPB::Core->parse_hosts_file($state->{config}, $state);
|
||||
}
|
||||
my $default_prop = {};
|
||||
|
||||
my $prop = {};
|
||||
if ($state->opt('j')) {
|
||||
$prop->{jobs} = $state->opt('j');
|
||||
$default_prop->{jobs} = $state->opt('j');
|
||||
}
|
||||
if ($state->opt('p')) {
|
||||
$prop->{parallel} = $state->opt('p');
|
||||
$default_prop->{parallel} = $state->opt('p');
|
||||
}
|
||||
if ($state->{stuck_timeout}) {
|
||||
$prop->{stuck} = $state->{stuck_timeout};
|
||||
$default_prop->{stuck} = $state->{stuck_timeout};
|
||||
}
|
||||
if ($state->opt('J')) {
|
||||
$default_prop->{junk} = $state->opt('J');
|
||||
}
|
||||
|
||||
if ($state->opt('j') || !$state->{config}) {
|
||||
DPB::Core::Factory->new('localhost', $prop);
|
||||
if ($state->{config}) {
|
||||
DPB::Core->parse_hosts_file($state->{config}, $state, $default_prop);
|
||||
}
|
||||
|
||||
if (!$state->{config}) {
|
||||
DPB::Core::Factory->new('localhost', $default_prop);
|
||||
}
|
||||
DPB::Core::Factory->init_cores($state);
|
||||
}
|
||||
@ -564,8 +570,9 @@ sub handle_non_waiting_jobs
|
||||
}
|
||||
$state->engine->start_new_fetch;
|
||||
}
|
||||
my $current = time();
|
||||
DPB::Core->log_concurrency($current, $state->{concurrent});
|
||||
if ($need_clock) {
|
||||
my $current = time();
|
||||
if ($current >= $last_time + $state->{display_timeout} ||
|
||||
$reaped) {
|
||||
$reporter->report;
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# ex:ts=8 sw=4:
|
||||
# $OpenBSD: Core.pm,v 1.15 2012/09/24 20:41:57 espie Exp $
|
||||
# $OpenBSD: Core.pm,v 1.16 2012/10/08 12:41:03 espie Exp $
|
||||
#
|
||||
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
|
||||
#
|
||||
@ -476,6 +476,26 @@ my $available = [];
|
||||
my %stopped = ();
|
||||
|
||||
my $logdir;
|
||||
my $lastcount = 0;
|
||||
|
||||
sub log_concurrency
|
||||
{
|
||||
my ($class, $time, $fh) = @_;
|
||||
my $j = 0;
|
||||
while (my ($k, $c) = each %{$class->repository}) {
|
||||
$j++;
|
||||
if (defined $c->{swallow}) {
|
||||
$j += $c->{swallow};
|
||||
}
|
||||
if (defined $c->{swallowed}) {
|
||||
$j += scalar(@{$c->{swallowed}});
|
||||
}
|
||||
}
|
||||
if ($j != $lastcount) {
|
||||
print $fh "$$ $time $j\n";
|
||||
$lastcount = $j;
|
||||
}
|
||||
}
|
||||
|
||||
sub set_logdir
|
||||
{
|
||||
@ -579,6 +599,7 @@ sub can_swallow
|
||||
my ($core, $n) = @_;
|
||||
$core->{swallow} = $n;
|
||||
$core->{swallowed} = [];
|
||||
$core->{realjobs} = $n+1;
|
||||
$core->host->{swallow}{$core} = $core;
|
||||
|
||||
# try to reswallow freed things right away.
|
||||
@ -674,7 +695,7 @@ sub has_sf
|
||||
|
||||
sub parse_hosts_file
|
||||
{
|
||||
my ($class, $filename, $state) = @_;
|
||||
my ($class, $filename, $state, $default) = @_;
|
||||
open my $fh, '<', $filename or
|
||||
$state->fatal("Can't read host files #1: #2", $filename, $!);
|
||||
my $_;
|
||||
@ -688,7 +709,8 @@ sub parse_hosts_file
|
||||
$state->{startup_script} = $1;
|
||||
next;
|
||||
}
|
||||
my $prop = {};
|
||||
# copy default [properties
|
||||
my $prop = { %$default };
|
||||
my ($host, @properties) = split(/\s+/, $_);
|
||||
for my $_ (@properties) {
|
||||
if (m/^(.*?)=(.*)$/) {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# ex:ts=8 sw=4:
|
||||
# $OpenBSD: Engine.pm,v 1.49 2012/09/23 18:13:32 espie Exp $
|
||||
# $OpenBSD: Engine.pm,v 1.50 2012/10/08 12:41:03 espie Exp $
|
||||
#
|
||||
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
|
||||
#
|
||||
@ -240,7 +240,7 @@ sub start_build
|
||||
my $special = $self->{engine}{heuristics}->
|
||||
special_parameters($core->host, $v);
|
||||
$self->log('J', $v, " ".$core->hostname." ".$special);
|
||||
$self->{builder}->build($v, $core, $special, $core->parallel,
|
||||
$self->{builder}->build($v, $core, $special,
|
||||
$lock, sub {$self->end($core, $v)});
|
||||
}
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# ex:ts=8 sw=4:
|
||||
# $OpenBSD: Port.pm,v 1.34 2012/09/23 18:13:32 espie Exp $
|
||||
# $OpenBSD: Port.pm,v 1.35 2012/10/08 12:41:03 espie Exp $
|
||||
#
|
||||
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
|
||||
#
|
||||
@ -478,7 +478,7 @@ use Time::HiRes qw(time);
|
||||
|
||||
sub new
|
||||
{
|
||||
my ($class, $log, $v, $builder, $special, $parallel, $endcode) = @_;
|
||||
my ($class, $log, $v, $builder, $special, $core, $endcode) = @_;
|
||||
my $e;
|
||||
if ($builder->{rebuild}) {
|
||||
$e = sub { $builder->register_built($v); &$endcode; };
|
||||
@ -492,22 +492,34 @@ sub new
|
||||
builder => $builder, endcode => $e},
|
||||
$class;
|
||||
|
||||
if ($parallel && $v->{info}{DPB_PROPERTIES}{parallel}) {
|
||||
$job->{parallel} = $parallel;
|
||||
my $prop = $core->prop;
|
||||
if ($prop->{parallel} =~ m/^\/(\d+)$/) {
|
||||
if ($prop->{jobs} == 1) {
|
||||
$prop->{parallel} = 0;
|
||||
} else {
|
||||
$prop->{parallel} = int($prop->{jobs}/$1);
|
||||
if ($prop->{parallel} < 2) {
|
||||
$prop->{parallel} = 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($prop->{parallel} && $v->{info}{DPB_PROPERTIES}{parallel}) {
|
||||
$job->{parallel} = $prop->{parallel};
|
||||
}
|
||||
|
||||
if ($builder->{rebuild}) {
|
||||
push(@{$job->{tasks}},
|
||||
DPB::Task::Port::Signature->new('signature'));
|
||||
} else {
|
||||
$job->add_normal_tasks($builder->{dontclean}{$v->pkgpath});
|
||||
$job->add_normal_tasks($builder->{dontclean}{$v->pkgpath},
|
||||
$prop);
|
||||
}
|
||||
return $job;
|
||||
}
|
||||
|
||||
sub add_normal_tasks
|
||||
{
|
||||
my ($self, $dontclean) = @_;
|
||||
my ($self, $dontclean, $hostprop) = @_;
|
||||
|
||||
my @todo;
|
||||
my $builder = $self->{builder};
|
||||
@ -515,9 +527,9 @@ sub add_normal_tasks
|
||||
push @todo, "clean";
|
||||
}
|
||||
push(@todo, qw(depends prepare show-prepare-results));
|
||||
if ($builder->{junk}) {
|
||||
if ($builder->{junk_count}++ >= $builder->{junk}) {
|
||||
$builder->{junk_count} = 0;
|
||||
if ($hostprop->{junk}) {
|
||||
if ($hostprop->{junk_count}++ >= $hostprop->{junk}) {
|
||||
$hostprop->{junk_count} = 0;
|
||||
push(@todo, 'junk');
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# ex:ts=8 sw=4:
|
||||
# $OpenBSD: PortBuilder.pm,v 1.23 2012/09/23 18:13:32 espie Exp $
|
||||
# $OpenBSD: PortBuilder.pm,v 1.24 2012/10/08 12:41:03 espie Exp $
|
||||
#
|
||||
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
|
||||
#
|
||||
@ -34,7 +34,6 @@ sub new
|
||||
dontclean => $state->{dontclean},
|
||||
fetch => $state->opt('f'),
|
||||
size => $state->opt('s'),
|
||||
junk => $state->opt('J'),
|
||||
rebuild => $state->opt('R'),
|
||||
fullrepo => $state->fullrepo,
|
||||
heuristics => $state->heuristics}, $class;
|
||||
@ -158,6 +157,9 @@ sub report
|
||||
return if $job->{signature_only};
|
||||
my $pkgpath = $v->fullpkgpath;
|
||||
my $host = $core->fullhostname;
|
||||
if ($core->{realjobs}) {
|
||||
$host .= '*'.$core->{realjobs};
|
||||
}
|
||||
my $log = $self->{global};
|
||||
my $sz = (stat $self->logger->log_pkgpath($v))[7];
|
||||
if (defined $job->{offset}) {
|
||||
@ -195,7 +197,7 @@ sub end_lock
|
||||
|
||||
sub build
|
||||
{
|
||||
my ($self, $v, $core, $special, $parallel, $lock, $final_sub) = @_;
|
||||
my ($self, $v, $core, $special, $lock, $final_sub) = @_;
|
||||
my $start = time();
|
||||
my $log = $self->logger->make_logs($v);
|
||||
|
||||
@ -205,7 +207,7 @@ sub build
|
||||
close($fh);
|
||||
|
||||
my $job;
|
||||
$job = DPB::Job::Port->new($log, $v, $self, $special, $parallel,
|
||||
$job = DPB::Job::Port->new($log, $v, $self, $special, $core,
|
||||
sub {$self->end_lock($lock, $core, $job); $self->report($v, $job, $core); &$final_sub;});
|
||||
$job->{lock} = $lock;
|
||||
$core->start_job($job, $v);
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
.\" $OpenBSD: dpb.1,v 1.41 2012/08/31 17:27:50 espie Exp $
|
||||
.\" $OpenBSD: dpb.1,v 1.42 2012/10/08 12:41:03 espie Exp $
|
||||
.\"
|
||||
.\" Copyright (c) 2010 Marc Espie <espie@openbsd.org>
|
||||
.\"
|
||||
@ -14,7 +14,7 @@
|
||||
.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
.\"
|
||||
.Dd $Mdocdate: August 31 2012 $
|
||||
.Dd $Mdocdate: October 8 2012 $
|
||||
.Dt DPB 1
|
||||
.Os
|
||||
.Sh NAME
|
||||
@ -237,6 +237,11 @@ Architecture of the concerned host.
|
||||
currently this has to be set manually on heterogeneous networks.)
|
||||
.It jobs=n
|
||||
Number of jobs to run on that host, defaults to hw.ncpu.
|
||||
.It junk=n
|
||||
Junk unused packages each n steps.
|
||||
See
|
||||
.Fl J
|
||||
option.
|
||||
.It memory=thr
|
||||
Builds everything below that wrkdir threshold in /tmp, assuming
|
||||
it is a memory filesystem.
|
||||
@ -244,6 +249,11 @@ Avoid for now, as
|
||||
.Xr mfs 8
|
||||
has serious race conditions which yield
|
||||
random errors under stress conditions such as bulk build.
|
||||
.It parallel=p
|
||||
Run big ports on several cores.
|
||||
See
|
||||
.Fl p
|
||||
option.
|
||||
.It sf=n
|
||||
Speed factor.
|
||||
An estimate of that machine's speed with that number of jobs
|
||||
@ -276,6 +286,9 @@ List of
|
||||
to install, on the local box.
|
||||
This will also add them to the list of things to build.
|
||||
.It Fl J Ar p
|
||||
Default value for the
|
||||
.Dq junk
|
||||
propery.
|
||||
Delete unneeded installed packages during the build.
|
||||
Each
|
||||
.Ar prepare
|
||||
@ -314,7 +327,7 @@ Defaults to
|
||||
Can be disabled by setting to
|
||||
.Ar 0 .
|
||||
.It Fl j Ar n
|
||||
Number of concurrent local jobs to run (defaults to hw.ncpu if no hosts file).
|
||||
Number of jobs to run on a single host (defaults to hw.ncpu).
|
||||
.It Fl L Ar logdir
|
||||
Choose a log directory.
|
||||
.Po
|
||||
@ -334,6 +347,34 @@ Force tty-style reporting.
|
||||
Read list of
|
||||
.Xr pkgpath 7
|
||||
from file.
|
||||
.It Fl p Ar parallel
|
||||
Default value for the
|
||||
.Dq parallel
|
||||
property.
|
||||
.Pp
|
||||
Run big jobs on several cores on the same host, by using
|
||||
MAKE_JOBS=k .
|
||||
.Pp
|
||||
Once such a job has started,
|
||||
.Nm
|
||||
will not start new jobs on the same host until the big job has
|
||||
stolen enough cores from other finishing jobs.
|
||||
.Pp
|
||||
Only big ports which are safe for parallel building (annotated with
|
||||
DPB_PROPERTIES = parallel in their Makefile will be affected.
|
||||
.Pp
|
||||
It is advisable to set k to an integral fraction of the
|
||||
number of cores available on a given host.
|
||||
.Ar parameter
|
||||
can be an integer, or of the form
|
||||
.Sq /n ,
|
||||
in which case,
|
||||
.Nm
|
||||
will set k to a fraction of the total number of jobs
|
||||
on the machine, but never below 2.
|
||||
.Pp
|
||||
Defaults to
|
||||
.Sq /2 .
|
||||
.It Fl q
|
||||
Don't quit while errors/locks are around.
|
||||
.It Fl r
|
||||
|
||||
Loading…
Reference in New Issue
Block a user