From cdc017bcb0c9ad024809dc0a0f0f0a86a1a3c80b Mon Sep 17 00:00:00 2001 From: espie Date: Mon, 8 Oct 2012 12:41:03 +0000 Subject: [PATCH] small change for people running multiple hosts: options on the command line now define *defaults* that host files can override (for instance -j, stuck, -p, -J). Add -p /n to mean "take number of jobs, if >1, divide by n, round up to 2, and use that for parallel. Document -p. Make junk be 'by host' (and it's a prop, so you can tweak it). concurrent log that records how many jobs are running each time it changes. tag parallel builds *n in the time record. --- infrastructure/bin/dpb | 37 ++++++++++++--------- infrastructure/lib/DPB/Core.pm | 28 ++++++++++++++-- infrastructure/lib/DPB/Engine.pm | 4 +-- infrastructure/lib/DPB/Job/Port.pm | 30 ++++++++++++----- infrastructure/lib/DPB/PortBuilder.pm | 10 +++--- infrastructure/man/man1/dpb.1 | 47 +++++++++++++++++++++++++-- 6 files changed, 120 insertions(+), 36 deletions(-) diff --git a/infrastructure/bin/dpb b/infrastructure/bin/dpb index 6efcae6caba..41d0a3ae1b2 100755 --- a/infrastructure/bin/dpb +++ b/infrastructure/bin/dpb @@ -1,7 +1,7 @@ #! /usr/bin/perl # ex:ts=8 sw=4: -# $OpenBSD: dpb,v 1.53 2012/09/23 18:13:32 espie Exp $ +# $OpenBSD: dpb,v 1.54 2012/10/08 12:41:03 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -163,9 +163,9 @@ sub handle_options }, }; - $state->SUPER::handle_options('acemqrRstuUvh:xA:C:f:F:I:j:J:M:p:P:b:L:S:', + $state->SUPER::handle_options('acemqrRsuUvh:xA:C:f:F:I:j:J:M:p:P:b:L:S:', "[-acemrRsuUvx] [-A arch] [-C plist] [-f N] [-F N] [-I plist] [-J p] [-j N]", - "[-P plist] [-h hosts] [-L logdir] [-b log] [-t ctimeout] [-M threshold]", + "[-p parallel] [-P plist] [-h hosts] [-L logdir] [-b log] [-M threshold]", "[path ...]"); $state->{fullrepo} = join("/", $state->{repo}, $state->arch, "all"); $state->{logdir} //= $ENV{LOGDIR} // '%p/logs/%a'; @@ -180,6 +180,7 @@ sub handle_options } $state->{opt}{f} //= 2; $state->{opt}{J} //= 250; + $state->{opt}{p} //= '/2'; if (defined $state->opt('j')) { if ($state->localarch ne $state->arch) { $state->usage( @@ -224,37 +225,42 @@ sub handle_options $state->{logger} = DPB::Logger->new($state->logdir, $state->opt('c')); $state->heuristics->set_logger($state->logger); $state->{display_timeout} = - $state->{subst}->value('DISPLAY_TIMEOUT') // $state->opt('T') // 10; + $state->{subst}->value('DISPLAY_TIMEOUT') // 10; $state->{connection_timeout} = - $state->{subst}->value('CONNECTION_TIMEOUT') // $state->opt('t'); + $state->{subst}->value('CONNECTION_TIMEOUT'); $state->{stuck_timeout} = $state->{subst}->value('STUCK_TIMEOUT'); $state->{build_once} = $state->{all}; if ($state->defines("DONT_BUILD_ONCE")) { $state->{build_once} = 0; } + $state->{concurrent} = $state->{logger}->open("concurrent"); } sub start_cores { my $state = shift; - if ($state->{config}) { - DPB::Core->parse_hosts_file($state->{config}, $state); - } + my $default_prop = {}; - my $prop = {}; if ($state->opt('j')) { - $prop->{jobs} = $state->opt('j'); + $default_prop->{jobs} = $state->opt('j'); } if ($state->opt('p')) { - $prop->{parallel} = $state->opt('p'); + $default_prop->{parallel} = $state->opt('p'); } if ($state->{stuck_timeout}) { - $prop->{stuck} = $state->{stuck_timeout}; + $default_prop->{stuck} = $state->{stuck_timeout}; + } + if ($state->opt('J')) { + $default_prop->{junk} = $state->opt('J'); } - if ($state->opt('j') || !$state->{config}) { - DPB::Core::Factory->new('localhost', $prop); + if ($state->{config}) { + DPB::Core->parse_hosts_file($state->{config}, $state, $default_prop); + } + + if (!$state->{config}) { + DPB::Core::Factory->new('localhost', $default_prop); } DPB::Core::Factory->init_cores($state); } @@ -564,8 +570,9 @@ sub handle_non_waiting_jobs } $state->engine->start_new_fetch; } + my $current = time(); + DPB::Core->log_concurrency($current, $state->{concurrent}); if ($need_clock) { - my $current = time(); if ($current >= $last_time + $state->{display_timeout} || $reaped) { $reporter->report; diff --git a/infrastructure/lib/DPB/Core.pm b/infrastructure/lib/DPB/Core.pm index a9903e1c6a6..9cdaeca0a40 100644 --- a/infrastructure/lib/DPB/Core.pm +++ b/infrastructure/lib/DPB/Core.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: Core.pm,v 1.15 2012/09/24 20:41:57 espie Exp $ +# $OpenBSD: Core.pm,v 1.16 2012/10/08 12:41:03 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -476,6 +476,26 @@ my $available = []; my %stopped = (); my $logdir; +my $lastcount = 0; + +sub log_concurrency +{ + my ($class, $time, $fh) = @_; + my $j = 0; + while (my ($k, $c) = each %{$class->repository}) { + $j++; + if (defined $c->{swallow}) { + $j += $c->{swallow}; + } + if (defined $c->{swallowed}) { + $j += scalar(@{$c->{swallowed}}); + } + } + if ($j != $lastcount) { + print $fh "$$ $time $j\n"; + $lastcount = $j; + } +} sub set_logdir { @@ -579,6 +599,7 @@ sub can_swallow my ($core, $n) = @_; $core->{swallow} = $n; $core->{swallowed} = []; + $core->{realjobs} = $n+1; $core->host->{swallow}{$core} = $core; # try to reswallow freed things right away. @@ -674,7 +695,7 @@ sub has_sf sub parse_hosts_file { - my ($class, $filename, $state) = @_; + my ($class, $filename, $state, $default) = @_; open my $fh, '<', $filename or $state->fatal("Can't read host files #1: #2", $filename, $!); my $_; @@ -688,7 +709,8 @@ sub parse_hosts_file $state->{startup_script} = $1; next; } - my $prop = {}; + # copy default [properties + my $prop = { %$default }; my ($host, @properties) = split(/\s+/, $_); for my $_ (@properties) { if (m/^(.*?)=(.*)$/) { diff --git a/infrastructure/lib/DPB/Engine.pm b/infrastructure/lib/DPB/Engine.pm index 41e6ce5475d..37633e8b085 100644 --- a/infrastructure/lib/DPB/Engine.pm +++ b/infrastructure/lib/DPB/Engine.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: Engine.pm,v 1.49 2012/09/23 18:13:32 espie Exp $ +# $OpenBSD: Engine.pm,v 1.50 2012/10/08 12:41:03 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -240,7 +240,7 @@ sub start_build my $special = $self->{engine}{heuristics}-> special_parameters($core->host, $v); $self->log('J', $v, " ".$core->hostname." ".$special); - $self->{builder}->build($v, $core, $special, $core->parallel, + $self->{builder}->build($v, $core, $special, $lock, sub {$self->end($core, $v)}); } diff --git a/infrastructure/lib/DPB/Job/Port.pm b/infrastructure/lib/DPB/Job/Port.pm index de44504366c..04eb52b87e9 100644 --- a/infrastructure/lib/DPB/Job/Port.pm +++ b/infrastructure/lib/DPB/Job/Port.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: Port.pm,v 1.34 2012/09/23 18:13:32 espie Exp $ +# $OpenBSD: Port.pm,v 1.35 2012/10/08 12:41:03 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -478,7 +478,7 @@ use Time::HiRes qw(time); sub new { - my ($class, $log, $v, $builder, $special, $parallel, $endcode) = @_; + my ($class, $log, $v, $builder, $special, $core, $endcode) = @_; my $e; if ($builder->{rebuild}) { $e = sub { $builder->register_built($v); &$endcode; }; @@ -492,22 +492,34 @@ sub new builder => $builder, endcode => $e}, $class; - if ($parallel && $v->{info}{DPB_PROPERTIES}{parallel}) { - $job->{parallel} = $parallel; + my $prop = $core->prop; + if ($prop->{parallel} =~ m/^\/(\d+)$/) { + if ($prop->{jobs} == 1) { + $prop->{parallel} = 0; + } else { + $prop->{parallel} = int($prop->{jobs}/$1); + if ($prop->{parallel} < 2) { + $prop->{parallel} = 2; + } + } + } + if ($prop->{parallel} && $v->{info}{DPB_PROPERTIES}{parallel}) { + $job->{parallel} = $prop->{parallel}; } if ($builder->{rebuild}) { push(@{$job->{tasks}}, DPB::Task::Port::Signature->new('signature')); } else { - $job->add_normal_tasks($builder->{dontclean}{$v->pkgpath}); + $job->add_normal_tasks($builder->{dontclean}{$v->pkgpath}, + $prop); } return $job; } sub add_normal_tasks { - my ($self, $dontclean) = @_; + my ($self, $dontclean, $hostprop) = @_; my @todo; my $builder = $self->{builder}; @@ -515,9 +527,9 @@ sub add_normal_tasks push @todo, "clean"; } push(@todo, qw(depends prepare show-prepare-results)); - if ($builder->{junk}) { - if ($builder->{junk_count}++ >= $builder->{junk}) { - $builder->{junk_count} = 0; + if ($hostprop->{junk}) { + if ($hostprop->{junk_count}++ >= $hostprop->{junk}) { + $hostprop->{junk_count} = 0; push(@todo, 'junk'); } } diff --git a/infrastructure/lib/DPB/PortBuilder.pm b/infrastructure/lib/DPB/PortBuilder.pm index 11ce007120b..8129e1c5589 100644 --- a/infrastructure/lib/DPB/PortBuilder.pm +++ b/infrastructure/lib/DPB/PortBuilder.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: PortBuilder.pm,v 1.23 2012/09/23 18:13:32 espie Exp $ +# $OpenBSD: PortBuilder.pm,v 1.24 2012/10/08 12:41:03 espie Exp $ # # Copyright (c) 2010 Marc Espie # @@ -34,7 +34,6 @@ sub new dontclean => $state->{dontclean}, fetch => $state->opt('f'), size => $state->opt('s'), - junk => $state->opt('J'), rebuild => $state->opt('R'), fullrepo => $state->fullrepo, heuristics => $state->heuristics}, $class; @@ -158,6 +157,9 @@ sub report return if $job->{signature_only}; my $pkgpath = $v->fullpkgpath; my $host = $core->fullhostname; + if ($core->{realjobs}) { + $host .= '*'.$core->{realjobs}; + } my $log = $self->{global}; my $sz = (stat $self->logger->log_pkgpath($v))[7]; if (defined $job->{offset}) { @@ -195,7 +197,7 @@ sub end_lock sub build { - my ($self, $v, $core, $special, $parallel, $lock, $final_sub) = @_; + my ($self, $v, $core, $special, $lock, $final_sub) = @_; my $start = time(); my $log = $self->logger->make_logs($v); @@ -205,7 +207,7 @@ sub build close($fh); my $job; - $job = DPB::Job::Port->new($log, $v, $self, $special, $parallel, + $job = DPB::Job::Port->new($log, $v, $self, $special, $core, sub {$self->end_lock($lock, $core, $job); $self->report($v, $job, $core); &$final_sub;}); $job->{lock} = $lock; $core->start_job($job, $v); diff --git a/infrastructure/man/man1/dpb.1 b/infrastructure/man/man1/dpb.1 index a439537b67b..490e794dcd5 100644 --- a/infrastructure/man/man1/dpb.1 +++ b/infrastructure/man/man1/dpb.1 @@ -1,4 +1,4 @@ -.\" $OpenBSD: dpb.1,v 1.41 2012/08/31 17:27:50 espie Exp $ +.\" $OpenBSD: dpb.1,v 1.42 2012/10/08 12:41:03 espie Exp $ .\" .\" Copyright (c) 2010 Marc Espie .\" @@ -14,7 +14,7 @@ .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. .\" -.Dd $Mdocdate: August 31 2012 $ +.Dd $Mdocdate: October 8 2012 $ .Dt DPB 1 .Os .Sh NAME @@ -237,6 +237,11 @@ Architecture of the concerned host. currently this has to be set manually on heterogeneous networks.) .It jobs=n Number of jobs to run on that host, defaults to hw.ncpu. +.It junk=n +Junk unused packages each n steps. +See +.Fl J +option. .It memory=thr Builds everything below that wrkdir threshold in /tmp, assuming it is a memory filesystem. @@ -244,6 +249,11 @@ Avoid for now, as .Xr mfs 8 has serious race conditions which yield random errors under stress conditions such as bulk build. +.It parallel=p +Run big ports on several cores. +See +.Fl p +option. .It sf=n Speed factor. An estimate of that machine's speed with that number of jobs @@ -276,6 +286,9 @@ List of to install, on the local box. This will also add them to the list of things to build. .It Fl J Ar p +Default value for the +.Dq junk +propery. Delete unneeded installed packages during the build. Each .Ar prepare @@ -314,7 +327,7 @@ Defaults to Can be disabled by setting to .Ar 0 . .It Fl j Ar n -Number of concurrent local jobs to run (defaults to hw.ncpu if no hosts file). +Number of jobs to run on a single host (defaults to hw.ncpu). .It Fl L Ar logdir Choose a log directory. .Po @@ -334,6 +347,34 @@ Force tty-style reporting. Read list of .Xr pkgpath 7 from file. +.It Fl p Ar parallel +Default value for the +.Dq parallel +property. +.Pp +Run big jobs on several cores on the same host, by using +MAKE_JOBS=k . +.Pp +Once such a job has started, +.Nm +will not start new jobs on the same host until the big job has +stolen enough cores from other finishing jobs. +.Pp +Only big ports which are safe for parallel building (annotated with +DPB_PROPERTIES = parallel in their Makefile will be affected. +.Pp +It is advisable to set k to an integral fraction of the +number of cores available on a given host. +.Ar parameter +can be an integer, or of the form +.Sq /n , +in which case, +.Nm +will set k to a fraction of the total number of jobs +on the machine, but never below 2. +.Pp +Defaults to +.Sq /2 . .It Fl q Don't quit while errors/locks are around. .It Fl r