cpet/openbsd-ports
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

work-in-progress: new distributed ports builder.

Browse Source 
will replace old dpb once we're satisfied it works.
some features:
- no more waiting, starts building right away, the dependencies discovery
process is just another job,
- monitor display that makes it easy to know when jobs get stuck, or machines
get down.
- loads of logs,
- locks that stay around in case of error, and that you can remove when
the error has been fixed (dpb picks it up),
- smart choice of which package to build,
- can take into account previous build times,
- builds on machines of differing speed, offloading "lighter" jobs to slower
boxes.

(again, ports is still locked for most people)
This commit is contained in:
espie 2010-02-24 11:33:31 +00:00
parent a2bd8e8fc3
commit 61d822e8d5
17 changed files with 3759 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

492
infrastructure/build/DPB/Core.pm Normal file
View File

@ -0,0 +1,492 @@
# ex:ts=8 sw=4:
# $OpenBSD: Core.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
# here, a "core" is an entity responsible for scheduling cpu, such as
# running a job, which is a collection of tasks.
# the "abstract core" part only sees about registering/unregistering cores,
# and having a global event handler that gets run whenever possible.
package DPB::Core::Abstract;
use POSIX ":sys_wait_h";
use OpenBSD::Error;
use DPB::Util;
use DPB::Job;
# note that we play dangerously, e.g., we only keep cores that are running
# something in there, the code can keep some others.
my ($running, $special) = ({}, {});
sub repositories
{
return ($running, $special);
}
my @extra_stuff = ();
sub register_event
{
my ($class, $code) = @_;
push(@extra_stuff, $code);
}
sub handle_events
{
for my $code (@extra_stuff) {
&$code;
}
}
sub is_alive
{
return 1;
}
sub new
{
my ($class, $host) = @_;
bless {host => $host}, $class;
}
sub host
{
my $self = shift;
return $self->{host};
}
sub register
{
my ($self, $pid) = @_;
$self->{pid} = $pid;
$self->repository->{$self->{pid}} = $self;
}
sub unregister
{
my ($self, $status) = @_;
delete $self->repository->{$self->{pid}};
delete $self->{pid};
$self->{status} = $status;
return $self;
}
sub terminate
{
my $self = shift;
if (defined $self->{pid}) {
waitpid($self->{pid}, 0);
$self->unregister($?);
return $self;
} else {
return undef;
}
}
sub reap_kid
{
my ($class, $kid) = @_;
if (defined $kid && $kid > 0) {
for my $repo ($class->repositories) {
if (defined $repo->{$kid}) {
$repo->{$kid}->unregister($?)->continue;
last;
}
}
}
return $kid;
}
my $inited = 0;
sub reap
{
my ($class, $all) = @_;
my $reaped = 0;
if (!$inited) {
DPB::Core::Factory->init_cores;
}
$class->handle_events;
$reaped++ while $class->reap_kid(waitpid(-1, WNOHANG)) > 0;
return $reaped;
}
sub reap_wait
{
my ($class, $reporter) = @_;
return $class->reap_kid(waitpid(-1, 0));
}
sub cleanup
{
my $class = shift;
for my $repo ($class->repositories) {
for my $pid (keys %$repo) {
kill INT => $pid;
}
}
}
OpenBSD::Handler->register( sub { __PACKAGE__->cleanup });
# this is a core that can run jobs
package DPB::Core::WithJobs;
our @ISA = qw(DPB::Core::Abstract);
sub fh
{
my $self = shift;
return $self->task->{fh};
}
sub job
{
my $self = shift;
return $self->{job};
}
sub task
{
my $self = shift;
return $self->{task};
}
sub terminate
{
my $self = shift;
$self->task->end if $self->task;
if ($self->SUPER::terminate) {
$self->job->finalize($self);
$self->mark_ready;
}
}
sub run_task
{
my ($core, $task) = @_;
$core->{task} = $task;
my $pid = $task->fork($core);
if (!defined $pid) {
die "Oops: task couldn't start\n";
} elsif ($pid == 0) {
for my $sig (keys %SIG) {
$SIG{$sig} = 'DEFAULT';
}
if (!$task->run($core)) {
exit(1);
}
exit(0);
} else {
$core->register($pid);
}
}
sub continue
{
my $core = shift;
my $task = $core->job->next_task($core);
if (defined $task) {
return $core->run_task($task);
} else {
$core->job->finalize($core);
}
}
sub mark_ready
{
my $self = shift;
delete $self->{job};
return $self;
}
use Time::HiRes qw(time);
sub start_job
{
my ($core, $job) = @_;
$core->{job} = $job;
$core->{started} = time;
$core->{status} = 0;
$core->continue;
}
sub start_clock
{
my ($class, $tm) = @_;
DPB::Core::Clock->start($tm);
}
package DPB::Job::Init;
our @ISA = qw(DPB::Job::Normal);
# no tasks for now
sub next_task
{
return undef;
}
sub new
{
my $class = shift;
bless {name => "init"}, $class;
}
# if everything is okay, we mark our jobs as ready
sub finalize
{
my ($self, $core) = @_;
for my $i (@{$core->{list}}) {
$i->mark_ready;
}
}
# this is a weird one !
package DPB::Core::Factory;
our @ISA = qw(DPB::Core::WithJobs);
my $init = {};
sub new
{
my ($class, $host, $prop) = @_;
my $cloner;
if ($host eq "localhost" or $host eq DPB::Core::Local->host) {
$cloner = $init->{localhost} //= DPB::Core::Local->new_noreg($host, $prop);
} else {
require DPB::Core::Distant;
$cloner = $init->{$host} //= DPB::Core::Distant->new_noreg($host, $prop);
}
my $o = ref($cloner)->new($host, $prop);
push(@{$cloner->{list}}, $o);
return $o;
}
sub init_cores
{
for my $core (values %$init) {
$core->start_job(DPB::Job::Init->new);
}
$inited = 1;
}
package DPB::Core;
our @ISA = qw(DPB::Core::WithJobs);
my @available = ();
my @extra_report = ();
my @extra_important = ();
sub register_report
{
my ($self, $code, $important) = @_;
push (@extra_report, $code);
push (@extra_important, $important);
}
sub repository
{
return $running;
}
sub one_core
{
my ($core, $time) = @_;
return $core->job->name." [$core->{pid}] on ".$core->host.
$core->job->watched($time);
}
sub report
{
my $current = time();
my $s = join("\n", map {one_core($_, $current)} sort {$a->{started} <=> $b->{started}} values %$running). "\n";
for my $a (@extra_report) {
$s .= &$a;
}
return $s;
}
sub important
{
my $current = time();
my $s = '';
for my $j (values %$running) {
if ($j->job->really_watch($current)) {
$s .= one_core($j, $current)."\n";
}
}
for my $a (@extra_important) {
$s .= &$a;
}
return $s;
}
sub mark_ready
{
my $self = shift;
$self->SUPER::mark_ready;
push(@available, $self);
return $self;
}
sub avail
{
my $self = shift;
return @available > 0;
}
sub running
{
return scalar(%$running);
}
sub get
{
if (@available > 1) {
@available = sort {$b->{sf} <=> $a->{sf}} @available;
}
return shift @available;
}
my @all_cores = ();
sub all_sf
{
my $l = [];
for my $j (@all_cores) {
next unless $j->is_alive;
push(@$l, $j->{sf});
}
return [sort {$a <=> $b} @$l];
}
sub new
{
my ($class, $host, $prop) = @_;
my $o = $class->SUPER::new($host);
$o->{sf} //= $prop->{sf};
$o->{sf} //= 1;
if (defined $prop->{memory}) {
$o->{memory} = $prop->{memory};
}
push(@all_cores, $o);
return $o;
}
sub new_noreg
{
my ($class, $host, $prop) = @_;
$class->SUPER::new($host);
}
my $has_sf = 0;
sub has_sf
{
return $has_sf;
}
sub parse_hosts_file
{
my ($class, $filename, $arch) = @_;
open my $fh, '<', $filename or die "Can't read host files $filename\n";
my $_;
my $sf;
my $cores = {};
while (<$fh>) {
chomp;
s/\s*\#.*$//;
next if m/^$/;
my $prop = {};
my ($host, @properties) = split(/\s+/, $_);
for my $_ (@properties) {
if (m/^(.*?)=(.*)$/) {
$prop->{$1} = $2;
}
}
if (defined $prop->{arch} && $prop->{arch} != $arch) {
next;
}
$prop->{jobs} //= 1;
if (defined $prop->{mem}) {
$prop->{memory} = $prop->{mem};
}
$sf //= $prop->{sf};
if (defined $prop->{sf} && $prop->{sf} != $sf) {
$has_sf = 1;
}
for my $j (1 .. $prop->{jobs}) {
DPB::Core::Factory->new($host, $prop);
}
}
DPB::Core::Factory->init_cores;
}
sub start_pipe
{
my ($self, $code, $name) = @_;
$self->start_job(DPB::Job::Pipe->new($code, $name));
}
sub start
{
my ($self, $code, $endcode, $name) = @_;
$self->start_job(DPB::Job::Normal->new($code, $endcode, $name));
}
package DPB::Core::Special;
our @ISA = qw(DPB::Core::WithJobs);
sub repository
{
return $special;
}
package DPB::Core::Local;
our @ISA = qw(DPB::Core);
my $host;
sub host
{
if (!defined $host) {
chomp($host = `hostname`);
}
return $host;
}
package DPB::Job::Clock;
our @ISA = qw(DPB::Job::Infinite);
sub new
{
my ($class, $timeout) = @_;
$timeout //= 10;
return $class->SUPER::new(sub {
sleep($timeout);
exit(0);
}, 'clock');
}
package DPB::Core::Clock;
our @ISA = qw(DPB::Core::Special);
sub start
{
my ($class, $timeout) = @_;
my $core = $class->new('localhost');
$core->start_job(DPB::Job::Clock->new($timeout));
}
1;

225
infrastructure/build/DPB/Core/Distant.pm Normal file
View File

@ -0,0 +1,225 @@
# ex:ts=8 sw=4:
# $OpenBSD: Distant.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
use DPB::Core;
use OpenBSD::Paths;
package DPB::Ssh;
sub ssh
{
my ($class, $socket, $timeout) = @_;
return ('ssh', '-o', "connectTimeout=3",
'-o', "serverAliveInterval=3",
'-S', $socket);
}
# '-o', 'clearAllForwardings=yes',
# '-o', 'EscapeChar=none',
sub new
{
my ($class, $host) = @_;
bless {master => DPB::Ssh::Master->find($host)}, $class;
}
sub is_alive
{
shift->{master}->is_alive;
}
sub socket
{
shift->{master}->socket;
}
sub timeout
{
shift->{master}->timeout;
}
sub host
{
shift->{master}->host;
}
sub run
{
my ($self, $cmd) = @_;
exec {OpenBSD::Paths->ssh}
($self->ssh($self->socket, $self->timeout),
$self->host, $cmd);
}
sub make
{
my $self = shift;
return OpenBSD::Paths->make;
}
package DPB::Job::SshMaster;
our @ISA = qw(DPB::Job::Infinite);
my $TMPDIR;
sub new
{
my ($class, $host) = @_;
$TMPDIR //= $ENV{PKG_TMPDIR} || '/var/tmp';
my $timeout = 60;
my $socket = "$TMPDIR/ssh-$host";
my $o = $class->SUPER::new(sub {
close STDOUT;
close STDERR;
open STDOUT, '>/dev/null';
open STDERR, '>&STDOUT';
exec {OpenBSD::Paths->ssh}
(DPB::Ssh->ssh($socket, $timeout),
'-N', '-M', $host);
exit(1);
}, "ssh master for $host");
$o->{host} = $host;
$o->{timeout} = $timeout;
$o->{socket} = $socket;
return $o;
}
package DPB::Ssh::Master;
our @ISA = qw(DPB::Core::Special);
my $master = {};
sub socket
{
my $self = shift;
return $self->job->{socket};
}
sub timeout
{
my $self = shift;
return $self->job->{timeout};
}
sub is_alive
{
my $self = shift;
return -e $self->socket;
}
sub create
{
my ($class, $host) = @_;
my $core = $class->SUPER::new($host);
$core->start_job(DPB::Job::SshMaster->new($host));
}
sub find
{
my ($class, $host, $timeout) = @_;
$master->{$host} //= $class->create($host, $timeout);
}
sub alive_hosts
{
my @l = ();
for my $shell (values %$master) {
my $host = $shell->host;
if ($shell->is_alive) {
push(@l, $host);
} else {
push(@l, $host.'-');
}
}
return "Distant hosts: ".join(' ', sort(@l))."\n";
}
sub changed_hosts
{
my @l = ();
for my $shell (values %$master) {
my $host = $shell->host;
my $was_alive = $shell->{is_alive};
if ($shell->is_alive) {
$shell->{is_alive} = 1;
} else {
$shell->{is_alive} = 0;
}
if ($was_alive && !$shell->{is_alive}) {
push(@l, "$host went down\n");
} elsif (!$was_alive && $shell->{is_alive}) {
push(@l, "$host came up\n");
}
}
return join('', sort(@l));
}
DPB::Core->register_report(\&alive_hosts, \&changed_hosts);
package DPB::Core::Distant;
our @ISA = qw(DPB::Core);
my @dead_cores = ();
sub new
{
my ($class, $host, $prop) = @_;
my $o = $class->SUPER::new($host, $prop);
$o->{shell} = DPB::Ssh->new($host);
return $o;
}
sub new_noreg
{
my ($class, $host, $prop) = @_;
my $o = $class->SUPER::new_noreg($host, $prop);
$o->{shell} = DPB::Ssh->new($host);
return $o;
}
sub is_alive
{
my $self = shift;
return $self->{shell}->is_alive;
}
sub mark_ready
{
my $self = shift;
if ($self->is_alive) {
$self->SUPER::mark_ready;
} else {
delete $self->{job};
# DPB::Reporter->myprint("Found dead core on ".$self->{shell}->host."\n");
push(@dead_cores, $self);
return undef;
}
}
sub check_dead_hosts
{
# DPB::Reporter->myprint("Checking dead hosts\n");
my @redo = @dead_cores;
@dead_cores = ();
for my $core (@redo) {
$core->mark_ready;
}
}
DPB::Core->register_event(\&check_dead_hosts);
1;

421
infrastructure/build/DPB/Engine.pm Normal file
View File

@ -0,0 +1,421 @@
# ex:ts=8 sw=4:
# $OpenBSD: Engine.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
package DPB::Engine;
use DPB::Heuristics;
use DPB::Util;
sub new
{
my ($class, $builder, $heuristics, $logger, $locker) = @_;
my $o = bless {built => {}, tobuild => {},
buildable => $heuristics->new_queue,
later => {}, building => {},
installable => {}, builder => $builder,
packages => {},
all => {},
heuristics => $heuristics,
locker => $locker,
logger => $logger,
errors => [],
ignored => []}, $class;
$o->{log} = DPB::Util->make_hot($logger->open("engine"));
$o->{stats} = DPB::Util->make_hot($logger->open("stats"));
return $o;
}
sub log_no_ts
{
my ($self, $kind, $v, $extra) = @_;
$extra //= '';
my $fh = $self->{log};
print $fh "$$\@$self->{ts}: $kind: ", $v->fullpkgpath, "$extra\n";
}
sub log
{
my $self = shift;
$self->{ts} = time();
$self->log_no_ts(@_);
}
sub count
{
my ($self, $field) = @_;
my $r = $self->{$field};
if (ref($r) eq 'HASH') {
return scalar keys %$r;
} elsif (ref($r) eq 'ARRAY') {
return scalar @$r;
} else {
return "?";
}
}
sub errors_string
{
my $self = shift;
my @l = ();
for my $e (@{$self->{errors}}) {
if (defined $e->{host}) {
push(@l, $e->fullpkgpath."($e->{host})");
} else {
push(@l, $e->fullpkgpath);
}
}
return join(' ', @l);
}
sub report
{
my $self = shift;
return join(" ",
"P=".$self->count("packages"),
"I=".$self->count("installable"),
"B=".$self->count("built"),
"Q=".$self->{buildable}->count,
"T=".$self->count("tobuild"),
"!=".$self->count("ignored"))."\n".
"E=".$self->errors_string."\n";
}
sub stats
{
my $self = shift;
my $fh = $self->{stats};
$self->{statline} //= "";
my $line = join(" ",
"P=".$self->count("packages"),
"I=".$self->count("installable"),
"B=".$self->count("built"),
"Q=".$self->{buildable}->count,
"T=".$self->count("tobuild"));
if ($line ne $self->{statline}) {
$self->{statline} = $line;
print $fh $self->{ts}, " ", $line, "\n";
}
}
my $done_scanning = 0;
sub finished_scanning
{
my $self = shift;
$done_scanning = 1;
# this is scary, we need to do it by-pkgname
my $needed_by = {};
my $bneeded_by = {};
for my $v (values %{$self->{all}}) {
# also, this is an approximation, we could be more specific wrt
# BUILD/RUN_DEPENDS, this leads to more code in check_buildable...
for my $kind (qw(RUN_DEPENDS LIB_DEPENDS)) {
next unless defined $v->{info}{$kind};
for my $depend (values %{$v->{info}{$kind}}) {
next if $depend eq $v;
$needed_by->{$depend->fullpkgname}{$v} = $v;
}
}
if (defined $v->{info}{BUILD_DEPENDS}) {
for my $depend (values %{$v->{info}{BUILD_DEPENDS}}) {
next if $depend eq $v;
$bneeded_by->{$depend->fullpkgname}{$v} = $v;
}
}
}
# then we link each pkgpath to its array
for my $v (values %{$self->{all}}) {
if (defined $needed_by->{$v->fullpkgname}) {
$v->{info}{NEEDED_BY} = $needed_by->{$v->fullpkgname};
bless $v->{info}{NEEDED_BY}, "AddDepends";
}
if (defined $bneeded_by->{$v->fullpkgname}) {
$v->{info}{BNEEDED_BY} = $bneeded_by->{$v->fullpkgname};
bless $v->{info}{BNEEDED_BY}, "AddDepends";
}
}
}
sub important
{
my $self = shift;
$self->{lasterrors} //= 0;
if (@{$self->{errors}} > $self->{lasterrors}) {
my $i = 0;
my @msg;
for my $v (@{$self->{errors}}) {
next if $i++ < $self->{lasterrors};
push(@msg, $v->fullpkgpath);
}
$self->{lasterrors} = @{$self->{errors}};
return "Error in ".join(' ', @msg)."\n";
}
}
sub adjust
{
my ($self, $v, $kind) = @_;
return 0 if !exists $v->{info}{$kind};
my $not_yet = 0;
for my $d (values %{$v->{info}{$kind}}) {
$self->{heuristics}->mark_depend($d, $v);
if ($self->{installable}{$d} ||
(defined $d->{info} &&
$d->fullpkgname eq $v->fullpkgname)) {
delete $v->{info}{$kind}{$d};
} else {
$not_yet++;
}
}
return $not_yet if $not_yet;
delete $v->{info}{$kind};
return 0;
}
sub adjust_extra
{
my ($self, $v, $kind) = @_;
return 0 if !exists $v->{info}{$kind};
my $not_yet = 0;
for my $d (values %{$v->{info}{$kind}}) {
$self->{heuristics}->mark_depend($d, $v);
if ((defined $d->{info} && !$self->{tobuild}{$d}) ||
(defined $d->fullpkgname &&
$d->fullpkgname eq $v->fullpkgname)) {
delete $v->{info}{$kind}{$d};
} else {
$not_yet++;
}
}
return $not_yet if $not_yet;
delete $v->{info}{$kind};
return 0;
}
sub can_package
{
my ($self, $v) = @_;
if (defined $v->{info}{NEEDED_BY}) {
for my $w (values %{$v->{info}{NEEDED_BY}}) {
if ($self->{packages}{$w}) {
delete $v->{info}{NEEDED_BY}{$w};
} else {
return 0;
}
}
}
if (defined $v->{info}{BNEEDED_BY}) {
for my $w (values %{$v->{info}{BNEEDED_BY}}) {
if ($self->{packages}{$w} || $self->{built}{$w} ||
$self->{installable}{$w}) {
delete $v->{info}{BNEEDED_BY}{$w};
} else {
return 0;
}
}
}
return 1;
}
sub check_buildable
{
my $self = shift;
$self->{ts} = time();
my $changes;
do {
$changes = 0;
# move stuff to packages once we know all reverse dependencies
if ($done_scanning) {
for my $v (values %{$self->{installable}}) {
if ($self->can_package($v)) {
$self->log_no_ts('P', $v);
$self->{packages}{$v} = $v;
delete $self->{installable}{$v};
$changes++;
}
}
}
for my $v (values %{$self->{tobuild}}) {
if ($self->was_built($v)) {
$changes++;
} elsif (defined $v->{info}{IGNORE}) {
delete $self->{tobuild}{$v};
push(@{$self->{ignored}}, $v);
$changes++;
}
}
for my $v (values %{$self->{built}}) {
if ($self->adjust($v, 'RDEPENDS') == 0) {
delete $self->{built}{$v};
$self->{installable}{$v} = $v;
$self->log_no_ts('I', $v);
$changes++;
}
}
for my $v (values %{$self->{tobuild}}) {
if ($self->was_built($v)) {
$changes++;
next;
}
my $has = $self->adjust($v, 'DEPENDS');
$has += $self->adjust_extra($v, 'EXTRA');
if ($has == 0) {
$self->{buildable}->add($v);
$self->log_no_ts('Q', $v);
delete $self->{tobuild}{$v};
$changes++;
}
}
} while ($changes);
$self->stats;
}
sub was_built
{
my ($self, $v) = @_;
if ($self->{builder}->check($v)) {
$self->{built}{$v}= $v;
$self->log('B', $v);
delete $self->{tobuild}{$v};
return 1;
} else {
return 0;
}
}
sub new_path
{
my ($self, $v) = @_;
$self->{all}{$v} = $v;
if (!$self->was_built($v)) {
$self->{tobuild}{$v} = $v;
$self->log('T', $v);
}
}
sub end_job
{
my ($self, $core, $v) = @_;
my $e = $core->mark_ready;
if (!$self->was_built($v)) {
if (!$e || $core->{status} == 65280) {
$self->{buildable}->add($v);
$self->{locker}->unlock($v);
$self->log('N', $v);
} else {
push(@{$self->{errors}}, $v);
$v->{host} = $core->host;
$self->{locker}->simple_unlock($v);
$self->log('E', $v);
}
} else {
$self->{locker}->unlock($v);
}
$self->job_done($v);
}
sub add_fatal
{
my ($self, $v) = @_;
push(@{$self->{errors}}, $v);
$self->{locker}->lock($v);
}
sub job_done
{
my ($self, $v) = @_;
for my $candidate (values %{$self->{later}}) {
if ($candidate->{pkgpath} eq $v->{pkgpath}) {
delete $self->{later}{$candidate};
$self->log('V', $candidate);
$self->{buildable}->add($candidate);
}
}
delete $self->{building}{$v->{pkgpath}};
$self->{locker}->recheck_errors($self);
}
sub new_job
{
my ($self, $core, $v, $lock) = @_;
my $special = $self->{heuristics}->special_parameters($core, $v);
$self->log('J', $v, " ".$core->host." ".$special);
$self->{builder}->build($v, $core, $special,
$lock, sub {$self->end_job($core, $v)});
}
sub start_new_job
{
my $self = shift;
my $core = $self->{builder}->get;
my $o = $self->{buildable}->sorted($core);
while (my $v = $o->next) {
$self->{buildable}->remove($v);
if ($self->was_built($v)) {
$self->{logger}->make_log_link($v);
$self->job_done($v);
next;
}
if ($self->{building}{$v->{pkgpath}}) {
$self->{later}{$v} = $v;
$self->log('^', $v);
} elsif (my $lock = $self->{locker}->lock($v)) {
$self->{building}{$v->{pkgpath}} = 1;
$self->new_job($core, $v, $lock);
return;
} else {
push(@{$self->{errors}}, $v);
$self->log('L', $v);
}
}
$core->mark_ready;
}
sub can_build
{
my $self = shift;
return $self->{buildable}->non_empty;
}
sub dump_category
{
my ($self, $k, $fh) = @_;
$fh //= \*STDOUT;
$k =~ m/^./;
my $q = "\u$&: ";
for my $v (sort {$a->fullpkgpath cmp $b->fullpkgpath}
values %{$self->{$k}}) {
print $fh $q;
$v->quick_dump($fh);
}
}
sub dump
{
my ($self, $fh) = @_;
$fh //= \*STDOUT;
for my $k (qw(packages built tobuild installable)) {
$self->dump_category($k, $fh);
}
print $fh "\n";
}
1;

388
infrastructure/build/DPB/Heuristics.pm Normal file
View File

@ -0,0 +1,388 @@
# ex:ts=8 sw=4:
# $OpenBSD: Heuristics.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
# this package is responsible for the initial weighing of pkgpaths, and handling
# consequences
package DPB::Heuristics;
# for now, we don't create a separate object, we assume everything here is
# "global"
my (%weight, %needed_by);
sub new
{
my ($class, $opt_r) = @_;
if ($opt_r) {
bless {}, "DPB::Heuristics::random";
} else {
bless {}, $class;
}
}
sub set_logger
{
my ($self, $logger) = @_;
$self->{logger} = $logger;
}
# we set the "unknown" weight as max weight if we parsed a file.
my $default = 1;
sub finished_parsing
{
my $self = shift;
my @l = sort values %weight;
$default = $l[@l/2];
}
sub intrinsic_weight
{
my ($self, $v) = @_;
$weight{$v} //= $default;
}
my $threshold;
sub set_threshold
{
my ($self, $t) = @_;
$threshold = $t;
}
sub special_parameters
{
my ($self, $core, $v) = @_;
my $t = $core->{memory} // $threshold;
# we build in memory if we know this port and it's light enough
if (!defined $t || !defined $weight{$v} || $weight{$v} > $t) {
return 0;
} else {
return 1;
}
}
sub set_weight
{
my ($self, $v, $w) = @_;
$weight{$v} = $w + 0;
}
my $cache;
sub mark_depend
{
my ($self, $d, $v) = @_;
if (!defined $needed_by{$d}{$v}) {
$needed_by{$d}{$v} = $v;
$cache = {};
}
}
sub compute_measure
{
my ($self, $v) = @_;
my $dependencies = {$v => $v};
my @todo = values %{$needed_by{$v}};
while (my $k = pop (@todo)) {
next if $dependencies->{$k};
$dependencies->{$k} = $k;
push(@todo, values %{$needed_by{$k}});
}
my $sum = 0;
for my $k (values %$dependencies) {
$sum += $self->intrinsic_weight($k);
}
return $sum;
}
sub measure
{
my ($self, $v) = @_;
$cache->{$v} //= $self->compute_measure($v);
}
sub compare
{
my ($self, $a, $b) = @_;
my $r = $self->measure($a) <=> $self->measure($b);
return $r if $r != 0;
# XXX if we don't know, we prefer paths "later in the game"
# so if you abort dpb and restart it, it will start doing
# things earlier.
return $a->fullpkgpath cmp $b->fullpkgpath;
}
my $has_build_info;
sub add_build_info
{
my ($self, $pkgpath, $host, $time, $sz) = @_;
$self->set_weight($pkgpath, $time);
$has_build_info = 1;
}
sub compare_weights
{
my ($self, $a, $b) = @_;
return $self->intrinsic_weight($a) <=> $self->intrinsic_weight($b);
}
sub new_queue
{
my $self = shift;
if ($has_build_info && DPB::Core->has_sf) {
return DPB::Heuristics::Queue::Part->new($self);
} else {
return DPB::Heuristics::Queue->new($self);
}
}
package DPB::Heuristics::SimpleSorter;
sub new
{
my ($class, $o) = @_;
bless $o->sorted_values, $class;
}
sub next
{
my $self = shift;
return pop @$self;
}
package DPB::Heuristics::Sorter;
sub new
{
my ($class, $list) = @_;
my $o = bless {list => $list, l => []}, $class;
$o->next_bin;
return $o;
}
sub next_bin
{
my $self = shift;
if (my $bin = pop @{$self->{list}}) {
$self->{l} = $bin->sorted_values;
} else {
return;
}
}
sub next
{
my $self = shift;
if (my $r = pop @{$self->{l}}) {
return $r;
} else {
if ($self->next_bin) {
return $self->next;
} else {
return;
}
}
}
package DPB::Heuristics::Bin;
sub new
{
my ($class, $h) = @_;
bless {o => {}, weight => 0, h => $h}, $class;
}
sub add
{
my ($self, $v) = @_;
$self->{o}{$v} = $v;
}
sub remove
{
my ($self, $v) = @_;
delete $self->{o}{$v};
}
sub weight
{
my $self = shift;
return $self->{weight};
}
sub count
{
my $self = shift;
return scalar keys %{$self->{o}};
}
sub non_empty
{
my $self = shift;
return scalar %{$self->{o}};
}
sub sorted_values
{
my $self = shift;
return [sort {$self->{h}->compare($a, $b)} values %{$self->{o}}];
}
package DPB::Heuristics::Bin::Heavy;
our @ISA = qw(DPB::Heuristics::Bin);
sub add
{
my ($self, $v) = @_;
$self->SUPER::add($v);
$self->{weight} += $weight{$v};
}
sub remove
{
my ($self, $v) = @_;
$self->{weight} -= $weight{$v};
$self->SUPER::remove($v);
}
package DPB::Heuristics::Queue;
our @ISA = qw(DPB::Heuristics::Bin);
sub sorted
{
my $self = shift;
return DPB::Heuristics::SimpleSorter->new($self);
}
package DPB::Heuristics::Queue::Part;
our @ISA = qw(DPB::Heuristics::Queue);
# 20 bins, binary....
sub find_bin
{
my $w = shift;
return 10 if !defined $w;
if ($w > 65536) {
if ($w > 1048576) { 9 } else { 8 }
} elsif ($w > 256) {
if ($w > 4096) {
if ($w > 16384) { 7 } else { 6 }
} elsif ($w > 1024) { 5 } else { 4 }
} elsif ($w > 16) {
if ($w > 64) { 3 } else { 2 }
} elsif ($w > 4) { 1 } else { 0 }
}
sub add
{
my ($self, $v) = @_;
$self->SUPER::add($v);
$v->{weight} = $weight{$v};
$self->{bins}[find_bin($v->{weight})]->add($v);
}
sub remove
{
my ($self, $v) = @_;
$self->SUPER::remove($v);
$self->{bins}[find_bin($v->{weight})]->remove($v);
}
sub sorted
{
my ($self, $core) = @_;
my $all = DPB::Core->all_sf;
if ($core->{sf} > $all->[-1] - 1) {
return $self->SUPER::sorted($core);
} else {
my $want = $self->bin_part($core->{sf}, DPB::Core->all_sf);
return DPB::Heuristics::Sorter->new($want);
}
}
# simpler partitioning
sub bin_part
{
my ($self, $wanted, $all_sf) = @_;
# note that all_sf is sorted
# compute totals
my $sum_sf = 0;
for my $i (@$all_sf) {
$sum_sf += $i;
}
my @bins = @{$self->{bins}};
my $sum_weight = 0.0;
for my $bin (@bins) {
$sum_weight += $bin->weight;
}
# setup for the main loop
my $partial_weight = 0.0;
my $partial_sf = 0.0;
my $result = [];
# go through speed factors until we've gone thru the one we want
while (my $sf = shift @$all_sf) {
# passed it -> give result
last if $sf > $wanted+1;
# compute threshold for total weight
$partial_sf += $sf;
my $thr = $sum_weight * $partial_sf / $sum_sf;
# grab weights until we reach the desired amount
while (my $bin = shift @bins) {
$partial_weight += $bin->weight;
push(@$result, $bin);
last if $partial_weight > $thr;
}
}
return $result;
}
sub new
{
my ($class, $h) = @_;
my $o = $class->SUPER::new($h);
my $bins = $o->{bins} = [];
for my $i (0 .. 9) {
push(@$bins, DPB::Heuristics::Bin::Heavy->new($h));
}
push(@$bins, DPB::Heuristics::Bin->new($h));
return $o;
}
package DPB::Heuristics::random;
our @ISA = qw(DPB::Heuristics);
my %any;
sub compare
{
my ($self, $a, $b) = @_;
return ($any{$a} //= random()) <=> ($any{$b} //= random());
}
sub new_queue
{
my $self = shift;
return DPB::Heuristics::Queue->new($self);
}
1;

156
infrastructure/build/DPB/Job.pm Normal file
View File

@ -0,0 +1,156 @@
# ex:ts=8 sw=4:
# $OpenBSD: Job.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
# a "job" is the actual stuff a core runs at some point.
# it's mostly an abstract class here... it's organized
# as a list of tasks, with a finalization routine
package DPB::Task;
sub end
{
}
sub code
{
my $self = shift;
return $self->{code};
}
sub new
{
my ($class, $code) = @_;
bless {code => $code}, $class;
}
sub run
{
my ($self, $core) = @_;
&{$self->code($core)}($core->{shell});
}
package DPB::Task::Pipe;
our @ISA =qw(DPB::Task);
sub fork
{
my $self = shift;
open($self->{fh}, "-|");
}
sub end
{
my $self = shift;
close($self->{fh});
}
package DPB::Task::Fork;
our @ISA =qw(DPB::Task);
sub fork
{
CORE::fork();
}
package DPB::Job;
sub next_task
{
my ($self, $core) = @_;
if ($core->{status} == 0) {
return shift @{$self->{tasks}};
} else {
return undef;
}
}
sub name
{
my $self = shift;
return $self->{name};
}
sub finalize
{
}
sub watched
{
my $self = shift;
return $self->{status};
}
sub really_watch
{
}
sub new
{
my ($class, $name) = @_;
bless {name => $name, status => ""}, $class;
}
sub set_status
{
my ($self, $status) = @_;
$self->{status} = $status;
}
package DPB::Job::Normal;
our @ISA =qw(DPB::Job);
sub new
{
my ($class, $code, $endcode, $name) = @_;
my $o = $class->SUPER::new($name);
$o->{tasks} = [DPB::Task::Fork->new($code)];
$o->{endcode} = $endcode;
return $o;
}
sub finalize
{
my $self = shift;
&{$self->{endcode}}(@_);
}
package DPB::Job::Infinite;
our @ISA = qw(DPB::Job);
sub next_task
{
my $job = shift;
return $job->{task};
}
sub new
{
my ($class, $code, $name) = @_;
my $o = $class->SUPER::new($name);
$o->{task} = DPB::Task::Fork->new($code);
return $o;
}
package DPB::Job::Pipe;
our @ISA = qw(DPB::Job);
sub new
{
my ($class, $code, $name) = @_;
my $o = $class->SUPER::new($name);
$o->{tasks} = [DPB::Task::Pipe->new($code)];
return $o;
}
1;

225
infrastructure/build/DPB/Job/Port.pm Normal file
View File

@ -0,0 +1,225 @@
# ex:ts=8 sw=4:
# $OpenBSD: Port.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
use DPB::Job;
package DPB::Task::Port;
our @ISA = qw(DPB::Task::Fork);
sub new
{
my ($class, $phase) = @_;
bless {phase => $phase}, $class;
}
sub fork
{
my ($self, $core) = @_;
my $job = $core->job;
$job->clock;
$job->{current} = $self->{phase};
return $self->SUPER::fork($core);
}
sub run
{
my ($self, $core) = @_;
my $job = $core->job;
my $t = $self->{phase};
my $ports = $job->{builder}->{ports};
my $fullpkgpath = $job->{v}->fullpkgpath;
my $log = $job->{log};
my $make = $job->{builder}->{make};
my $shell = $core->{shell};
close STDOUT;
close STDERR;
open STDOUT, '>>', $log or die "Can't write to $log";
open STDERR, '>&STDOUT' or die "bad redirect";
my @args = ($t, "TRUST_PACKAGES=Yes",
"REPORT_PROBLEM='exit 1'");
if ($job->{special}) {
push(@args, "WRKOBJDIR=/tmp/ports");
}
if (defined $shell) {
$shell->run("cd $ports && SUBDIR=".
$fullpkgpath." ".join(' ',$shell->make, @args));
} else {
chdir($ports) or
die "Wrong ports tree $ports";
$ENV{SUBDIR} = $fullpkgpath;
exec {$make} ("make", @args);
}
exit(1);
}
package DPB::Job::Port;
our @ISA = qw(DPB::Job::Normal);
use Time::HiRes qw(time);
my @list = qw(prepare fetch patch configure build fake package clean);
my $alive = {};
sub stopped_clock
{
my ($class, $gap) = @_;
for my $t (values %$alive) {
if (defined $t->{started}) {
$t->{started} += $gap;
}
}
}
sub new
{
my ($class, $log, $v, $builder, $special, $endcode) = @_;
my @todo = @list;
if ($builder->{clean}) {
unshift @todo, "clean";
}
my $o = bless {tasks => [map {DPB::Task::Port->new($_)} @todo],
log => $log, v => $v,
special => $special, current => '',
builder => $builder, endcode => $endcode},
$class;
$alive->{$o} = $o;
return $o;
}
sub pkgpath
{
my $self = shift;
return $self->{v};
}
sub name
{
my $self = shift;
return $self->{v}->fullpkgpath."($self->{current})";
}
sub clock
{
my $self = shift;
if (defined $self->{started}) {
push(@{$self->{times}}, [$self->{current}, time() - $self->{started}]);
}
$self->{started} = time();
}
sub finalize
{
my $self = shift;
$self->clock;
$self->SUPER::finalize(@_);
delete $alive->{$self};
}
sub totaltime
{
my $self = shift;
my $t = 0;
for my $plus (@{$self->{times}}) {
next if $plus->[0] eq 'fetch' or $plus->[0] eq 'prepare'
or $plus->[0] eq 'clean';
$t += $plus->[1];
}
return sprintf("%.2f", $t);
}
sub timings
{
my $self = shift;
return join('/', map {sprintf("%s=%.2f", @$_)} @{$self->{times}});
}
my $logsize = {};
sub add_build_info
{
my ($class, $pkgpath, $host, $time, $sz) = @_;
$logsize->{$pkgpath} = $sz;
}
sub set_watch
{
my ($self, $logger, $v) = @_;
for my $w ($logger->pathlist($v)) {
if (defined $logsize->{$w}) {
$self->{expected} = $logsize->{$w};
last;
}
}
$self->{watched} = $logger->log_pkgpath($v);
}
sub watch
{
my $self = shift;
my $sz = (stat $self->{watched})[7];
if (!defined $self->{sz} || $self->{sz} != $sz) {
$self->{sz} = $sz;
$self->{time} = time();
}
}
sub watched
{
my ($self, $current) = @_;
return "" unless defined $self->{watched};
$self->watch;
my $progress = '';
if (defined $self->{sz}) {
if (defined $self->{expected} &&
$self->{sz} < 4 * $self->{expected}) {
$progress = ' '.
int($self->{sz}*100/$self->{expected}). '%';
} else {
$progress = ' '.$self->{sz};
}
}
my $diff = $current - $self->{time};
if ($diff > 7200) {
return "$progress unchanged for ".int($diff/3600)." hours";
} elsif ($diff > 300) {
return "$progress unchanged for ".int($diff/60)." minutes";
} elsif ($diff > 10) {
return "$progress unchanged for ".int($diff)." seconds";
} else {
return $progress;
}
}
sub really_watch
{
my ($self, $current) = @_;
return "" unless defined $self->{watched};
$self->watch;
my $diff = $current - $self->{time};
$self->{lastdiff} //= 5;
if ($diff > $self->{lastdiff} * 2) {
$self->{lastdiff} = $diff;
return 1;
} elsif ($diff < $self->{lastdiff}) {
$self->{lastdiff} = 5;
}
return 0;
}
1;

121
infrastructure/build/DPB/Locks.pm Normal file
View File

@ -0,0 +1,121 @@
# ex:ts=8 sw=4:
# $OpenBSD: Locks.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
package DPB::Locks;
use File::Path;
use Fcntl;
sub new
{
my ($class, $lockdir) = @_;
File::Path::make_path($lockdir);
bless {lockdir => $lockdir}, $class;
}
sub build_lockname
{
my ($self, $f) = @_;
$f =~ s|/|.|g;
return "$self->{lockdir}/$f";
}
sub simple_lockname
{
my ($self, $v) = @_;
return $self->build_lockname($v->{pkgpath});
}
sub lockname
{
my ($self, $v) = @_;
return $self->build_lockname($v->fullpkgpath);
}
sub dolock
{
my ($self, $name, $v) = @_;
if (sysopen my $fh, $name, O_CREAT|O_EXCL|O_WRONLY, 0666) {
print $fh "fullpkgpath=", $v->fullpkgpath, "\n";
return $fh;
} else {
return 0;
}
}
sub lock
{
my ($self, $v) = @_;
my $simple = $self->simple_lockname($v);
my $fh = $self->dolock($simple, $v);
if ($fh) {
my $lk = $self->lockname($v);
if ($simple eq $lk) {
return $fh;
}
my $fh2 = $self->dolock($lk, $v);
if ($fh2) {
return $fh2;
} else {
$self->simple_unlock($v);
}
}
return undef;
}
sub unlock
{
my ($self, $v) = @_;
unlink($self->lockname($v));
$self->simple_unlock($v);
}
sub simple_unlock
{
my ($self, $v) = @_;
my $simple = $self->simple_lockname($v);
if ($self->lockname($v) ne $simple) {
unlink($simple);
}
}
sub locked
{
my ($self, $v) = @_;
return -e $self->lockname($v) || -e $self->simple_lockname($v);
}
sub recheck_errors
{
my ($self, $engine) = (@_);
my $e = $engine->{errors};
$engine->{errors} = [];
while (my $v = shift @$e) {
if ($self->locked($v)) {
push(@{$engine->{errors}}, $v);
} else {
$engine->{buildable}->add($v);
}
}
}
1;

119
infrastructure/build/DPB/Logger.pm Normal file
View File

@ -0,0 +1,119 @@
# ex:ts=8 sw=4:
# $OpenBSD: Logger.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
package DPB::Logger;
use File::Path;
use File::Basename;
sub new
{
my ($class, $logdir, $clean) = @_;
bless {logdir => $logdir, clean => $clean}, $class;
}
sub logfile
{
my ($self, $name) = @_;
my $log = "$self->{logdir}/$name.log";
File::Path::mkpath(File::Basename::dirname($log));
return $log;
}
sub open
{
my ($self, $name) = @_;
my $log = $self->logfile($name);
open my $fh, ">>", $log or die "Can't write to $log: $!\n";
return $fh;
}
sub log_pkgpath
{
my ($self, $v) = @_;
return $self->logfile("/paths/".$v->fullpkgpath);
}
sub log_pkgname
{
my ($self, $v) = @_;
return $self->logfile("/packages/".$v->fullpkgname);
}
sub link
{
my ($self, $a, $b) = @_;
if ($self->{clean}) {
unlink($b);
}
my $src = File::Spec->catfile(
File::Spec->abs2rel($self->{logdir}, File::Basename::dirname($b)),
File::Spec->abs2rel($a, $self->{logdir}));
symlink($src, $b);
}
sub pathlist
{
my ($self, $v) = @_;
my @l = ($v);
my $stem = $v->pkgpath_and_flavors;
my $w = DPB::PkgPath->new($stem);
if ($w ne $v) {
push(@l, $w);
}
for my $m (keys %{$v->{info}->{MULTI_PACKAGES}}) {
next if $m eq '-';
my $w = DPB::PkgPath->new("$stem,$m");
if ($w ne $v) {
push(@l, $w);
}
}
return @l;
}
sub make_logs
{
my ($self, $v) = @_;
my $log = $self->log_pkgpath($v);
if ($self->{clean}) {
unlink($log);
}
for my $w ($self->pathlist($v)) {
$self->link($log, $self->log_pkgname($w));
}
return $log;
}
sub make_log_link
{
my ($self, $v) = @_;
my $file = $self->log_pkgname($v);
# we were built, but we don't link, so try the main pkgpath.
if (!-e $file) {
my $mainlog = $self->log_pkgpath(DPB::PkgPath->new($v->pkgpath_and_flavors));
if (-e $mainlog) {
$self->link($mainlog, $file);
}
# okay, so it was built through another flavor, don't bother
# for now, it will all solve itself eventually
}
$self->link($file, $self->log_pkgpath($v));
}
1;

214
infrastructure/build/DPB/PkgPath.pm Normal file
View File

@ -0,0 +1,214 @@
# ex:ts=8 sw=4:
# $OpenBSD: PkgPath.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
# Handles PkgPath;
# all this code is *seriously* dependent on unique objects
# everything is done to normalize PkgPaths, so that we have
# one pkgpath object for each distinct flavor/subpackage combination
package DPB::PkgPath;
my $cache = {};
my $seen = {};
sub create
{
my ($class, $fullpkgpath) = @_;
# subdivide into flavors/multi
my @list = split /,/, $fullpkgpath;
my $pkgpath = shift @list;
my %flavors = map {($_, 1)} grep { !/^\-/} @list;
my @multi = grep { /^\-/} @list;
my $multi = pop(@multi) || undef;
if (@multi > 0) {
die "$fullpkgpath has >1 multi\n";
}
bless {pkgpath => $pkgpath,
flavors => \%flavors,
multi => $multi}, $class;
}
# cache just once, put into standard order, so that we don't
# create different objects for path,f1,f2 and path,f2,f1
sub normalize
{
my $o = shift;
my $fullpkgpath = $o->fullpkgpath;
return $cache->{$fullpkgpath} //= $o;
}
# actual user constructor that doesn't record into seen
sub new_hidden
{
my ($class, $fullpkgpath) = @_;
if (defined $cache->{$fullpkgpath}) {
return $cache->{$fullpkgpath};
} else {
return $class->create($fullpkgpath)->normalize;
}
}
# actual user constructor that records into seen
sub new
{
my ($class, $fullpkgpath) = @_;
my $o = $class->new_hidden($fullpkgpath);
$seen->{$o} //= $o;
}
sub seen
{
return values %$seen;
}
sub basic_list
{
my $self = shift;
my @list = ($self->{pkgpath});
if (keys %{$self->{flavors}}) {
push(@list, sort keys %{$self->{flavors}});
}
return @list;
}
# string version, with everything in a standard order
sub fullpkgpath
{
my $self = shift;
my @list = $self->basic_list;
if ($self->{multi}) {
push(@list, $self->{multi});
}
return join (',', @list);
}
# without multi. Used by the SUBDIRs code to make sure we get the right
# value for default subpackage.
sub pkgpath_and_flavors
{
my $self = shift;
return join (',', $self->basic_list);
}
sub add_to_subdirlist
{
my ($self, $list) = @_;
push(@$list, $self->pkgpath_and_flavors);
}
sub copy_flavors
{
my $self = shift;
return {map {($_, 1)} keys %{$self->{flavors}}};
}
# XXX
# in the ports tree, when you build with SUBDIR=n/value, you'll
# get all the -multi packages, but with the default flavor.
# we have to strip the flavor part to match the SUBDIR we asked for.
sub compose
{
my ($class, $fullpkgpath, $pseudo) = @_;
my $o = $class->create($fullpkgpath);
$o->{flavors} = $pseudo->copy_flavors;
return $o->normalize;
}
# XXX All this code knows too much about PortInfo for proper OO
sub fullpkgname
{
my $self = shift;
return (defined $self->{info}) ? $self->{info}->fullpkgname : undef;
}
sub zap_default
{
my ($self, $subpackage) = @_;
return $self unless defined $subpackage;
if ($subpackage->string eq $self->{multi}) {
my $o = bless {pkgpath => $self->{pkgpath},
flavors => $self->copy_flavors}, ref($self);
return $o->normalize;
} else {
return $self;
}
}
# default subpackage leads to pkgpath,-default = pkgpath
sub handle_default
{
my ($self, $h) = @_;
my $m = $self->zap_default($self->{info}->{SUBPACKAGE});
if ($m ne $self) {
#print $m->fullpkgpath, " vs. ", $self->fullpkgpath,"\n";
$m->{info} = $self->{info};
$h->{$m} = $m;
}
}
sub dump
{
my ($self, $fh) = @_;
print $fh $self->fullpkgpath, "\n";
if (defined $self->{info}) {
$self->{info}->dump($fh);
}
}
sub quick_dump
{
my ($self, $fh) = @_;
print $fh $self->fullpkgpath, "\n";
if (defined $self->{info}) {
$self->{info}->quick_dump($fh);
}
}
sub merge_depends
{
my ($class, $h) = @_;
my $global = bless {}, "AddDepends";
for my $v (values %$h) {
for my $k (qw(LIB_DEPENDS BUILD_DEPENDS)) {
if (defined $v->{info}{$k}) {
for my $d (values %{$v->{info}{$k}}) {
$global->{$d} = $d;
}
}
}
if (defined $v->{info}{RUN_DEPENDS}) {
for my $d (values %{$v->{info}{RUN_DEPENDS}}) {
$v->{info}{RDEPENDS}{$d} = $d;
bless $v->{info}{RDEPENDS}, "AddDepends";
}
}
}
if (values %$global > 0) {
for my $v (values %$h) {
$v->{info}{DEPENDS} = $global;
}
}
}
1;

125
infrastructure/build/DPB/PortBuilder.pm Normal file
View File

@ -0,0 +1,125 @@
# ex:ts=8 sw=4:
# $OpenBSD: PortBuilder.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
# this object is responsible for launching the build of ports
# which mostly includes starting the right jobs
package DPB::PortBuilder;
use File::Path;
use DPB::Util;
use DPB::Job::Port;
sub new
{
my $class = shift;
my ($opt_c, $fullrepo, $logger, $ports, $make, $h) = @_;
my $self = bless {clean => $opt_c,
fullrepo => $fullrepo,
logger => $logger, ports => $ports, make => $make,
heuristics => $h}, $class;
$self->init;
return $self;
}
sub init
{
my $self = shift;
File::Path::make_path($self->{fullrepo});
$self->{global} = $self->{logger}->open("build");
}
sub check
{
my ($self, $v) = @_;
my $name = $v->fullpkgname;
return -f "$self->{fullrepo}/$name.tgz";
}
sub report
{
my ($self, $v, $job, $host) = @_;
my $pkgpath = $v->fullpkgpath;
my $sz = (stat $self->{logger}->log_pkgpath($v))[7];
my $log = $self->{global};
print $log "$pkgpath $host ", $job->totaltime, " ", $sz, " ",
$job->timings;
if ($self->check($v)) {
print $log "\n";
} else {
print $log "!\n";
}
}
sub get
{
my $self = shift;
return DPB::Core->get;
}
sub end_lock
{
my ($self, $lock, $core) = @_;
my $end = time();
print $lock "status=$core->{status}\n";
print $lock "end=$end (", DPB::Util->time2string($end), ")\n";
close $lock;
}
sub build
{
my ($self, $v, $core, $special, $lock, $final_sub) = @_;
my $start = time();
my $log = $self->{logger}->make_logs($v);
my $job;
$job = DPB::Job::Port->new($log, $v, $self, $special,
sub {$self->end_lock($lock, $core); $self->report($v, $job, $core->host); &$final_sub;});
$core->start_job($job, $v);
# (sub {
# }, $v, " (".$self->{heuristics}->measure($v).")");
print $lock "host=", $core->host, "\n";
print $lock "pid=$core->{pid}\n";
print $lock "start=$start (", DPB::Util->time2string($start), ")\n";
$job->set_watch($self->{logger}, $v);
return $core;
}
package DPB::DummyCore;
sub host
{
return "dummy";
}
my $dummy = bless {}, "DPB::DummyCore";
package DPB::PortBuilder::Test;
our @ISA = qw(DPB::PortBuilder);
sub build
{
my ($self, $v, $core, $lock, $code) = @_;
my $name = $v->fullpkgname;
# my $log = $self->{logger}->make_logs(make_logs($v), $self->{clean});
# open my $out, ">>", $log or die "Can't write to $log";
open my $fh, ">", "$self->{fullrepo}/$name.tgz";
close $fh;
&$code;
return $dummy;
}
1;

186
infrastructure/build/DPB/PortInfo.pm Normal file
View File

@ -0,0 +1,186 @@
# ex:ts=8 sw=4:
# $OpenBSD: PortInfo.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
package AddInfo;
sub add
{
my ($class, $var, $o, $value) = @_;
return if $value =~ m/^[\s\-]*$/;
$o->{$var} = $class->new($value, $o);
}
sub new
{
my ($class, $value) = @_;
bless \$value, $class;
}
sub string
{
my $self = shift;
return $$self;
}
sub quickie
{
return 0;
}
package AddInfoShow;
our @ISA = qw(AddInfo);
sub quickie
{
return 1;
}
package AddList;
our @ISA = qw(AddInfo);
sub make_list
{
my ($class, $value) = @_;
$value =~ s/^\s+//;
$value =~ s/\s+$//;
return split(/\s+/, $value);
}
sub new
{
my ($class, $value) = @_;
my %values = map {($_, 1)} $class->make_list($value);
bless \%values, $class;
}
sub string
{
my $self = shift;
return join(', ', keys %$self);
}
package AddDepends;
our @ISA = qw(AddList);
sub new
{
my ($class, $value, $self) = @_;
my $r = {};
for my $_ ($class->make_list($value)) {
next if m/^$/;
s/^.*?\:.*?\://;
if (s/\:(?:patch|build|configure)$//) {
Extra->add('EXTRA', $self, $_);
} else {
s/\:$//;
my $info = DPB::PkgPath->new($_);
$r->{$info} = $info;
}
}
bless $r, $class;
}
sub string
{
my $self = shift;
return '['.join(';', map {$_->fullpkgpath} (values %$self)).']';
}
sub quickie
{
return 1;
}
package Extra;
our @ISA = qw(AddDepends);
sub add
{
my ($class, $key, $self, $value) = @_;
$self->{$key} //= bless {}, $class;
my $info = DPB::PkgPath->new($value);
$self->{$key}{$info} = $info;
return $self;
}
package DPB::PortInfo;
my %adder = (
FULLPKGNAME => "AddInfoShow",
RUN_DEPENDS => "AddDepends",
BUILD_DEPENDS => "AddDepends",
LIB_DEPENDS => "AddDepends",
SUBPACKAGE => "AddInfo",
MULTI_PACKAGES => "AddList",
EXTRA => "Extra",
DEPENDS => "AddDepends",
RDEPENDS => "AddDepends",
IGNORE => "AddInfo",
NEEDED_BY => "AddDepends",
BNEEDED_BY => "AddDepends",
);
sub wanted
{
my ($class, $var) = @_;
return $adder{$var};
}
sub new
{
my ($class, $pkgpath) = @_;
$pkgpath->{info} //= bless {}, $class;
}
sub add
{
my ($self, $var, $value) = @_;
$adder{$var}->add($var, $self, $value);
}
sub dump
{
my ($self, $fh) = @_;
for my $k (sort keys %adder) {
print $fh "\t $k = ", $self->{$k}->string, "\n"
if defined $self->{$k};
}
}
use Data::Dumper;
sub quick_dump
{
my ($self, $fh) = @_;
for my $k (sort keys %adder) {
if (defined $self->{$k} and $adder{$k}->quickie) {
print $fh "\t $k = ";
if (ref($self->{$k}) eq 'HASH') {
print $fh "????\n";
} else {
print $fh $self->{$k}->string, "\n" ;
}
}
}
}
sub fullpkgname
{
my $self = shift;
return (defined $self->{FULLPKGNAME}) ?
$self->{FULLPKGNAME}->string : undef;
}
1;

262
infrastructure/build/DPB/Reporter.pm Normal file
View File

@ -0,0 +1,262 @@
# ex:ts=8 sw=4:
# $OpenBSD: Reporter.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
use Term::Cap;
use OpenBSD::Error;
use DPB::Util;
use Time::HiRes qw(time);
package DPB::Reporter;
my $width;
my $wsz_format = 'SSSS';
our %sizeof;
sub find_window_size
{
my $self = shift;
# try to get exact window width
my $r;
$r = pack($wsz_format, 0, 0, 0, 0);
$sizeof{'struct winsize'} = 8;
require 'sys/ttycom.ph';
$width = 80;
if (ioctl(STDOUT, &TIOCGWINSZ, $r)) {
my ($rows, $cols, $xpix, $ypix) =
unpack($wsz_format, $r);
$self->{width} = $cols;
$self->{height} = $rows;
}
}
sub term_send
{
my ($self, $seq) = @_;
if (defined $self->{terminal}) {
$self->{terminal}->Tputs($seq, 1, \*STDOUT);
}
}
sub reset_cursor
{
my $self = shift;
$self->term_send("ve");
}
sub set_cursor
{
my $self = shift;
$self->term_send("vi");
}
sub reset
{
my $self = shift;
$self->reset_cursor;
$self->term_send("cl");
}
my $stopped_clock;
sub set_sigtstp
{
my $self =shift;
$SIG{TSTP} = sub {
$self->reset_cursor;
$stopped_clock = time();
$SIG{TSTP} = 'DEFAULT';
kill TSTP => $$;
};
}
sub set_sig_handlers
{
my $self = shift;
$SIG{'WINCH'} = sub {
$self->find_window_size;
$self->{write} = 'go_write_home';
};
$self->set_sigtstp;
$SIG{'CONT'} = sub {
$self->set_sigtstp;
$self->{continued} = 1;
$self->set_cursor;
$self->find_window_size;
$self->{write} = 'go_write_home';
DPB::Job::Port->stopped_clock(time() - $stopped_clock);
};
OpenBSD::Handler->register(sub {
$self->reset_cursor; });
$SIG{'__DIE__'} = sub {
$self->reset_cursor;
};
}
my $extra = '';
my $interrupted;
sub new
{
my $class = shift;
my $notty = shift;
my $isatty = !$notty && -t STDOUT;
my $self = bless {msg => '', tty => $isatty,
producers => \@_, continued => 0}, $class;
if ($isatty) {
my $oldfh = select(STDOUT);
$| = 1;
# XXX go back to totally non-buffered raw shit
binmode(STDOUT, ':pop');
select($oldfh);
use POSIX;
my $termios = POSIX::Termios->new;
$termios->getattr(0);
$self->{terminal} = Term::Cap->Tgetent({ OSPEED =>
$termios->getospeed });
$self->find_window_size;
$self->set_sig_handlers;
if ($self->{terminal}->Tputs("ho", 1)) {
$self->{write} = "go_write_home";
} else {
$self->{write} = "write_clear";
}
# no cursor, to avoid flickering
$self->set_cursor;
} else {
$self->{write} = "no_write";
}
$SIG{INFO} = sub {
# use Carp;
# Carp::cluck();
# print $self->{msg};
# if ($self->{write} eq 'write_home') {
# $self->{write} = 'go_write_home';
# }
$interrupted++;
};
return $self;
}
sub write_clear
{
my ($self, $msg) = @_;
$self->term_send("cl");
$self->{oldlines} = [$self->cut_lines($msg)];
for my $line (@{$self->{oldlines}}) {
print $line, "\n";
}
}
sub cut_lines
{
my ($self, $msg) = @_;
my @lines = ();
for my $line (split("\n", $msg)) {
while (length $line >= $self->{width}) {
push(@lines, substr($line, 0, $self->{width}-1));
$line = substr($line, $self->{width}-1);
}
push(@lines, $line);
}
return @lines;
}
sub print_clamped
{
my ($self, $line) = @_;
print substr($line, 0, $self->{width}-1)."\n";
}
sub write_home
{
my ($self, $msg) = @_;
my @new = $self->cut_lines($msg);
$self->term_send("ho");
while (my $newline = shift @new) {
my $oldline = shift @{$self->{oldlines}};
# line didn't change: try to go down
if (defined $oldline && $oldline eq $newline) {
if ($self->term_send("do")) {
next;
}
}
# adjust newline to correct length
if (defined $oldline && (length $oldline) > (length $newline)) {
$newline .= " "x ((length $oldline) - (length $newline));
}
$self->print_clamped($newline);
}
# extra lines must disappear
while (my $line = shift(@{$self->{oldlines}})) {
$line = " "x (length $line);
$self->print_clamped($line);
}
$self->{oldlines} = [$self->cut_lines($msg)];
}
sub go_write_home
{
# first report has to clear the screen
my ($self, $msg) = @_;
$self->write_clear($msg);
$self->{write} = 'write_home';
}
sub no_write
{
}
sub report
{
my $self = shift;
my $msg = DPB::Util->time2string(time)."\n";
for my $prod (@{$self->{producers}}) {
$msg.= $prod->report;
}
if ($interrupted) {
$interrupted = 0;
$self->reset_cursor;
$DB::single = 1;
}
if (!$self->{tty}) {
for my $prod (@{$self->{producers}}) {
my $important = $prod->important;
if ($important) {
print $important;
}
}
}
$msg .= $extra;
if ($msg ne $self->{msg} || $self->{continued}) {
$self->{continued} = 0;
my $method = $self->{write};
$self->$method($msg);
$self->{msg} = $msg;
}
}
sub myprint
{
my $self = shift;
for my $string (@_) {
$extra .= $string;
}
}
1;

109
infrastructure/build/DPB/Signature.pm Normal file
View File

@ -0,0 +1,109 @@
# ex:ts=8 sw=4:
# $OpenBSD: Signature.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
use OpenBSD::LibSpec;
package DPB::Signature::Dir;
sub best
{
my ($h, $lib) = @_;
my $old = $h->{$lib->stem} //= $lib;
return if $old eq $lib;
return if $old->major > $lib->major;
return if $old->major == $lib->major && $old->minor > $lib->minor;
$h->{$lib->stem} = $lib;
}
sub new
{
my $class = shift;
bless {}, $class;
}
sub compare1
{
my ($s1, $s2) = @_;
while (my ($stem, $lib) = each %$s1) {
if (!defined $s2->{$stem}) {
return "Can't find ".$lib->to_string;
}
if ($s2->{$stem}->to_string ne $lib->to_string) {
return "versions don't match: ".
$s2->{$stem}->to_string." vs ". $lib->to_string;
}
}
return 0;
}
sub compare
{
my ($s1, $s2) = @_;
return compare1($s1, $s2) || compare1($s2, $s1);
}
package DPB::Signature;
my $byhost = {};
sub signature
{
my ($class, $job) = @_;
$byhost->{$job->host} //= $class->compute_signature($job);
}
sub compute_signature
{
my ($class, $job) = @_;
my $o = bless {host => $job->host}, $class;
for my $base (OpenBSD::Paths->library_dirs) {
my $repo = $o->{$base} = DPB::Signature::Dir->new;
my $dir = "$base/lib";
$job->start_pipe(sub {
my $shell = shift;
if (defined $shell) {
$shell->run("ls $dir");
} else {
exec{"/bin/ls"} ("ls", $dir);
}
exit(1);
}, "ls");
my $fh = $job->fh;
while (<$fh>) {
my $lib = OpenBSD::Library->from_string("$dir/$_");
next unless $lib->is_valid;
$repo->best($lib);
}
$job->terminate;
}
return $o;
}
sub compare
{
my ($s1, $s2) = @_;
for my $dir (OpenBSD::Paths->library_dirs) {
my $r = $s1->{$dir}->compare($s2->{$dir});
if ($r) {
print STDERR "Error between $s1->{host} and $s2->{host}: $r\n";
return 1;
}
}
return 0;
}
1;

46
infrastructure/build/DPB/Util.pm Normal file
View File

@ -0,0 +1,46 @@
# ex:ts=8 sw=4:
# $OpenBSD: Util.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
package DPB::Util;
sub make_hot
{
my ($self, $fh) = @_;
my $oldfh = select($fh);
$| = 1;
select($oldfh);
return $fh;
}
sub safe_join
{
my ($self, $sep, @l) = @_;
$_ //= "undef" for @l;
return join($sep, @l);
}
my @name =qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
sub time2string
{
my ($self, $time) = @_;
my ($sec, $min, $hour, $mday, $mon) = (localtime $time)[0 .. 4];
return sprintf("%d %s %02d:%02d:%02d", $mday, $name[$mon],
$hour, $min, $sec);
}
1

145
infrastructure/build/DPB/Vars.pm Normal file
View File

@ -0,0 +1,145 @@
# ex:ts=8 sw=4:
# $OpenBSD: Vars.pm,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
package DPB::Vars;
use OpenBSD::Paths;
my @errors = ();
my $last_errors = 0;
sub report
{
return join('', @errors);
}
sub important
{
my $class = shift;
if (@errors > $last_errors) {
$last_errors = @errors;
return $class->report;
}
}
sub get
{
my ($class, $make, @names) = @_;
pipe(my $rh, my $wh);
my $pid = fork();
if ($pid == 0) {
print $wh "all:\n";
for my $_ (@names) {
print $wh "\t\@echo \${$_}\n";
}
print $wh <<EOT;
COMMENT = test
CATEGORIES = test
PKGPATH = test/a
PERMIT_PACKAGE_CDROM=Yes
PERMIT_PACKAGE_FTP=Yes
PERMIT_DISTFILES_CDROM=Yes
PERMIT_DISTFILES_FTP=Yes
WRKOBJDIR=
IGNORE=Yes
ECHO_MSG=:
.include <bsd.port.mk>
EOT
close $wh;
exit 0;
}
close $wh;
my @list;
my $pid2 = open(my $output, "-|");
if ($pid2) {
close $rh;
@list = <$output>;
chomp for @list;
waitpid($pid2, 0);
waitpid($pid, 0);
} else {
close STDIN;
open(STDIN, '<&', $rh);
exec {$make} ('make', '-f', '-');
die "oops";
}
return @list;
}
sub grab_list
{
my ($class, $core, $ports, $make, $subdirs, $log, $dpb, $code) = @_;
$core->start_pipe(sub {
my $shell = shift;
my @args = ('dump-vars', "DPB=$dpb", "BATCH=Yes", "REPORT_PROBLEM=:");
if (defined $shell) {
my $s='';
if (defined $subdirs) {
$s="SUBDIR='".join(' ', sort @$subdirs)."'";
}
$shell->run("cd $ports && $s ".
join(' ', $shell->make, @args));
} else {
if (defined $subdirs) {
$ENV{SUBDIR} = join(' ', sort @$subdirs);
}
chdir($ports) or die "Bad directory $ports";
exec {$make} ('make', @args);
}
exit(1);
}, "LISTING");
my $h = {};
my $fh = $core->fh;
my $subdir;
my $reset = sub {
for my $v (values %$h) {
$v->handle_default($h);
}
DPB::PkgPath->merge_depends($h);
&$code($h);
$h = {};
};
while(<$fh>) {
chomp;
if (m/^\=\=\=\>\s*Exiting (.*) with an error$/) {
push(@errors, "Problem in $1\n");
}
if (m/^\=\=\=\>\s*(.*)/) {
print $log $_, "\n";
$core->job->set_status(" at $1");
$subdir = DPB::PkgPath->new_hidden($1);
&$reset;
} elsif (my ($pkgpath, $var, $value) =
m/^(.*?)\.([A-Z][A-Z_0-9]*)\=(.*)$/) {
next unless DPB::PortInfo->wanted($var);
if ($value =~ m/^\"(.*)\"$/) {
$value = $1;
}
my $o = DPB::PkgPath->compose($pkgpath, $subdir);
my $info = DPB::PortInfo->new($o, $subdir);
$h->{$o} = $o;
$info->add($var, $value);
}
}
&$reset;
$core->terminate;
}
1;

272
infrastructure/build/dpb3 Executable file
View File

@ -0,0 +1,272 @@
#! /usr/bin/perl
# ex:ts=8 sw=4:
# $OpenBSD: dpb3,v 1.1 2010/02/24 11:33:31 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
my $ports1;
BEGIN {
$ports1 = $ENV{PORTSDIR} || '/usr/ports';
}
$0 = 'dpb';
use FindBin;
use lib "$FindBin::Bin";
use lib "$ports1/infrastructure/build";
use DPB::PkgPath;
use DPB::Core;
use DPB::Vars;
use DPB::PortInfo;
use DPB::Engine;
use DPB::PortBuilder;
use DPB::Reporter;
use OpenBSD::Getopt;
use OpenBSD::Error;
use DPB::Heuristics;
use DPB::Locks;
use DPB::Logger;
use DPB::Job;
use OpenBSD::Paths;
my $make = $ENV{MAKE} || OpenBSD::Paths->make;
our ($opt_t, $opt_e, $opt_T, $opt_c, $opt_h, $opt_A, $opt_j, $opt_a,
$opt_r,
$opt_L, $opt_m, $opt_f, $opt_x);
my @subdirlist;
sub parse_build_line
{
return split(/\s+/, shift);
}
sub parse_build_file
{
my ($fname, @consumers) = @_;
open my $fh, '<', $fname or die "Couldn't open build file $fname\n";
my $_;
while (<$fh>) {
chomp;
my ($pkgpath, $host, $time, $sz, @rest) = parse_build_line($_);
$sz =~ s/!$//;
my $o = DPB::PkgPath->new_hidden($pkgpath);
for my $c (@consumers) {
$c->add_build_info($o, $host, $time, $sz);
}
}
}
my $parsed = 0;
my $heuristics = DPB::Heuristics->new($opt_r);
set_usage("dpb3 [-acertx] [-A arch] [-j N] [-P plist] [-h hosts] [-L lockdir] [-b log] ",
"[-T timeout] [-m threshold] [path ...]");
try {
getopts('acerh:txA:f:j:m:P:b:L:T:', {
P => sub {
my $file = shift;
open my $fh, '<', $file or die "Can't open $file\n";
my $_;
while (<$fh>) {
chomp;
next if m/^\#/;
unshift @ARGV, $_;
}
},
b => sub {
parse_build_file(shift, $heuristics, "DPB::Job::Port");
$parsed = 1;
},
}
);
} catchall {
Usage($_);
};
if ($parsed) {
$heuristics->finished_parsing;
}
if ($opt_m) {
$heuristics->set_threshold($opt_m);
}
my $dpb = $opt_f ? "fetch" : "normal";
if (@ARGV == 0 && !$opt_a) {
Usage("Need at least one pkgpath to work with");
}
for my $arg (@ARGV) {
my ($path, $weight) = ($arg, 5000);
if ($arg =~ m/^(.*)\=(\d+)$/) {
($path, $weight) = ($1, $2);
}
my $pkgpath = DPB::PkgPath->new($path);
$heuristics->set_weight($pkgpath, $weight);
$pkgpath->add_to_subdirlist(\@subdirlist);
}
my ($ports, $repo, $localarch, $distdir) = DPB::Vars->get($make, "PORTSDIR",
"PACKAGE_REPOSITORY", "MACHINE_ARCH", "DISTDIR");
my $arch = $opt_A // $localarch;
$opt_A //= '';
my $logdir = $ENV{LOGDIR} || "$ports/log/$opt_A";
my $lockdir = $opt_L // "$ports/locks/$opt_A";
my $logger = DPB::Logger->new($logdir, $opt_c);
$heuristics->set_logger($logger);
if (defined $opt_j && $localarch ne $arch) {
Usage("Can't use -j if -A arch is not local architecture");
}
if (defined $opt_j && $opt_j !~ m/^\d+$/) {
Usage("-j takes a numerical argument");
}
my $fullrepo = $opt_t ? "$repo/$arch/test" : "$repo/$arch/all";
if ($opt_t) {
$logdir = "$logdir/test";
}
if ($opt_h) {
DPB::Core->parse_hosts_file($opt_h, $arch);
} else {
$opt_j //= 1;
}
$opt_j //= 0;
for (1 .. $opt_j) {
DPB::Core::Factory->new('localhost');
}
my $builder = ($opt_t ? "DPB::PortBuilder::Test" : "DPB::PortBuilder")->new(
$opt_c, $fullrepo, $logger, $ports, $make, $heuristics);
my $locker = DPB::Locks->new($lockdir);
my $engine = DPB::Engine->new($builder, $heuristics, $logger, $locker);
my $reporter = DPB::Reporter->new($opt_x, "DPB::Core", "DPB::Vars", $engine);
while (!DPB::Core->avail) {
DPB::Core->reap;
sleep 1;
}
my $core = DPB::Core->get;
my $loglist = DPB::Util->make_hot($logger->open("vars"));
#my $dump = DPB::Util->make_hot($logger->open('dump'));
my $keep_going = 1;
$opt_T //= 10;
my $last_time = time() - $opt_T;
sub handle_non_waiting_jobs
{
my $need_clock = shift;
my $reaped = DPB::Core->reap;
$keep_going = !-e "$logdir/stop";
if (DPB::Core->avail) {
$engine->check_buildable;
}
while ($keep_going && DPB::Core->avail && $engine->can_build) {
$engine->start_new_job;
}
if ($need_clock) {
my $current = time();
if ($current >= $last_time + $opt_T || $reaped) {
$reporter->report;
$last_time = $current;
}
} else {
$reporter->report;
}
}
sub finish
{
my $h = shift;
for my $v (values %$h) {
$engine->new_path($v);
}
handle_non_waiting_jobs(1);
}
sub grab_subdirs
{
my $list = shift;
DPB::Vars->grab_list($core, $ports, $make, $list, $loglist, $dpb,
sub {
finish(shift);
});
}
sub complete_subdirs
{
# more passes if necessary
while ($keep_going) {
my @subdirlist = ();
for my $v (DPB::PkgPath->seen) {
next if defined $v->{info};
if (defined $v->{tried}) {
$engine->add_fatal($v);
}
$v->add_to_subdirlist(\@subdirlist);
$v->{tried} = 1;
}
last if @subdirlist == 0;
grab_subdirs(\@subdirlist);
}
}
if (@subdirlist > 0) {
grab_subdirs(\@subdirlist);
}
complete_subdirs();
if ($opt_a) {
grab_subdirs(undef);
}
complete_subdirs();
# give back "our" core to the pool.
if (!$opt_e) {
$core->mark_ready;
}
# and let's wait for all jobs now.
close($loglist);
$engine->finished_scanning;
#$engine->dump($dump);
$engine->check_buildable;
#$engine->dump($dump);
DPB::Core->start_clock($opt_T);
while (1) {
handle_non_waiting_jobs(0);
if (!DPB::Core->running && (!$keep_going || !$engine->can_build)) {
last;
}
if (DPB::Core->running) {
DPB::Core->reap_wait;
}
}
$reporter->reset;
DPB::Core->cleanup;
print $engine->report;
$engine->dump_category('tobuild');

253
infrastructure/build/dpb3.1 Normal file
View File

@ -0,0 +1,253 @@
.\" $OpenBSD: dpb3.1,v 1.1 2010/02/24 11:33:31 espie Exp $
.\"
.Dd $Mdocdate: February 24 2010 $
.Dt DPB3 1
.Os
.Sh NAME
.Nm dpb3
.Nd distributed ports builder
.Sh SYNOPSIS
.Nm dpb3
.Bk -words
.Op Fl acertx
.Op Fl A Ar arch
.Op Fl b Ar logfile
.Op Fl h Ar hosts
.Op Fl j Ar n
.Op Fl L Ar lockdir
.Op Fl P Ar subdirlist
.Op Fl T timeout
.Op Ar pkgpath ...
.Ek
.Sh DESCRIPTION
The
.Nm
command is used to build ports on a cluster of machines.
Its name is an acronym for
.Sq distributed ports builder .
.Nm
walks ports to figure out dependencies, and starts building ports
as soon as it can.
It can take
.Ar pkgpath ...
to build as parameters.
Options are as follows:
.Bl -tag -width pkgpathlonger
.It Fl A Ar arch
Build packages for given architecture, selecting relevant hosts from the
cluster.
.It Fl a
Walk the whole tree and builds all packages.
.It Fl b Ar logfile
Prime the heuristics module with a previous build log, so that packages that
take a long time to build will happen earlier.
.It Fl c
Clean ports tree and log before each build.
.It Fl e
The listing job is extra and won't be given back to the pool when it's
finished.
.It Fl h Ar hosts
hosts to use for building.
One host per line, plus properties, such as:
.Bd -literal
espie@aeryn jobs=4 arch=i386
.Ed
.It Fl j Ar n
Number of concurrent local jobs to run.
.It Fl L Ar lockdir
Choose a locks directory
.Po
default is
.Pa locks/${ARCH}
.Pc .
.It Fl P Ar subdirlist
Read list of pkgpaths from file
.It Fl r
Random build order.
Disregard any kind of smart heuristics.
Useful to try to find missing build dependencies.
.It Fl t
Test mode. Only pretend to build packages. Used to check that the engine works.
.It Fl T Ar timeout
Timeout (in seconds) while waiting for jobs to finish, so that the display
is updated even if jobs didn't finish.
Defaults to 10 seconds.
.It Fl x
No tty report, only report really important things, like hosts going down
and coming back up, build errors, or builds not progressing.
.El
.Pp
.Nm
figures out in which order to build things on the fly, and constantly
displays information relative to what's currently building.
There's a list currently running, one line per task, with the task name,
local pid, the build host name, and advancement based on the log file size.
This is followed by a two-line display:
.Bl -tag -width BB=
.It P=
number of built packages, that could be installable, but are not needed
by anything yet to build.
.It I=
number of packages that can be installed, and can be needed for other builds.
.It B=
number of built packages, not yet known to be installable.
.It Q=
number of packages in the queue, e.g., stuff that can be built now, assuming
we have a free slot.
.It T=
number of packages to_build, where dependencies are not yet resolved.
.It !=
number of ignored packages.
.It E=
list of packages in error, that cannot currently be built.
.El
.Pp
Note that those numbers refer to pkgpaths known to
.Nm .
In general, those numbers will be slightly higher than the actual number
of packages being built, since several paths may lead to the same package.
.Pp
P will stay at zero until the listing job is finished, since
.Nm
needs full backwards dependencies to compute it.
.Pp
.Nm
uses some heuristics to try to maximise Q as soon as possible.
There's also a provision for a feedback-directed build, where timings from
a previous build can be used to try to build long-running jobs first.
.Sh LOCKS AND ERRORS
When building a package,
.Nm
produces a lockfile in the lock directory, whose name is deduced from
the basic pkgpath with slashes replaced by dots, and a possible second lock
with the fullpkgpath.
This lockfile is filled with such info as the build start time or the host.
.Pp
At the end of a succesful build, these lockfiles are removed.
The fullpkgpath will stay around in case of errors.
.Pp
At the end of each job,
.Nm
rechecks the lock directory for existing lockfiles.
If some locks have vanished,
it will put the corresponding paths back in the queue and attempt
another build.
.Pp
This eases manual repairs: if a package does not build, the user can look
at the log, go to the port directory, fix the problem, and then remove the lock.
.Nm
will pick up the ball and keep building without interruption.
.Pp
One can also run several
.Nm
in parallel.
This is not optimal, since each
.Nm
ignores the others, and only uses the lock info to avoid the other's
current work, but it can be handy: in an emergency, one can start a second
.Nm
to obtain a specific package right now, in parallel with the original
.Nm .
.Sh SHUTTING DOWN GRACEFULLY
.Nm
periodically checks for a file named
.Pa ${PORTSDIR}/log/${ARCH}/stop .
If this file exists, then it won't start new jobs, and shutdown when
the current jobs are finished.
.Sh FILES
Apart from producing packages,
.Nm
will create a number of log files under
.Pa ${PORTSDIR}/log/{$ARCH} :
.Bl -tag -width engine.log
.It build.log
Actual build log.
Each line summarizes build of a single pkgpath, as:
.Sq pkgpath host time logsize (detailed timing)[!]
where time is the actual build time in seconds, host is the machine name
where this occurred, logsize is the corresponding log file size,
and a ! is appended in case the build didn't succeed.
.Pp
The detailed timing info gives a run-down of the build, with clean, fetch,
prepare, patch (actually extract+patch), configure, build, fake, package, clean
detailed timing info.
Note that the actual build time starts at
.Sq extract
and finishes at
.Sq package .
.It engine.log
Build engine log.
Each line corresponds to a state change for a pkgpath and starts with the pid
of
.Nm ,
plus a timestamp of the log entry.
.Bl -tag -width BB:
.It ^
pkgpath temporarily put aside, because a job is running in the same directory.
.It B
pkgpath built.
.It I
pkgpath can be installed.
.It J
job to build pkgpath started.
Also records the host used for the build.
.It L
job did not start, existing lock detected.
.It N
job did not finish.
The host may have gone down.
.It P
built package is no longer required for anything.
.It Q
pkgpath queued as buildable whenever a slot is free.
.It T
pkgpath to build.
.It V
pkgpath put back in the buildable queue, after job that was running in
the same directory returned.
.El
.It packages/pkgname.log
one file or symlink per pkgname.
.It paths/some/path.log
one file or symlink per pkgpath.
.It stats.log
Simple log of the B=... line summaries.
Mostly useful for making plots and tweaking performance.
.It vars.log
Logs the directories that were walked in the ports tree for dependency
information.
.El
.Sh AUTHOR
Marc Espie
.Sh HISTORY
The original
.Nm dpb
command was written by Nikolai Sturm.
This version is a complete rewrite from scratch using all the stuff
we learnt over the years to make it better.
.Pp
There are still a number of changes to make.
.Pp
Being able to update packages on an existing machine would be nice as well.
Better build feedback for next builds would be nice: we need a way to
calibrate build logs that contain info for several machines (so that we
can gauge whether a machine is fast or slow).
It might make sense to have some kind of machine affinity for big packages
in a cluster, so that we avoid reinstalling big things on each machine if
we can get away with installing stuff on a single machine.
We should probably keep the pkgnames around with the pkgpath in the build-log,
so that we give more credibility to build times that correspond to the
exact same pkgnames.
.Pp
We should integrate mirroring functionalities.
This mostly involves having
.Sq special
jobs with no cpu requirements that can run locally,
and to have a step prior to
.Sq tobuild ,
where fetch would occur.
The same logic that was used for pkgpaths should be used to handle distfiles,
and we should probably add some kind of lock based on the ftp site being
used to grab distfiles.
(This is low priority, as most build machines currently being used already
have the distfiles).