364 lines
8.0 KiB
Perl
Executable File
364 lines
8.0 KiB
Perl
Executable File
#! /usr/bin/perl
|
|
|
|
# ex:ts=8 sw=4:
|
|
# $OpenBSD: dpb,v 1.142 2021/10/05 12:31:34 espie Exp $
|
|
#
|
|
# Copyright (c) 2010-2013 Marc Espie <espie@openbsd.org>
|
|
#
|
|
# Permission to use, copy, modify, and distribute this software for any
|
|
# purpose with or without fee is hereby granted, provided that the above
|
|
# copyright notice and this permission notice appear in all copies.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
|
|
use strict;
|
|
use warnings;
|
|
my $ports1;
|
|
use FindBin;
|
|
BEGIN {
|
|
$ports1 = $ENV{PORTSDIR} || '/usr/ports';
|
|
}
|
|
|
|
use lib ("$ports1/infrastructure/lib", "$FindBin::Bin/../lib");
|
|
|
|
package main;
|
|
|
|
use DPB::State;
|
|
use DPB::PkgPath;
|
|
use DPB::Core;
|
|
use DPB::Core::Init;
|
|
use DPB::HostProperties;
|
|
use DPB::Shell;
|
|
use DPB::Host;
|
|
use DPB::Vars;
|
|
use DPB::PortInfo;
|
|
use DPB::Engine;
|
|
use DPB::PortBuilder;
|
|
use OpenBSD::Error;
|
|
use DPB::Job;
|
|
use DPB::Grabber;
|
|
use DPB::Trace;
|
|
use File::Basename;
|
|
|
|
my $keep_going = 1;
|
|
|
|
sub show_days
|
|
{
|
|
my $days = shift;
|
|
if ($days == 0) {
|
|
return "";
|
|
} elsif ($days == 1) {
|
|
return "1 day ";
|
|
} else {
|
|
return "$days days ";
|
|
}
|
|
}
|
|
|
|
sub show_duration
|
|
{
|
|
my $secs = int(shift);
|
|
|
|
my $mn = int($secs/60);
|
|
my $hours = int($mn/60);
|
|
my $days= int($hours/24);
|
|
$secs %= 60;
|
|
$mn %= 60;
|
|
$hours %= 24;
|
|
|
|
return show_days($days).sprintf("%02d:%02d:%02d", $hours, $mn, $secs);
|
|
}
|
|
|
|
sub report_tty
|
|
{
|
|
my ($self, $state) = @_;
|
|
$state->{socket_location} //= basename($state->{external}{path});
|
|
my $t = CORE::time();
|
|
return DPB::Util->time2string($t)." [$$] ".$state->{socket_location}.
|
|
($keep_going ? "" : " STOPPED!").
|
|
" elapsed: ".show_duration($t-$state->{starttime})."\n";
|
|
}
|
|
|
|
sub affinityclass
|
|
{
|
|
if (DPB::Core::Init->hostcount > 1 ||
|
|
DPB::HostProperties->has_mem) {
|
|
require DPB::Affinity;
|
|
return "DPB::Affinity";
|
|
} else {
|
|
require DPB::AffinityStub;
|
|
return "DPB::AffinityStub";
|
|
}
|
|
}
|
|
|
|
my $subdirlist = {};
|
|
|
|
my $state = DPB::State->new;
|
|
|
|
my $cleanup =
|
|
sub {
|
|
my $sig = shift;
|
|
if ($$ == $state->{master_pid}) {
|
|
$> = 0;
|
|
DPB::Core->cleanup($sig // "INT", 0);
|
|
}
|
|
eval { $state->{external}->cleanup; };
|
|
};
|
|
|
|
for my $sig (qw(INT HUP TERM QUIT)) {
|
|
$SIG{$sig} = sub {
|
|
&$cleanup($sig);
|
|
$SIG{$sig} = 'DEFAULT';
|
|
kill $sig => $$;
|
|
};
|
|
}
|
|
|
|
my $trace = DPB::Trace->new($cleanup);
|
|
|
|
$state->handle_options;
|
|
$state->{reporter}->add_producers("main", "DPB::Core");
|
|
$trace->set_reporter($state->{reporter});
|
|
|
|
$state->{all} = 1;
|
|
|
|
my $default_handling =
|
|
sub {
|
|
my ($pkgpath, $weight) = @_;
|
|
if (defined $weight) {
|
|
$state->heuristics->set_weight($pkgpath);
|
|
}
|
|
$pkgpath->add_to_subdirlist($subdirlist);
|
|
$state->{all} = 0;
|
|
};
|
|
|
|
$state->interpret_paths(@{$state->{paths}}, @ARGV,
|
|
sub {
|
|
&$default_handling(@_);
|
|
});
|
|
$state->interpret_paths(@{$state->{ipaths}},
|
|
sub {
|
|
&$default_handling(@_);
|
|
my $p = shift;
|
|
$p->{wantinstall} = 1;
|
|
});
|
|
$state->interpret_paths(@{$state->{cpaths}},
|
|
sub {
|
|
my $p = shift;
|
|
$state->{dontclean}{$p->pkgpath} = 1;
|
|
});
|
|
|
|
$state->interpret_paths(@{$state->{xpaths}},
|
|
sub {
|
|
my $p = shift;
|
|
$p->{dontjunk} = 1;
|
|
});
|
|
|
|
if ($state->{bad_paths}) {
|
|
$state->usage("Bad package path #1",
|
|
join(" ", @{$state->{bad_paths}}));
|
|
}
|
|
|
|
if ($state->opt('a')) {
|
|
$state->{all} = 1;
|
|
}
|
|
$state->{build_once} //= $state->{all};
|
|
|
|
DPB::Core->reap;
|
|
$state->handle_build_files;
|
|
|
|
$state->{builder} = DPB::PortBuilder->new($state);
|
|
|
|
$state->{affinity} = affinityclass()->new($state, join("/", $state->logdir, "affinity"));
|
|
|
|
$state->{engine} = DPB::Engine->new($state);
|
|
$state->{reporter}->add_producers($state->engine);
|
|
$state->{grabber} = DPB::Grabber->new($state, \&handle_non_waiting_jobs);
|
|
|
|
print "Waiting for hosts to finish STARTUP...";
|
|
while (!DPB::Core->avail) {
|
|
DPB::Core->reap;
|
|
sleep 1;
|
|
}
|
|
|
|
# XXX placeholder
|
|
sub reread_config
|
|
{
|
|
}
|
|
|
|
my $core = DPB::Core->get;
|
|
print "ready on ", $core->hostname, "\n";
|
|
|
|
if (!$state->{fetch_only} && !$state->{scan_only} &&
|
|
DPB::Core::Init->hostcount == 1 && $core->prop->{jobs} == 1) {
|
|
$core->clone->mark_ready;
|
|
$state->{opt}{e} = 1;
|
|
}
|
|
|
|
my $dump = DPB::Util->make_hot($state->logger->append('dump'));
|
|
my $debug = DPB::Util->make_hot($state->logger->append('debug'));
|
|
$trace->set_logger($debug);
|
|
|
|
# this shouldn't happen too often
|
|
sub show_spinning
|
|
{
|
|
my $name = shift;
|
|
my $q = $state->engine->{$name}{queue};
|
|
print $debug "SPINNING ON $name\n";
|
|
while (my ($k, $v) = each %{$q->{o}}) {
|
|
print $debug $k, "=>", $v->logname, "\n";
|
|
}
|
|
}
|
|
|
|
# this is the usual event loop
|
|
sub handle_non_waiting_jobs
|
|
{
|
|
my $force_report = 0;
|
|
DPB::Core->reap;
|
|
$state->{external}->receive_commands;
|
|
$keep_going = !-e $state->logdir."/stop";
|
|
if ($keep_going) {
|
|
if (DPB::Core->avail > 1) {
|
|
$state->engine->recheck_errors;
|
|
}
|
|
if (DPB::Core->avail || DPB::Core::Fetcher->avail) {
|
|
$state->engine->check_buildable;
|
|
}
|
|
|
|
while (DPB::Core->avail && $state->engine->can_build) {
|
|
$force_report = 1;
|
|
next if $state->engine->start_new_job;
|
|
show_spinning('tobuild');
|
|
last;
|
|
}
|
|
while (DPB::Core::Fetcher->avail) {
|
|
if ($state->engine->can_fetch) {
|
|
$force_report = 1;
|
|
next if $state->engine->start_new_fetch;
|
|
show_spinning('tofetch');
|
|
}
|
|
if ($state->engine->can_roach) {
|
|
$force_report = 1;
|
|
next if $state->engine->start_new_roach;
|
|
show_spinning('toroach');
|
|
}
|
|
|
|
last;
|
|
}
|
|
}
|
|
DPB::Core->log_concurrency(CORE::time(), $state->{concurrent});
|
|
DPB::Core->wake_jobs;
|
|
$state->{reporter}->report($force_report);
|
|
}
|
|
|
|
sub main_loop
|
|
{
|
|
while (1) {
|
|
while (1) {
|
|
handle_non_waiting_jobs();
|
|
if (!DPB::Core->running) {
|
|
last if !$keep_going;
|
|
if (!$state->engine->can_build) {
|
|
$state->engine->check_buildable(1);
|
|
if (!$state->engine->can_build) {
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
if ($state->{fetch_only}) {
|
|
if (!DPB::Core::Fetcher->running) {
|
|
last if !$keep_going;
|
|
if (!$state->engine->can_fetch) {
|
|
$state->engine->check_buildable;
|
|
if (!$state->engine->can_fetch) {
|
|
last;
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
if (DPB::Core->running) {
|
|
DPB::Core->reap_wait;
|
|
}
|
|
}
|
|
if (!$state->opt('q') || !$state->engine->recheck_errors) {
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
|
|
my $skip = undef;
|
|
|
|
if (keys %$subdirlist > 0) {
|
|
$state->grabber->grab_subdirs($core, $subdirlist, $skip);
|
|
}
|
|
|
|
if ($state->{all} && !$state->{random} && !$state->defines("NO_QUICK_SCAN")) {
|
|
# when restarting interrupted dpb,
|
|
# find the most important paths first
|
|
my $list = $state->engine->find_best($state->{dependencies_log}, 25);
|
|
# if we have them, list them before the full ports tree walk.
|
|
if (@$list > 0) {
|
|
$skip = {};
|
|
for my $name (@$list) {
|
|
DPB::PkgPath->new($name)->add_to_subdirlist($skip);
|
|
}
|
|
$state->grabber->grab_subdirs($core, $skip, undef, 1);
|
|
}
|
|
}
|
|
|
|
$state->grabber->complete_subdirs($core, $skip);
|
|
|
|
if ($state->{all}) {
|
|
$state->grabber->grab_subdirs($core, undef, $skip);
|
|
}
|
|
|
|
|
|
$state->grabber->complete_subdirs($core);
|
|
# give back "our" core to the pool.
|
|
|
|
my $occupied = 0;
|
|
|
|
$state->grabber->forget_cache;
|
|
|
|
if ($state->{all}) {
|
|
$state->engine->dump_dependencies;
|
|
if (!$state->defines('NO_HISTORY')) {
|
|
if ($state->grabber->expire_old_distfiles($core,
|
|
$state->opt('e'))) {
|
|
$occupied = 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!$state->opt('e') && !$occupied) {
|
|
$core->mark_ready;
|
|
}
|
|
|
|
DPB::PkgPath->sanity_check($state);
|
|
$state->engine->check_buildable;
|
|
|
|
if ($state->{scan_only}) {
|
|
# very shortened loop
|
|
$state->{reporter}->report;
|
|
if (DPB::Core->running) {
|
|
DPB::Core->reap_wait;
|
|
}
|
|
} else {
|
|
# and let's wait for all jobs now.
|
|
DPB::Core->start_clock($state->{reporter});
|
|
main_loop();
|
|
}
|
|
|
|
$state->{reporter}->reset;
|
|
DPB::Core->cleanup;
|
|
$state->{external}->cleanup;
|
|
print "Elapsed time=", show_duration(CORE::time()-$state->{starttime}),"\n";
|
|
print $state->engine->report_tty($state);
|
|
$state->engine->end_dump($state->logger->append('dump'));
|
|
$state->engine->smart_dump($state->logger->append('summary'));
|