cpet/openbsd-ports
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- use separated Heuristics::Size object

Browse Source 
- choose size/nosize in Config.
- use affinity or affinitystub
- move locker creation to State, so we can init cores right away.
- as core init has run, we can call DPB::Core->reap (event loop) while
processing size and build stats, so that the startup script may start
slightly earlier.
- setup more options in Config for Reporter to use.
- scaffolding in SubEngine/Build to be able to use affinity info right away
This commit is contained in:
espie 2013-10-13 18:31:50 +00:00
parent 4bd97c750a
commit 8877767529
7 changed files with 66 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
infrastructure/bin/dpb
View File

@ -1,7 +1,7 @@
#! /usr/bin/perl
# ex:ts=8 sw=4:
# $OpenBSD: dpb,v 1.101 2013/10/11 10:26:44 espie Exp $
# $OpenBSD: dpb,v 1.102 2013/10/13 18:31:50 espie Exp $
#
# Copyright (c) 2010-2013 Marc Espie <espie@openbsd.org>
#
@ -112,7 +112,6 @@ use DPB::Engine;
use DPB::PortBuilder;
use DPB::Reporter;
use OpenBSD::Error;
use DPB::Locks;
use DPB::Job;
use DPB::Grabber;
@ -125,12 +124,22 @@ sub report
"\n";
}
sub affinityclass
{
if (DPB::Core::Init->hostcount > 1 ||
DPB::HostProperties->has_mem) {
require DPB::Affinity;
return "DPB::Affinity";
} else {
require DPB::AffinityStub;
return "DPB::AffinityStub";
}
}
my $subdirlist = {};
my $state = DPB::State->new('dpb');
$state->handle_options;
$state->{locker} = DPB::Locks->new($state, $state->{lockdir});
DPB::Core::Init->init_cores($state);
$state->{all} = 1;
@ -170,11 +179,13 @@ if ($state->opt('a')) {
$state->{all} = 1;
}
DPB::Core->reap;
$state->handle_build_files;
$state->{builder} = DPB::PortBuilder->new($state);
$state->{affinity} = DPB::Affinity->new($state, join("/", $state->logdir, "affinity"));
$state->{affinity} = affinityclass()->new($state, join("/", $state->logdir, "affinity"));
$state->{engine} = DPB::Engine->new($state);
$reporter = DPB::Reporter->new($state, "main", "DPB::Core", $state->engine);
while (!DPB::Core->avail) {

15
infrastructure/lib/DPB/Config.pm
View File

@ -1,5 +1,5 @@
# ex:ts=8 sw=4:
# $OpenBSD: Config.pm,v 1.16 2013/10/13 10:34:55 espie Exp $
# $OpenBSD: Config.pm,v 1.17 2013/10/13 18:31:50 espie Exp $
#
# Copyright (c) 2010-2013 Marc Espie <espie@openbsd.org>
#
@ -116,6 +116,19 @@ sub parse_command_line
} elsif (DPB::HostProperties->has_mem) {
$state->{wantsize} = 1;
}
if ($state->define_present('COLOR')) {
$state->{color} = $state->{subst}->value('COLOR');
}
if ($state->define_present('NO_CURSOR')) {
$state->{nocursor} = $state->{subst}->value('NO_CURSOR');
}
if (DPB::HostProperties->has_mem || $state->{wantsize}) {
require DPB::Heuristics::Size;
$state->{sizer} = DPB::Heuristics::Size->new($state);
} else {
require DPB::Heuristics::Nosize;
$state->{sizer} = DPB::Heuristics::Nosize->new($state);
}
if ($state->define_present('FETCH_JOBS') && !defined $state->{opt}{f}) {
$state->{opt}{f} = $state->{subst}->value('FETCH_JOBS');
}

5
infrastructure/lib/DPB/Engine.pm
View File

@ -1,5 +1,5 @@
# ex:ts=8 sw=4:
# $OpenBSD: Engine.pm,v 1.96 2013/10/09 06:20:56 espie Exp $
# $OpenBSD: Engine.pm,v 1.97 2013/10/13 18:31:50 espie Exp $
#
# Copyright (c) 2010-2013 Marc Espie <espie@openbsd.org>
#
@ -47,6 +47,7 @@ sub new
state => $state,
installable => {},
heuristics => $state->heuristics,
sizer => $state->sizer,
locker => $state->locker,
logger => $state->logger,
affinity => $state->{affinity},
@ -417,7 +418,7 @@ sub requeue
{
my ($self, $v) = @_;
$self->{buildable}->add($v);
$self->{heuristics}->finish_special($v);
$self->{sizer}->finished($v);
}
sub requeue_dist

70
infrastructure/lib/DPB/Heuristics.pm
View File

@ -1,5 +1,5 @@
# ex:ts=8 sw=4:
# $OpenBSD: Heuristics.pm,v 1.29 2013/10/12 14:11:23 espie Exp $
# $OpenBSD: Heuristics.pm,v 1.30 2013/10/13 18:31:50 espie Exp $
#
# Copyright (c) 2010-2013 Marc Espie <espie@openbsd.org>
#
@ -25,7 +25,7 @@ package DPB::Heuristics;
# for now, we don't create a separate object, we assume everything here is
# "global"
my (%bad_weight, %wrkdir, %needed_by, %pkgname);
my (%bad_weight, %needed_by);
our %weight;
sub new
@ -40,12 +40,6 @@ sub random
bless $self, "DPB::Heuristics::random";
}
sub set_logger
{
my ($self, $logger) = @_;
$self->{logger} = $logger;
}
# we set the "unknown" weight as max if we parsed a file.
my $default = 1;
@ -79,66 +73,6 @@ sub equates
}
}
sub add_size_info
{
my ($self, $path, $pkgname, $sz) = @_;
$wrkdir{$path->pkgpath_and_flavors} = $sz;
if (defined $pkgname) {
$pkgname{$path->fullpkgpath} = $pkgname;
}
}
sub match_pkgname
{
my ($self, $v) = @_;
my $p = $pkgname{$v->fullpkgpath};
if (!defined $p) {
return 0;
}
if ($p eq $v->fullpkgname) {
return 1;
}
return 0;
}
my $used_memory = {};
my $used_per_host = {};
sub build_in_memory
{
my ($self, $core, $v) = @_;
my $t = $core->memory;
return 0 if !defined $t;
# first match previous affinity
if ($v->{affinity}) {
return $v->{mem_affinity};
}
my $p = $v->pkgpath_and_flavors;
# we build in memory if we know this port and it's light enough
if (defined $wrkdir{$p}) {
my $hostname = $core->hostname;
$used_per_host->{$hostname} //= 0;
if ($used_per_host->{$hostname} + $wrkdir{$p} <= $t) {
$used_per_host->{$hostname} += $wrkdir{$p};
$used_memory->{$p} = $hostname;
return $wrkdir{$p};
}
}
return 0;
}
sub finish_special
{
my ($self, $v) = @_;
my $p = $v->pkgpath_and_flavors;
if (defined $used_memory->{$p}) {
my $hostname = $used_memory->{$p};
$used_per_host->{$hostname} -= $wrkdir{$p};
}
}
sub set_weight
{

7
infrastructure/lib/DPB/PortBuilder.pm
View File

@ -1,5 +1,5 @@
# ex:ts=8 sw=4:
# $OpenBSD: PortBuilder.pm,v 1.55 2013/10/07 17:50:29 espie Exp $
# $OpenBSD: PortBuilder.pm,v 1.56 2013/10/13 18:31:50 espie Exp $
#
# Copyright (c) 2010-2013 Marc Espie <espie@openbsd.org>
#
@ -39,6 +39,7 @@ sub new
fetch => $state->opt('f'),
wantsize => $state->{wantsize},
fullrepo => $state->fullrepo,
sizer => $state->sizer,
heuristics => $state->heuristics}, $class;
if ($state->opt('u') || $state->opt('U')) {
$self->{update} = 1;
@ -61,7 +62,7 @@ sub want_size
return 1;
}
# do we already have this stats ? don't run it every time
if ($self->{heuristics}->match_pkgname($v)) {
if ($self->{sizer}->match_pkgname($v)) {
return rand(10) < 1;
} else {
return 1;
@ -199,7 +200,7 @@ sub build
my ($self, $v, $core, $lock, $final_sub) = @_;
my $start = time();
my $log = $self->logger->make_logs($v);
my $memsize = $self->{heuristics}->build_in_memory($core, $v);
my $memsize = $self->{sizer}->build_in_memory($core, $v);
open my $fh, ">>", $log or die "can't open $log: $!";
if ($memsize) {

61
infrastructure/lib/DPB/State.pm
View File

@ -1,5 +1,5 @@
# ex:ts=8 sw=4:
# $OpenBSD: State.pm,v 1.2 2013/10/06 13:33:34 espie Exp $
# $OpenBSD: State.pm,v 1.3 2013/10/13 18:31:51 espie Exp $
#
# Copyright (c) 2010-2013 Marc Espie <espie@openbsd.org>
#
@ -26,11 +26,12 @@ use OpenBSD::Paths;
use DPB::Heuristics;
use DPB::PkgPath;
use DPB::Logger;
use DPB::Affinity;
use DPB::Config;
use File::Path;
use File::Basename;
use DPB::Core;
use DPB::Core::Init;
use DPB::Locks;
sub define_present
{
@ -135,13 +136,14 @@ sub handle_options
{
my $state = shift;
DPB::Config->parse_command_line($state);
$state->parse_size_file;
$state->{logger} = DPB::Logger->new($state->logdir, $state->opt('c'));
$state->{locker} = DPB::Locks->new($state, $state->{lockdir});
DPB::Core::Init->init_cores($state);
DPB::Core->reap;
$state->sizer->parse_size_file;
DPB::Limiter->setup($state->logger);
$state->heuristics->set_logger($state->logger);
$state->{concurrent} = $state->{logger}->open("concurrent");
DPB::Core->reap;
}
sub SUPER_handle_options
@ -160,6 +162,10 @@ sub heuristics
return shift->{heuristics};
}
sub sizer
{
return shift->{sizer};
}
sub locker
{
return shift->{locker};
@ -311,47 +317,4 @@ sub handle_build_files
$state->heuristics->finished_parsing;
}
sub parse_size_file
{
my $state = shift;
return if $state->{fetch_only};
open my $fh, '<', $state->opt('S') // $state->{size_log} or return;
print "Reading size stats...";
File::Path::mkpath(File::Basename::dirname($state->{size_log}));
my $rewrite = {};
my $_;
while (<$fh>) {
chomp;
my $pkgname;
my ($pkgpath, $sz, $sz2) = split(/\s+/, $_);
my $i = " $sz";
if ($pkgpath =~ m/^(.*)\((.*)\)$/) {
($pkgpath, $pkgname) = ($1, $2);
if ($state->opt('S')) {
undef $pkgname;
} else {
$i ="($pkgname) $sz";
}
}
if (defined $sz2) {
$sz += $sz2;
$i .=" $sz2";
}
$rewrite->{$pkgpath} = $i;
my $o = DPB::PkgPath->new($pkgpath);
DPB::Heuristics->add_size_info($o, $pkgname, $sz);
}
close $fh;
print "zapping old stuff...";
open $fh, '>', $state->{size_log}.'.part' or return;
for my $p (sort keys %$rewrite) {
print $fh "$p$rewrite->{$p}\n";
}
close $fh;
print "Done\n";
rename $state->{size_log}.'.part', $state->{size_log};
}
1;

17
infrastructure/lib/DPB/SubEngine/Build.pm
View File

@ -1,5 +1,5 @@
# ex:ts=8 sw=4:
# $OpenBSD: Build.pm,v 1.6 2013/10/07 20:23:13 espie Exp $
# $OpenBSD: Build.pm,v 1.7 2013/10/13 18:31:51 espie Exp $
#
# Copyright (c) 2010-2013 Marc Espie <espie@openbsd.org>
#
@ -265,7 +265,20 @@ sub end_build
{
my ($self, $v) = @_;
$self->{engine}{affinity}->finished($v);
$self->{engine}{heuristics}->finish_special($v);
$self->{engine}{sizer}->finished($v);
}
sub sorted
{
my ($self, $core) = @_;
return $self->{engine}{affinity}->sorted($self->{queue}, $core);
}
sub add
{
my ($self, $v) = @_;
$self->{engine}{affinity}->has_in_queue($v);
$self->SUPER::add($v);
}
1;