cpet/openbsd-ports
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

beginning of a fetch-only option that is supposed to replace mirror-maker

Browse Source 
eventually.

- fetch all files
- ignore ignores
- specific builder that doesn't look at existing packages

currently: does not stop when  fetch is finished, which is somewhat of the
remaining issue.

Also: change stats to store pid, to make sense of interleaved log files.
This commit is contained in:
espie 2011-09-13 09:46:53 +00:00
parent 15bab017a1
commit ed1516a867
5 changed files with 56 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
infrastructure/bin/dpb
View File

@ -1,7 +1,7 @@
#! /usr/bin/perl
# ex:ts=8 sw=4:
# $OpenBSD: dpb,v 1.23 2011/07/14 12:44:39 espie Exp $
# $OpenBSD: dpb,v 1.24 2011/09/13 09:46:53 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
@ -89,12 +89,21 @@ sub handle_options
},
};
$state->SUPER::handle_options('aceqrRstuUvh:xA:f:j:m:P:b:L:S:',
"[-acerRsuUvx] [-A arch] [-f N] [-j N] [-P plist] [-h hosts] [-L logdir]",
"[-b log] [-t ctimeout] [-m threshold] [path ...]");
$state->SUPER::handle_options('aceqrRstuUvh:xA:f:F:j:m:P:b:L:S:',
"[-acerRsuUvx] [-A arch] [-f N] [-F N][-j N] [-P plist] [-h hosts]",
"[-L logdir] [-b log] [-t ctimeout] [-m threshold] [path ...]");
$state->{fullrepo} = join("/", $state->{repo}, $state->arch, "all");
$state->{logdir} //= $ENV{LOGDIR} //
join("/", $state->ports, "logs", $state->arch);
if (defined $state->{opt}{F}) {
if (defined $state->{opt}{j} || defined $state->{opt}{f}) {
$state->usage("Can't use -F with -f or -j");
}
$state->{fetch_only} = 1;
$state->{opt}{f} = $state->{opt}{F};
$state->{opt}{j} = 1;
$state->{opt}{e} = 1;
}
$state->{opt}{f} //= 2;
if (defined $state->opt('j')) {
if ($state->localarch ne $state->arch) {

23
infrastructure/lib/DPB/Engine.pm
View File

@ -1,5 +1,5 @@
# ex:ts=8 sw=4:
# $OpenBSD: Engine.pm,v 1.25 2011/06/03 13:38:58 espie Exp $
# $OpenBSD: Engine.pm,v 1.26 2011/09/13 09:46:53 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
@ -211,6 +211,15 @@ sub end_build
$self->{engine}{heuristics}->finish_special($v);
}
# for fetch-only, we do the same as Build, except we're never happy
package DPB::SubEngine::NoBuild;
our @ISA = qw(DPB::SubEngine::Build);
sub is_done
{
return 0;
}
package DPB::SubEngine::Fetch;
our @ISA = qw(DPB::SubEngine);
sub new_queue
@ -266,7 +275,8 @@ sub new
locks => [],
requeued => [],
ignored => []}, $class;
$o->{buildable} = DPB::SubEngine::Build->new($o, $state->builder);
$o->{buildable} = ($state->{fetch_only} ? "DPB::SubEngine::NoBuild"
: "DPB::SubEngine::Build")->new($o, $state->builder);
if ($state->opt('f')) {
$o->{tofetch} = DPB::SubEngine::Fetch->new($o);
}
@ -331,7 +341,9 @@ sub fetchcount
{
my ($self, $q, $t)= @_;
return () unless defined $self->{tofetch};
if ($q < 30) {
if ($self->{state}{fetch_only}) {
$self->{tofetch}{queue}->set_fetchonly;
} elsif ($q < 30) {
$self->{tofetch}{queue}->set_h1;
} else {
$self->{tofetch}{queue}->set_h2;
@ -372,7 +384,7 @@ sub stats
my $line = $self->statline;
if ($line ne $self->{statline}) {
$self->{statline} = $line;
print $fh $self->{ts}, " ", $line, "\n";
print $fh $$, " ", $self->{ts}, " ", $line, "\n";
}
}
@ -511,7 +523,8 @@ sub new_path
$self->log_fetch($v);
delete $v->{info}{FETCH_MANUALLY};
}
if (defined $v->{info}{IGNORE}) {
if (defined $v->{info}{IGNORE} &&
!$self->{state}->{fetch_only}) {
push(@{$self->{ignored}}, $v);
return;
}

9
infrastructure/lib/DPB/Fetch.pm
View File

@ -1,5 +1,5 @@
# ex:ts=8 sw=4:
# $OpenBSD: Fetch.pm,v 1.14 2011/08/30 17:44:28 espie Exp $
# $OpenBSD: Fetch.pm,v 1.15 2011/09/13 09:46:53 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
@ -183,7 +183,7 @@ sub read_checksums
sub build_distinfo
{
my ($self, $h) = @_;
my ($self, $h, $fetch_only) = @_;
my $distinfo = {};
for my $v (values %$h) {
my $info = $v->{info};
@ -222,7 +222,10 @@ sub build_distinfo
$files->{$file} = $file;
}
for my $d (keys %{$info->{SUPDISTFILES}}) {
&$build($d);
my $file = &$build($d);
if ($fetch_only) {
$files->{$file} = $file;
}
}
for my $k (qw(DIST_SUBDIR CHECKSUM_FILE DISTFILES
PATCHFILES SUPDISTFILES MASTER_SITES MASTER_SITES0

17
infrastructure/lib/DPB/Heuristics.pm
View File

@ -1,5 +1,5 @@
# ex:ts=8 sw=4:
# $OpenBSD: Heuristics.pm,v 1.9 2011/07/14 11:03:13 espie Exp $
# $OpenBSD: Heuristics.pm,v 1.10 2011/09/13 09:46:53 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
@ -498,6 +498,11 @@ sub set_h2
bless shift, "DPB::Heuristics::FetchQueue2";
}
sub set_fetchonly
{
bless shift, "DPB::Heuristics::FetchOnlyQueue";
}
sub sorted
{
my $self = shift;
@ -546,4 +551,14 @@ sub sorted_values
@l];
}
package DPB::Heuristics::FetchOnlyQueue;
our @ISA = qw(DPB::Heuristics::FetchQueue);
# for fetch-only, grab all files, largest ones first.
sub sorted_values
{
my $self = shift;
return [sort {$a->{sz} <=> $b->{sz}} values %{$self->{o}}];
}
1;

5
infrastructure/lib/DPB/Vars.pm
View File

@ -1,5 +1,5 @@
# ex:ts=8 sw=4:
# $OpenBSD: Vars.pm,v 1.17 2011/06/04 12:58:24 espie Exp $
# $OpenBSD: Vars.pm,v 1.18 2011/09/13 09:46:53 espie Exp $
#
# Copyright (c) 2010 Marc Espie <espie@openbsd.org>
#
@ -121,7 +121,8 @@ sub grab_list
for my $v (values %$h) {
$v->handle_default($h);
}
$grabber->{fetch}->build_distinfo($h);
$grabber->{fetch}->build_distinfo($h,
$grabber->{state}->{fetch_only});
DPB::PkgPath->merge_depends($h);
&$code($h);
$h = {};