From 54abc612eb8e689858771dff5a1c9168f2aa9234 Mon Sep 17 00:00:00 2001 From: espie Date: Sat, 16 Nov 2013 16:39:28 +0000 Subject: [PATCH] make the data in permanent logs a bit more self-describing, so that we may add new fields in the future without atrocious hacks for bw compatibility. (grows the files a bit, but quite acceptable). --- infrastructure/lib/DPB/Heuristics/Size.pm | 31 +++------ infrastructure/lib/DPB/Job/Port.pm | 12 ++-- infrastructure/lib/DPB/PortBuilder.pm | 11 +++- infrastructure/lib/DPB/Serialize.pm | 80 +++++++++++++++++++++++ infrastructure/lib/DPB/State.pm | 25 +++---- 5 files changed, 114 insertions(+), 45 deletions(-) create mode 100644 infrastructure/lib/DPB/Serialize.pm diff --git a/infrastructure/lib/DPB/Heuristics/Size.pm b/infrastructure/lib/DPB/Heuristics/Size.pm index 8cec589dd13..e3fd477fb3e 100644 --- a/infrastructure/lib/DPB/Heuristics/Size.pm +++ b/infrastructure/lib/DPB/Heuristics/Size.pm @@ -1,6 +1,6 @@ # ex:ts=8 sw=4: -# $OpenBSD: Size.pm,v 1.2 2013/11/16 13:06:00 espie Exp $ +# $OpenBSD: Size.pm,v 1.3 2013/11/16 16:39:28 espie Exp $ # # Copyright (c) 2010-2013 Marc Espie # @@ -23,6 +23,8 @@ use warnings; package DPB::Heuristics::Size; my (%wrkdir, %pkgname); +use DPB::Serialize; + sub new { my ($class, $state) = @_; @@ -99,33 +101,20 @@ sub parse_size_file print "Reading size stats..."; File::Path::mkpath(File::Basename::dirname($state->{size_log})); - my $rewrite = {}; + my @rewrite = (); my $_; while (<$fh>) { chomp; - my $pkgname; - my ($pkgpath, $sz, $ts) = split(/\s+/, $_); - my $i = " $sz"; - if ($pkgpath =~ m/^(.*)\((.*)\)$/) { - ($pkgpath, $pkgname) = ($1, $2); - if ($state->opt('S')) { - undef $pkgname; - } else { - $i ="($pkgname) $sz"; - } - } - if (defined $ts) { - $i .=" $ts"; - } - $rewrite->{$pkgpath} = $i; - my $o = DPB::PkgPath->new($pkgpath); - $self->add_size_info($o, $pkgname, $sz); + my $s = DPB::Serialize::Size->read($_); + push(@rewrite, $s); + $self->add_size_info(DPB::PkgPath->new($s->{pkgpath}), + $s->{pkgname}, $s->{size}); } close $fh; print "zapping old stuff..."; open $fh, '>', $state->{size_log}.'.part' or return; - for my $p (sort keys %$rewrite) { - print $fh "$p$rewrite->{$p}\n"; + for my $p (sort {$a->{pkgpath} cmp $b->{pkgpath}} @rewrite) { + print $fh DPB::Serialize::Size->write($p), "\n"; } close $fh; print "Done\n"; diff --git a/infrastructure/lib/DPB/Job/Port.pm b/infrastructure/lib/DPB/Job/Port.pm index e2acc1f4fc6..3a3dc42bd62 100644 --- a/infrastructure/lib/DPB/Job/Port.pm +++ b/infrastructure/lib/DPB/Job/Port.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: Port.pm,v 1.139 2013/11/16 13:06:00 espie Exp $ +# $OpenBSD: Port.pm,v 1.140 2013/11/16 16:39:28 espie Exp $ # # Copyright (c) 2010-2013 Marc Espie # @@ -613,14 +613,18 @@ sub finalize if ($line =~ m/^\s*(\d+)\s+/) { my $sz = $1; my $job = $core->job; - my $info = $job->{path}."(".$job->{v}->fullpkgname.") $sz ".CORE::time()."\n"; - print {$job->{builder}{logsize}} $info; + my $info = DPB::Serialize::Size->write({ + pkgpath => $job->{path}, + pkname => $job->{v}->fullpkgname, + size => $sz, + ts => CORE::time }); + print {$job->{builder}{logsize}} $info, "\n"; # XXX the rolling log might be shared with other dpb # so it can be rewritten and sorted # don't keep a handle on it, so that we always # append new information to the correct filename open(my $fh2, '>>', $job->{builder}{state}{size_log}); - print $fh2 $info; + print $fh2 $info."\n"; } } close($fh); diff --git a/infrastructure/lib/DPB/PortBuilder.pm b/infrastructure/lib/DPB/PortBuilder.pm index 116ca287527..6fc225ed7b1 100644 --- a/infrastructure/lib/DPB/PortBuilder.pm +++ b/infrastructure/lib/DPB/PortBuilder.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: PortBuilder.pm,v 1.58 2013/11/16 13:06:00 espie Exp $ +# $OpenBSD: PortBuilder.pm,v 1.59 2013/11/16 16:39:28 espie Exp $ # # Copyright (c) 2010-2013 Marc Espie # @@ -24,6 +24,7 @@ package DPB::PortBuilder; use File::Path; use DPB::Util; use DPB::Job::Port; +use DPB::Serialize; sub new { @@ -175,8 +176,12 @@ sub report } else { print $log "\n"; open my $fh, '>>', $self->{state}{permanent_log}; - print $fh join(' ', $pkgpath, $host, $job->totaltime, $sz, - CORE::time()), "\n"; + print $fh DPB::Serialize::Build->write({ + pkgpath => $pkgpath, + host => $host, + time => $job->totaltime, + size => $sz, + ts => CORE::time }), "\n"; } } diff --git a/infrastructure/lib/DPB/Serialize.pm b/infrastructure/lib/DPB/Serialize.pm new file mode 100644 index 00000000000..586a7b6005d --- /dev/null +++ b/infrastructure/lib/DPB/Serialize.pm @@ -0,0 +1,80 @@ +# ex:ts=8 sw=4: +# $OpenBSD: Serialize.pm,v 1.1 2013/11/16 16:39:28 espie Exp $ +# +# Copyright (c) 2013 Marc Espie +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +use strict; +use warnings; + +# use to read/write some log files in a sensible fashion +package DPB::Serialize; + +sub read +{ + my ($class, $line) = @_; + + chomp $line; + my @list = $class->list; + my $r = {}; + my @e; + ($r->{pkgpath}, @e) = split(/\s+/, $line); + if ($r->{pkgpath} =~ m/^(.*)\((.*)\)$/) { + ($r->{pkgpath}, $r->{pkgname}) = ($1, $2); + } + while (@e > 0) { + my $v = shift @e; + if ($v =~ m/^(\w+)\=(.*)$/) { + $r->{$1} = $2; + } else { + my $k = shift @list or return; + $r->{$k} = $v; + } + } + return $r; +} + +sub write +{ + my ($class, $r) = @_; + + my @r = (); + if ($r->{pkgname}) { + push(@r, "$r->{pkgpath}($r->{pkgname})"); + } else { + push(@r, $r->{pkgpath}); + } + for my $k ($class->list) { + if (defined $r->{$k}) { + push(@r, "$k=$r->{$k}"); + } + } + return join(' ', @r); +} + +package DPB::Serialize::Build; +our @ISA = qw(DPB::Serialize); +sub list +{ + return (qw(host time size ts)); +} + +package DPB::Serialize::Size; +our @ISA = qw(DPB::Serialize); +sub list +{ + return (qw(size ts)); +} + +1; diff --git a/infrastructure/lib/DPB/State.pm b/infrastructure/lib/DPB/State.pm index 8150744ac5e..169bdecab76 100644 --- a/infrastructure/lib/DPB/State.pm +++ b/infrastructure/lib/DPB/State.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: State.pm,v 1.6 2013/11/16 13:06:00 espie Exp $ +# $OpenBSD: State.pm,v 1.7 2013/11/16 16:39:28 espie Exp $ # # Copyright (c) 2010-2013 Marc Espie # @@ -32,6 +32,7 @@ use File::Basename; use DPB::Core; use DPB::Core::Init; use DPB::Locks; +use DPB::Serialize; sub define_present { @@ -258,15 +259,10 @@ sub parse_build_file open my $fh, '<', $fname or return; my $_; while (<$fh>) { - chomp; - next if $_ =~ m/!$/; - my ($pkgpath, $host, $time, $sz, @rest) = parse_build_line($_); - next if !defined $sz; - my $o = DPB::PkgPath->new($pkgpath); - my $s = {host => $host, time => $time, sz => $sz}; - if (@rest > 0 && $rest[0] =~ m/^\d+$/) { - $s->{ts} = $rest[0]; - } + next if m/!$/; + my $s = DPB::Serialize::Build->read($_); + next if !defined $s->{size}; + my $o = DPB::PkgPath->new($s->{pkgpath}); push(@{$o->{stats}}, $s); } } @@ -279,7 +275,7 @@ sub add_build_info my ($i, $time, $sz, $host); for my $s (@{$p->{stats}}) { $time += $s->{time}; - $sz += $s->{sz}; + $sz += $s->{size}; $i++; $host = $s->{host}; # XXX } @@ -299,12 +295,7 @@ sub rewrite_build_info next unless defined $p->{stats}; shift @{$p->{stats}} while @{$p->{stats}} > 10; for my $s (@{$p->{stats}}) { - my @l = ($p->fullpkgpath, $s->{host}, $s->{time}, - $s->{sz}); - if ($s->{ts}) { - push(@l, $s->{ts}); - } - print $f join(' ', @l), "\n"; + print $f DPB::Serialize::Build->write($s), "\n"; } delete $p->{stats}; }