tweak the way library dependencies are resolved to speed them up.

Now, resolve-lib can take a big list of libraries with full paths,
and it can solve a big list of spec at once.
Basically, we move most of the parsing of spec paths into resolve-lib.

Since print-package-signature does build a full list of libs, let's solve
it all at once, instead of invoking a costly perl script repeatedly.

Add some caching possibilities for out-of-date. Specifically:
- store libraries for each package under the directory _PORT_LIBS_CACHE
- use the dependency cache _DEPENDS_FILE to avoid recreating dependency
chains, add a new file _DEPENDS_CACHE that will accumulate all dependencies,
and extract these with a simple script extract-dependencies.

Use echo to build libraries lists instead of ls, that's a bit simpler...

Some more clean-up will happen: it's probably simpler to parse libspecs
at once, extract the libraries needed and go fetch the corresponding libraries
just once.
This commit is contained in:
espie 2005-10-09 12:01:22 +00:00
parent da2d577443
commit 43ed8edb9c
3 changed files with 263 additions and 153 deletions

View File

@ -0,0 +1,56 @@
#! /usr/bin/perl
# $OpenBSD: extract-dependencies,v 1.1 2005/10/09 12:01:22 espie Exp $
#
# Copyright (c) 2005 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# Usage: extract-dependencies < 'tsort-pairs' seed
# extracts all the dependencies for seed from a list that contains more
# than that.
use strict;
use warnings;
# build dependency graph
my $dep = {};
while(<STDIN>) {
chomp;
my($a, $b) = split(/\s+/, $_);
$dep->{$a} = {} unless defined $dep->{$a};
$dep->{$a}->{$b} = 1;
}
# get the starting point
my $pkgpath = shift;
my @todo = ();
my $done = {};
# walk the graph repeatedly starting from it
push(@todo, $pkgpath);
while (my $x = shift @todo) {
next if $done->{$x};
$done->{$x} = 1;
next unless defined $dep->{$x};
push(@todo, keys %{$dep->{$x}});
}
# display all nodes, except for the seed
for my $d (keys %$done) {
next if $d eq $pkgpath;
print "$d\n";
}

View File

@ -1,8 +1,8 @@
#!/usr/bin/perl
# $OpenBSD: resolve-lib,v 1.4 2002/11/28 19:20:37 espie Exp $
#! /usr/bin/perl
# $OpenBSD: resolve-lib,v 1.5 2005/10/09 12:01:22 espie Exp $
#
# Copyright (c) 2001, 2005 Marc Espie <espie@openbsd.org>
#
# Copyright (c) 2001
# Marc Espie. All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
@ -25,108 +25,158 @@
# SUCH DAMAGE.
use strict;
use warnings;
my $LOCALBASE = $ENV{'LOCALBASE'};
my $X11BASE = $ENV{'X11BASE'};
sub parse_spec_path
{
my $req = shift;
my $spec = $req->{spec};
if ($spec =~ m|^/.*/|) {
$req->{specdir}=$&;
$req->{prefix} = '';
$req->{libname} = $';
} elsif ($spec =~ m|^.*/|) {
$req->{specdir} = "$LOCALBASE/$&";
$req->{prefix} =$&;
$req->{libname} = $';
} else {
$req->{libname} = $spec;
$req->{prefix} = '';
}
}
sub parse_spec_versions
{
my $spec = shift;
my $noshared = shift;
my $req = {noshared => $noshared, strict => 0, sharedonly => 0 };
$spec =~ s/\.$//;
if ($spec =~ m/\.a$/) {
$spec = $`;
$req->{noshared} = 1;
} else {
if ($spec =~ m/\.(\d+)\.(\d+)$/) {
$req->{major} = $1;
$req->{minor} = $2;
$spec = $`;
} elsif ($spec =~ m/\.\=(\d+)\.(\d+)$/) {
$req->{major} = $1;
$req->{minor} = $2;
$spec = $`;
$req->{strict} = 1;
} elsif ($spec =~ m/\.(\d+)$/) {
$req->{major} = $1;
$req->{minor} = 0;
$spec = $`;
} elsif ($spec =~ m/\.\=(\d+)$/) {
$req->{major} = $1;
$req->{minor} = 0;
$req->{strict} = 1;
$spec = $`;
} else {
$req->{major} = 0;
$req->{minor} = 0;
}
if ($spec =~ m/\.so$/) {
$spec = $`;
$req->{sharedonly} = 1;
}
}
$req->{spec} = $spec;
return $req;
}
sub parse_spec
{
my ($spec, $noshared) = @_;
my $req = parse_spec_versions($spec);
parse_spec_path($req);
return $req;
}
sub solve_spec
{
my $spec = shift;
my $noshared = shift;
my $req = parse_spec($spec, $noshared);
my $found_shared;
my $found_goodshared;
my $bestmajor=-1; my $bestminor=-1;
my $found_unshared;
my $libname = $req->{libname};
for my $file (@_) {
next unless $file =~ m|(.*/)|;
if (defined $req->{specdir}) {
next unless $1 eq $req->{specdir};
} else {
next unless $1 eq '/usr/lib/' or
$1 eq "$LOCALBASE/lib/" or
$1 eq "$X11BASE/lib/";
}
local $_=$';
if (!$req->{noshared} &&
m/^lib\Q$libname\E\.so\.(\d+)\.(\d+)$/) {
$found_shared = 1;
my $major = $1;
my $minor = $2;
if ($req->{strict}) {
if ($major > $req->{major}) {
print "Error: strict library too high\n";
last;
}
if ($major < $req->{major}) {
next;
}
if ($minor < $req->{minor}) {
next;
}
$found_goodshared = 1;
$bestmajor = $major;
if ($minor >= $bestminor) {
$bestminor = $minor;
}
} else {
if ($major < $req->{major} ||
($major == $req->{major} && $minor < $req->{minor})) {
next;
}
$found_goodshared = 1;
if ($major > $bestmajor || ($major == $bestmajor &&
$minor > $bestminor)) {
$bestmajor = $major;
$bestminor = $minor;
}
}
} elsif (!$req->{sharedonly} && m/^lib\Q$libname\E\.a$/) {
$found_unshared = 1;
}
}
if ($found_goodshared) {
print $req->{prefix},"$libname.$bestmajor.$bestminor\n";
} elsif ($found_shared) {
print STDERR "Error: bad shared library\n";
} elsif ($found_unshared) {
print $req->{prefix},"$libname.a\n";
} else {
print STDERR "Missing library for $spec\n";
}
}
my $noshared;
my $sharedonly;
my $strict;
my $reqmajor;
my $reqminor;
# Grab arguments
$_=shift;
if ($_ eq '-noshared') {
if ($ARGV[0] eq '-noshared') {
$noshared = 1;
$_ = shift;
}
s/\.$//;
# Parse spec
if (m/\.a$/) {
$_ = $`;
$noshared = 1;
} else {
if (m/\.(\d+)\.(\d+)$/) {
$reqmajor = $1;
$reqminor = $2;
$_ = $`;
} elsif (m/\.\=(\d+)\.(\d+)$/) {
$reqmajor = $1;
$reqminor = $2;
$_ = $`;
$strict = 1;
} elsif (m/\.(\d+)$/) {
$reqmajor = $1;
$reqminor = 0;
$_ = $`;
} elsif (m/\.\=(\d+)$/) {
$reqmajor = $1;
$reqminor = 0;
$strict = 1;
$_ = $`;
}
if (m/\.so$/) {
$_ = $`;
$sharedonly = 1;
}
}
{
my $libname=$_;
my $bestmajor=-1;
my $bestminor=-1;
my $found_shared;
my $found_goodshared;
my $found_unshared;
while(<>) {
chomp;
if (!$noshared && m/^lib\Q$libname\E\.so\.(\d+)\.(\d+)$/) {
$found_shared = 1;
my $major = $1;
my $minor = $2;
if ($strict) {
if ($major > $reqmajor) {
print "Error: strict library too high\n";
exit(0);
}
if ($major < $reqmajor) {
next;
}
if ($minor < $reqminor) {
next;
}
$found_goodshared = 1;
$bestmajor = $major;
if ($minor >= $bestminor) {
$bestminor = $minor;
}
} else {
if ($major < $reqmajor || ($major == $reqmajor &&
$minor < $reqminor)) {
next;
}
$found_goodshared = 1;
if ($major > $bestmajor || ($major == $bestmajor &&
$minor > $bestminor)) {
$bestmajor = $major;
$bestminor = $minor;
}
}
} elsif (!$sharedonly && m/^lib\Q$libname\E\.a$/) {
$found_unshared = 1;
}
}
if ($found_goodshared) {
print "$libname.$bestmajor.$bestminor\n";
} elsif ($found_shared) {
print "Error: bad shared library\n";
} elsif ($found_unshared) {
print "$libname.a\n";
} else {
print "Missing library\n";
shift;
}
my @available = split(/\s+/, <STDIN>);
for my $spec (@ARGV) {
solve_spec($spec, $noshared, @available);
}

View File

@ -1,6 +1,6 @@
#-*- mode: Makefile; tab-width: 4; -*-
# ex:ts=4 sw=4 filetype=make:
# $OpenBSD: bsd.port.mk,v 1.713 2005/10/07 21:08:16 espie Exp $
# $OpenBSD: bsd.port.mk,v 1.714 2005/10/09 12:01:22 espie Exp $
# $FreeBSD: bsd.port.mk,v 1.264 1996/12/25 02:27:44 imp Exp $
# $NetBSD: bsd.port.mk,v 1.62 1998/04/09 12:47:02 hubertf Exp $
#
@ -1044,22 +1044,20 @@ _noshared=
_libresolve_fragment = \
case "$$d" in \
*/*) shprefix="$${d%/*}/"; shdir="${LOCALBASE}/$${d%/*}"; \
d=$${d\#\#*/};; \
*) shprefix="" shdir="${LOCALBASE}/lib";; \
*/*) shdir="${LOCALBASE}/$${d%/*}";; \
*) shdir="${LOCALBASE}/lib";; \
esac; \
check=`eval $$listlibs| perl \
check=`eval $$listlibs 2>/dev/null| LOCALBASE=${LOCALBASE} X11BASE=${X11BASE} perl \
${PORTSDIR}/infrastructure/build/resolve-lib ${_noshared} $$d` \
|| true
_syslibresolve_fragment = \
case "$$d" in \
/*) shdir="$${d%/*}/"; shprefix=""; d=$${d\#\#*/};; \
*/*) shprefix="$${d%/*}/"; shdir="${DEPBASE}/$${d%/*}"; \
d=$${d\#\#*/};; \
*) shprefix="" shdir="${DEPBASE}/lib"; listlibs="$$listlibs; ls /usr/lib /usr/X11R6/lib 2>/dev/null";; \
/*) shdir="$${d%/*}/";; \
*/*) shdir="${DEPBASE}/$${d%/*}";; \
*) shdir="${DEPBASE}/lib"; listlibs="$$listlibs /usr/lib/lib* ${X11BASE}/lib/lib*";; \
esac; \
check=`eval $$listlibs| perl \
check=`eval $$listlibs 2>/dev/null| LOCALBASE=${LOCALBASE} X11BASE=${X11BASE} perl \
${PORTSDIR}/infrastructure/build/resolve-lib ${_noshared} $$d` \
|| true
@ -1070,7 +1068,7 @@ _lib_depends_fragment = \
found2=false; \
what="$$dep ($$pkg)"; \
IFS=,; bad=false; for d in $$dep; do \
listlibs='ls $$shdir 2>/dev/null'; \
listlibs='echo $$shdir/lib*'; \
${_libresolve_fragment}; \
case "$$check" in \
Missing\ library) bad=true; msg="$$msg $$d missing...";; \
@ -1172,9 +1170,15 @@ _FMN+= ${PKGPATH}/${FULLPKGNAME${_S}}
.if defined(LIB_DEPENDS) && ${NO_SHARED_LIBS:L} != "yes"
_ALWAYS_DEP2 = ${LIB_DEPENDS:C/^[^:]*:([^:]*:[^:]*).*$/\1/}
_ALWAYS_DEP= ${_ALWAYS_DEP2:C/[^:]*://}
_DEPLIBS=${LIB_DEPENDS:C/:.*//:S/,/ /g}
.else
_ALWAYS_DEP2=
_ALWAYS_DEP=
_DEPLIBS=
.endif
.if defined(WANTLIB)
_DEPLIBS+=${WANTLIB}
.endif
.if defined(RUN_DEPENDS)
@ -2401,9 +2405,15 @@ ${_i:L}-depends-list:
print-package-signature:
@echo -n ${FULLPKGNAME${SUBPACKAGE}}
@cd ${.CURDIR} && PACKAGING='${SUBPACKAGE}' LIST_LIBS=`${MAKE} _list-port-libs` ${MAKE} _print-package-signature-helper | \
.if !empty(_DEPLIBS)
@cd ${.CURDIR} && PACKAGING='${SUBPACKAGE}' LIST_LIBS=`${MAKE} _list-port-libs` ${MAKE} _print-package-signature-lib _print-package-signature-run| \
sort -u| \
while read i; do echo -n ",$$i"; done
.else
@cd ${.CURDIR} && PACKAGING='${SUBPACKAGE}' ${MAKE} _print-package-signature-run | \
sort -u| \
while read i; do echo -n ",$$i"; done
.endif
@echo
_print-package-args:
@ -2427,10 +2437,10 @@ _print-package-args:
default=`eval $$toset ${MAKE} _print-packagename`; \
case "X$$pkg" in X) pkg=`echo $$default|sed -e 's,-[0-9].*,-*,'`;; esac; \
if pkg_info -q -e $$pkg; then \
listlibs='ls ${DEPDIR}$$shdir 2>/dev/null'; \
listlibs='echo ${DEPDIR}$$shdir/lib*'; \
else \
eval 1>&2 $$toset ${MAKE} ${PKGREPOSITORY}/$$default.tgz; \
listlibs='pkg_info -L -K ${PKGREPOSITORY}/$$default.tgz|grep "^@lib $$shdir" |sed -e "s:@lib $$shdir/::"'; \
listlibs='pkg_info -L -K ${PKGREPOSITORY}/$$default.tgz|grep -e "^@lib $$shdir/lib" -e "^$$shdir/lib.*.a"|sed -e "s:@lib "'; \
fi; \
IFS=,; for d in $$dep; do \
${_libresolve_fragment}; \
@ -2440,14 +2450,14 @@ _print-package-args:
echo 1>&2 "Can't resolve libspec $$d"; \
exit 1;; \
*) \
echo "-W $$shprefix$$check";; \
echo "-W $$check";; \
esac; \
done; \
echo "-P $$pkgpath:$$pkg:$$default"; \
}
. endfor
. for _i in ${WANTLIB}
@d=${_i}; listlibs='ls $$shdir 2>/dev/null'; \
@d=${_i}; listlibs='echo $$shdir/lib*'; \
${_syslibresolve_fragment}; \
case "$$check" in \
*.a) ;; \
@ -2455,67 +2465,61 @@ _print-package-args:
echo 1>&2 "Can't resolve libspec $$d"; \
exit 1;; \
*) \
echo "-W $$shprefix$$check";; \
echo "-W $$check";; \
esac
. endfor
.endif
_list-port-libs:
.if defined(_PORT_LIBS_CACHE) && defined(_DEPENDS_CACHE) && defined(_DEPENDS_FILE)
@if ! fgrep -q -e "r|${FULLPKGPATH}|" -e "a|${FULLPKGPATH}" $${_DEPENDS_FILE}; then \
${MAKE} run-dir-depends >>${_DEPENDS_CACHE}; \
fi
@perl ${PORTSDIR}/infrastructure/build/extract-dependencies ${FULLPKGPATH} <${_DEPENDS_CACHE}|while read dir; do \
fulldir=$$dir; \
if test -f ${_PORT_LIBS_CACHE}/$$fulldir; then \
cat ${_PORT_LIBS_CACHE}/$$fulldir; \
else \
mkdir -p `dirname ${_PORT_LIBS_CACHE}/$$fulldir`; \
unset FLAVOR SUBPACKAGE || true; \
${_flavor_fragment}; \
eval $$toset _NODEPS=Yes ${MAKE} print-plist-contents | \
grep -e '^@lib ' -e '^@file .*/lib/.*\.a$$'| \
sed -e 's:@lib ::' -e 's:@file ::' |tee ${_PORT_LIBS_CACHE}/$$fulldir; \
fi; \
done
.else
@${MAKE} run-dir-depends|tsort -r|sed -e '$$d'|while read dir; do \
unset FLAVOR SUBPACKAGE || true; \
${_flavor_fragment}; \
eval $$toset _NODEPS=Yes ${MAKE} print-plist-contents ; \
done | grep -e '^@lib ' -e '^@file .*/lib/.*\.a$$'| \
sed -e 's:@lib ::' -e 's:@file ::'
.endif
@echo /usr/lib/lib* ${X11BASE}/lib/lib*
_print-package-signature-helper:
_print-package-signature-run:
.for _i in ${RUN_DEPENDS}
@unset FLAVOR SUBPACKAGE || true; \
echo '${_i}' |{ \
IFS=:; read dep pkg pkgpath target; \
dir=$$pkgpath; ${_flavor_fragment}; \
default=`eval $$toset ${MAKE} _print-packagename`; \
case "X$$pkg" in X) pkg=`echo $$default|sed -e 's,-[0-9].*,-*,'`;; esac; \
echo "$$default"; \
}
.endfor
.if ${NO_SHARED_LIBS:L} != "yes"
. for _i in ${LIB_DEPENDS}
_print-package-signature-lib:
@echo $$LIST_LIBS| LOCALBASE=${LOCALBASE} X11BASE=${X11BASE} perl ${PORTSDIR}/infrastructure/build/resolve-lib ${_DEPLIBS}
.for _i in ${LIB_DEPENDS}
@unset FLAVOR SUBPACKAGE || true; \
echo '${_i}'|{ \
echo '${_i}' |{ \
IFS=:; read dep pkg pkgpath target; \
dir=$$pkgpath; ${_flavor_fragment}; \
libspecs='';comma=''; \
default=`eval $$toset ${MAKE} _print-packagename`; \
case "X$$pkg" in X) pkg=`echo $$default|sed -e 's,-[0-9].*,-*,'`;; esac; \
listlibs='echo $$LIST_LIBS|tr "\040" "\012"|fgrep "$$shdir" |sed -e "s:$$shdir/::"'; \
IFS=,; for d in $$dep; do \
${_libresolve_fragment}; \
case "$$check" in \
*.a) continue;; \
Missing\ library|Error:*) \
echo 1>&2 "Can't resolve libspec $$d"; \
exit 1;; \
*) \
echo "$$shprefix$$check";; \
esac; \
done; \
echo "$$default"; \
}
. endfor
. for _i in ${WANTLIB}
@d=${_i}; listlibs='echo $$LIST_LIBS|tr "\040" "\012"|fgrep "$$shdir" |sed -e "s:$$shdir/::"'; \
${_syslibresolve_fragment}; \
case "$$check" in \
*.a) continue;; \
Missing\ library|Error:*) \
echo 1>&2 "Can't resolve libspec $$d"; \
exit 1;; \
*) \
echo "$$shprefix$$check";; \
esac
. endfor
.endif
.endfor
# recursively build a list of dirs for package running, ready for tsort
_recurse-run-dir-depends:
@ -2723,4 +2727,4 @@ uninstall deinstall:
_internal-newlib-depends-check \
_internal-manpages-check _internal-plist _internal-update-plist \
_internal-update update print-plist print-plist-contents \
_list-port-libs _print-package-signature-helper
_list-port-libs _print-package-signature-lib _print-package-signature-run