From 43ed8edb9c74730b757d6b6f90ca69d92ef1881a Mon Sep 17 00:00:00 2001 From: espie Date: Sun, 9 Oct 2005 12:01:22 +0000 Subject: [PATCH] tweak the way library dependencies are resolved to speed them up. Now, resolve-lib can take a big list of libraries with full paths, and it can solve a big list of spec at once. Basically, we move most of the parsing of spec paths into resolve-lib. Since print-package-signature does build a full list of libs, let's solve it all at once, instead of invoking a costly perl script repeatedly. Add some caching possibilities for out-of-date. Specifically: - store libraries for each package under the directory _PORT_LIBS_CACHE - use the dependency cache _DEPENDS_FILE to avoid recreating dependency chains, add a new file _DEPENDS_CACHE that will accumulate all dependencies, and extract these with a simple script extract-dependencies. Use echo to build libraries lists instead of ls, that's a bit simpler... Some more clean-up will happen: it's probably simpler to parse libspecs at once, extract the libraries needed and go fetch the corresponding libraries just once. --- infrastructure/build/extract-dependencies | 56 +++++ infrastructure/build/resolve-lib | 254 +++++++++++++--------- infrastructure/mk/bsd.port.mk | 106 ++++----- 3 files changed, 263 insertions(+), 153 deletions(-) create mode 100644 infrastructure/build/extract-dependencies diff --git a/infrastructure/build/extract-dependencies b/infrastructure/build/extract-dependencies new file mode 100644 index 00000000000..e37526d9654 --- /dev/null +++ b/infrastructure/build/extract-dependencies @@ -0,0 +1,56 @@ +#! /usr/bin/perl + +# $OpenBSD: extract-dependencies,v 1.1 2005/10/09 12:01:22 espie Exp $ +# +# Copyright (c) 2005 Marc Espie +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# Usage: extract-dependencies < 'tsort-pairs' seed +# extracts all the dependencies for seed from a list that contains more +# than that. + +use strict; +use warnings; + + +# build dependency graph +my $dep = {}; +while() { + chomp; + my($a, $b) = split(/\s+/, $_); + $dep->{$a} = {} unless defined $dep->{$a}; + $dep->{$a}->{$b} = 1; +} + +# get the starting point +my $pkgpath = shift; + +my @todo = (); +my $done = {}; + +# walk the graph repeatedly starting from it +push(@todo, $pkgpath); + +while (my $x = shift @todo) { + next if $done->{$x}; + $done->{$x} = 1; + next unless defined $dep->{$x}; + push(@todo, keys %{$dep->{$x}}); +} + +# display all nodes, except for the seed +for my $d (keys %$done) { + next if $d eq $pkgpath; + print "$d\n"; +} diff --git a/infrastructure/build/resolve-lib b/infrastructure/build/resolve-lib index b333929141b..406657e5ab7 100644 --- a/infrastructure/build/resolve-lib +++ b/infrastructure/build/resolve-lib @@ -1,8 +1,8 @@ -#!/usr/bin/perl -# $OpenBSD: resolve-lib,v 1.4 2002/11/28 19:20:37 espie Exp $ +#! /usr/bin/perl +# $OpenBSD: resolve-lib,v 1.5 2005/10/09 12:01:22 espie Exp $ +# +# Copyright (c) 2001, 2005 Marc Espie # -# Copyright (c) 2001 -# Marc Espie. All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: @@ -25,108 +25,158 @@ # SUCH DAMAGE. use strict; +use warnings; + +my $LOCALBASE = $ENV{'LOCALBASE'}; +my $X11BASE = $ENV{'X11BASE'}; + +sub parse_spec_path +{ + my $req = shift; + my $spec = $req->{spec}; + if ($spec =~ m|^/.*/|) { + $req->{specdir}=$&; + $req->{prefix} = ''; + $req->{libname} = $'; + } elsif ($spec =~ m|^.*/|) { + $req->{specdir} = "$LOCALBASE/$&"; + $req->{prefix} =$&; + $req->{libname} = $'; + } else { + $req->{libname} = $spec; + $req->{prefix} = ''; + } +} + +sub parse_spec_versions +{ + my $spec = shift; + my $noshared = shift; + + my $req = {noshared => $noshared, strict => 0, sharedonly => 0 }; + $spec =~ s/\.$//; + if ($spec =~ m/\.a$/) { + $spec = $`; + $req->{noshared} = 1; + } else { + if ($spec =~ m/\.(\d+)\.(\d+)$/) { + $req->{major} = $1; + $req->{minor} = $2; + $spec = $`; + } elsif ($spec =~ m/\.\=(\d+)\.(\d+)$/) { + $req->{major} = $1; + $req->{minor} = $2; + $spec = $`; + $req->{strict} = 1; + } elsif ($spec =~ m/\.(\d+)$/) { + $req->{major} = $1; + $req->{minor} = 0; + $spec = $`; + } elsif ($spec =~ m/\.\=(\d+)$/) { + $req->{major} = $1; + $req->{minor} = 0; + $req->{strict} = 1; + $spec = $`; + } else { + $req->{major} = 0; + $req->{minor} = 0; + } + if ($spec =~ m/\.so$/) { + $spec = $`; + $req->{sharedonly} = 1; + } + } + $req->{spec} = $spec; + return $req; +} + +sub parse_spec +{ + my ($spec, $noshared) = @_; + my $req = parse_spec_versions($spec); + parse_spec_path($req); + return $req; +} + +sub solve_spec +{ + my $spec = shift; + my $noshared = shift; + my $req = parse_spec($spec, $noshared); + + my $found_shared; + my $found_goodshared; + my $bestmajor=-1; my $bestminor=-1; + my $found_unshared; + my $libname = $req->{libname}; + + for my $file (@_) { + next unless $file =~ m|(.*/)|; + if (defined $req->{specdir}) { + next unless $1 eq $req->{specdir}; + } else { + next unless $1 eq '/usr/lib/' or + $1 eq "$LOCALBASE/lib/" or + $1 eq "$X11BASE/lib/"; + } + local $_=$'; + if (!$req->{noshared} && + m/^lib\Q$libname\E\.so\.(\d+)\.(\d+)$/) { + $found_shared = 1; + my $major = $1; + my $minor = $2; + if ($req->{strict}) { + if ($major > $req->{major}) { + print "Error: strict library too high\n"; + last; + } + if ($major < $req->{major}) { + next; + } + if ($minor < $req->{minor}) { + next; + } + $found_goodshared = 1; + $bestmajor = $major; + if ($minor >= $bestminor) { + $bestminor = $minor; + } + } else { + if ($major < $req->{major} || + ($major == $req->{major} && $minor < $req->{minor})) { + next; + } + $found_goodshared = 1; + if ($major > $bestmajor || ($major == $bestmajor && + $minor > $bestminor)) { + $bestmajor = $major; + $bestminor = $minor; + } + } + } elsif (!$req->{sharedonly} && m/^lib\Q$libname\E\.a$/) { + $found_unshared = 1; + } + } + if ($found_goodshared) { + print $req->{prefix},"$libname.$bestmajor.$bestminor\n"; + } elsif ($found_shared) { + print STDERR "Error: bad shared library\n"; + } elsif ($found_unshared) { + print $req->{prefix},"$libname.a\n"; + } else { + print STDERR "Missing library for $spec\n"; + } +} + my $noshared; -my $sharedonly; -my $strict; -my $reqmajor; -my $reqminor; - # Grab arguments -$_=shift; -if ($_ eq '-noshared') { +if ($ARGV[0] eq '-noshared') { $noshared = 1; - $_ = shift; -} - -s/\.$//; -# Parse spec -if (m/\.a$/) { - $_ = $`; - $noshared = 1; -} else { - if (m/\.(\d+)\.(\d+)$/) { - $reqmajor = $1; - $reqminor = $2; - $_ = $`; - } elsif (m/\.\=(\d+)\.(\d+)$/) { - $reqmajor = $1; - $reqminor = $2; - $_ = $`; - $strict = 1; - } elsif (m/\.(\d+)$/) { - $reqmajor = $1; - $reqminor = 0; - $_ = $`; - } elsif (m/\.\=(\d+)$/) { - $reqmajor = $1; - $reqminor = 0; - $strict = 1; - $_ = $`; - } - if (m/\.so$/) { - $_ = $`; - $sharedonly = 1; - } -} - -{ -my $libname=$_; - -my $bestmajor=-1; -my $bestminor=-1; - -my $found_shared; -my $found_goodshared; -my $found_unshared; - -while(<>) { - chomp; - if (!$noshared && m/^lib\Q$libname\E\.so\.(\d+)\.(\d+)$/) { - $found_shared = 1; - my $major = $1; - my $minor = $2; - if ($strict) { - if ($major > $reqmajor) { - print "Error: strict library too high\n"; - exit(0); - } - if ($major < $reqmajor) { - next; - } - if ($minor < $reqminor) { - next; - } - $found_goodshared = 1; - $bestmajor = $major; - if ($minor >= $bestminor) { - $bestminor = $minor; - } - } else { - if ($major < $reqmajor || ($major == $reqmajor && - $minor < $reqminor)) { - next; - } - $found_goodshared = 1; - if ($major > $bestmajor || ($major == $bestmajor && - $minor > $bestminor)) { - $bestmajor = $major; - $bestminor = $minor; - } - } - } elsif (!$sharedonly && m/^lib\Q$libname\E\.a$/) { - $found_unshared = 1; - } -} - -if ($found_goodshared) { - print "$libname.$bestmajor.$bestminor\n"; -} elsif ($found_shared) { - print "Error: bad shared library\n"; -} elsif ($found_unshared) { - print "$libname.a\n"; -} else { - print "Missing library\n"; + shift; } +my @available = split(/\s+/, ); +for my $spec (@ARGV) { + solve_spec($spec, $noshared, @available); } diff --git a/infrastructure/mk/bsd.port.mk b/infrastructure/mk/bsd.port.mk index 782a67aadee..85276100ca3 100644 --- a/infrastructure/mk/bsd.port.mk +++ b/infrastructure/mk/bsd.port.mk @@ -1,6 +1,6 @@ #-*- mode: Makefile; tab-width: 4; -*- # ex:ts=4 sw=4 filetype=make: -# $OpenBSD: bsd.port.mk,v 1.713 2005/10/07 21:08:16 espie Exp $ +# $OpenBSD: bsd.port.mk,v 1.714 2005/10/09 12:01:22 espie Exp $ # $FreeBSD: bsd.port.mk,v 1.264 1996/12/25 02:27:44 imp Exp $ # $NetBSD: bsd.port.mk,v 1.62 1998/04/09 12:47:02 hubertf Exp $ # @@ -1044,22 +1044,20 @@ _noshared= _libresolve_fragment = \ case "$$d" in \ - */*) shprefix="$${d%/*}/"; shdir="${LOCALBASE}/$${d%/*}"; \ - d=$${d\#\#*/};; \ - *) shprefix="" shdir="${LOCALBASE}/lib";; \ + */*) shdir="${LOCALBASE}/$${d%/*}";; \ + *) shdir="${LOCALBASE}/lib";; \ esac; \ - check=`eval $$listlibs| perl \ + check=`eval $$listlibs 2>/dev/null| LOCALBASE=${LOCALBASE} X11BASE=${X11BASE} perl \ ${PORTSDIR}/infrastructure/build/resolve-lib ${_noshared} $$d` \ || true _syslibresolve_fragment = \ case "$$d" in \ - /*) shdir="$${d%/*}/"; shprefix=""; d=$${d\#\#*/};; \ - */*) shprefix="$${d%/*}/"; shdir="${DEPBASE}/$${d%/*}"; \ - d=$${d\#\#*/};; \ - *) shprefix="" shdir="${DEPBASE}/lib"; listlibs="$$listlibs; ls /usr/lib /usr/X11R6/lib 2>/dev/null";; \ + /*) shdir="$${d%/*}/";; \ + */*) shdir="${DEPBASE}/$${d%/*}";; \ + *) shdir="${DEPBASE}/lib"; listlibs="$$listlibs /usr/lib/lib* ${X11BASE}/lib/lib*";; \ esac; \ - check=`eval $$listlibs| perl \ + check=`eval $$listlibs 2>/dev/null| LOCALBASE=${LOCALBASE} X11BASE=${X11BASE} perl \ ${PORTSDIR}/infrastructure/build/resolve-lib ${_noshared} $$d` \ || true @@ -1070,7 +1068,7 @@ _lib_depends_fragment = \ found2=false; \ what="$$dep ($$pkg)"; \ IFS=,; bad=false; for d in $$dep; do \ - listlibs='ls $$shdir 2>/dev/null'; \ + listlibs='echo $$shdir/lib*'; \ ${_libresolve_fragment}; \ case "$$check" in \ Missing\ library) bad=true; msg="$$msg $$d missing...";; \ @@ -1172,9 +1170,15 @@ _FMN+= ${PKGPATH}/${FULLPKGNAME${_S}} .if defined(LIB_DEPENDS) && ${NO_SHARED_LIBS:L} != "yes" _ALWAYS_DEP2 = ${LIB_DEPENDS:C/^[^:]*:([^:]*:[^:]*).*$/\1/} _ALWAYS_DEP= ${_ALWAYS_DEP2:C/[^:]*://} +_DEPLIBS=${LIB_DEPENDS:C/:.*//:S/,/ /g} .else _ALWAYS_DEP2= _ALWAYS_DEP= +_DEPLIBS= +.endif + +.if defined(WANTLIB) +_DEPLIBS+=${WANTLIB} .endif .if defined(RUN_DEPENDS) @@ -2401,9 +2405,15 @@ ${_i:L}-depends-list: print-package-signature: @echo -n ${FULLPKGNAME${SUBPACKAGE}} - @cd ${.CURDIR} && PACKAGING='${SUBPACKAGE}' LIST_LIBS=`${MAKE} _list-port-libs` ${MAKE} _print-package-signature-helper | \ +.if !empty(_DEPLIBS) + @cd ${.CURDIR} && PACKAGING='${SUBPACKAGE}' LIST_LIBS=`${MAKE} _list-port-libs` ${MAKE} _print-package-signature-lib _print-package-signature-run| \ sort -u| \ while read i; do echo -n ",$$i"; done +.else + @cd ${.CURDIR} && PACKAGING='${SUBPACKAGE}' ${MAKE} _print-package-signature-run | \ + sort -u| \ + while read i; do echo -n ",$$i"; done +.endif @echo _print-package-args: @@ -2427,10 +2437,10 @@ _print-package-args: default=`eval $$toset ${MAKE} _print-packagename`; \ case "X$$pkg" in X) pkg=`echo $$default|sed -e 's,-[0-9].*,-*,'`;; esac; \ if pkg_info -q -e $$pkg; then \ - listlibs='ls ${DEPDIR}$$shdir 2>/dev/null'; \ + listlibs='echo ${DEPDIR}$$shdir/lib*'; \ else \ eval 1>&2 $$toset ${MAKE} ${PKGREPOSITORY}/$$default.tgz; \ - listlibs='pkg_info -L -K ${PKGREPOSITORY}/$$default.tgz|grep "^@lib $$shdir" |sed -e "s:@lib $$shdir/::"'; \ + listlibs='pkg_info -L -K ${PKGREPOSITORY}/$$default.tgz|grep -e "^@lib $$shdir/lib" -e "^$$shdir/lib.*.a"|sed -e "s:@lib "'; \ fi; \ IFS=,; for d in $$dep; do \ ${_libresolve_fragment}; \ @@ -2440,14 +2450,14 @@ _print-package-args: echo 1>&2 "Can't resolve libspec $$d"; \ exit 1;; \ *) \ - echo "-W $$shprefix$$check";; \ + echo "-W $$check";; \ esac; \ done; \ echo "-P $$pkgpath:$$pkg:$$default"; \ } . endfor . for _i in ${WANTLIB} - @d=${_i}; listlibs='ls $$shdir 2>/dev/null'; \ + @d=${_i}; listlibs='echo $$shdir/lib*'; \ ${_syslibresolve_fragment}; \ case "$$check" in \ *.a) ;; \ @@ -2455,67 +2465,61 @@ _print-package-args: echo 1>&2 "Can't resolve libspec $$d"; \ exit 1;; \ *) \ - echo "-W $$shprefix$$check";; \ + echo "-W $$check";; \ esac . endfor .endif _list-port-libs: +.if defined(_PORT_LIBS_CACHE) && defined(_DEPENDS_CACHE) && defined(_DEPENDS_FILE) + @if ! fgrep -q -e "r|${FULLPKGPATH}|" -e "a|${FULLPKGPATH}" $${_DEPENDS_FILE}; then \ + ${MAKE} run-dir-depends >>${_DEPENDS_CACHE}; \ + fi + @perl ${PORTSDIR}/infrastructure/build/extract-dependencies ${FULLPKGPATH} <${_DEPENDS_CACHE}|while read dir; do \ + fulldir=$$dir; \ + if test -f ${_PORT_LIBS_CACHE}/$$fulldir; then \ + cat ${_PORT_LIBS_CACHE}/$$fulldir; \ + else \ + mkdir -p `dirname ${_PORT_LIBS_CACHE}/$$fulldir`; \ + unset FLAVOR SUBPACKAGE || true; \ + ${_flavor_fragment}; \ + eval $$toset _NODEPS=Yes ${MAKE} print-plist-contents | \ + grep -e '^@lib ' -e '^@file .*/lib/.*\.a$$'| \ + sed -e 's:@lib ::' -e 's:@file ::' |tee ${_PORT_LIBS_CACHE}/$$fulldir; \ + fi; \ + done +.else @${MAKE} run-dir-depends|tsort -r|sed -e '$$d'|while read dir; do \ unset FLAVOR SUBPACKAGE || true; \ ${_flavor_fragment}; \ eval $$toset _NODEPS=Yes ${MAKE} print-plist-contents ; \ done | grep -e '^@lib ' -e '^@file .*/lib/.*\.a$$'| \ sed -e 's:@lib ::' -e 's:@file ::' +.endif + @echo /usr/lib/lib* ${X11BASE}/lib/lib* -_print-package-signature-helper: +_print-package-signature-run: .for _i in ${RUN_DEPENDS} @unset FLAVOR SUBPACKAGE || true; \ echo '${_i}' |{ \ IFS=:; read dep pkg pkgpath target; \ dir=$$pkgpath; ${_flavor_fragment}; \ default=`eval $$toset ${MAKE} _print-packagename`; \ - case "X$$pkg" in X) pkg=`echo $$default|sed -e 's,-[0-9].*,-*,'`;; esac; \ echo "$$default"; \ } .endfor -.if ${NO_SHARED_LIBS:L} != "yes" -. for _i in ${LIB_DEPENDS} + +_print-package-signature-lib: + @echo $$LIST_LIBS| LOCALBASE=${LOCALBASE} X11BASE=${X11BASE} perl ${PORTSDIR}/infrastructure/build/resolve-lib ${_DEPLIBS} +.for _i in ${LIB_DEPENDS} @unset FLAVOR SUBPACKAGE || true; \ - echo '${_i}'|{ \ + echo '${_i}' |{ \ IFS=:; read dep pkg pkgpath target; \ dir=$$pkgpath; ${_flavor_fragment}; \ - libspecs='';comma=''; \ default=`eval $$toset ${MAKE} _print-packagename`; \ - case "X$$pkg" in X) pkg=`echo $$default|sed -e 's,-[0-9].*,-*,'`;; esac; \ - listlibs='echo $$LIST_LIBS|tr "\040" "\012"|fgrep "$$shdir" |sed -e "s:$$shdir/::"'; \ - IFS=,; for d in $$dep; do \ - ${_libresolve_fragment}; \ - case "$$check" in \ - *.a) continue;; \ - Missing\ library|Error:*) \ - echo 1>&2 "Can't resolve libspec $$d"; \ - exit 1;; \ - *) \ - echo "$$shprefix$$check";; \ - esac; \ - done; \ echo "$$default"; \ } -. endfor -. for _i in ${WANTLIB} - @d=${_i}; listlibs='echo $$LIST_LIBS|tr "\040" "\012"|fgrep "$$shdir" |sed -e "s:$$shdir/::"'; \ - ${_syslibresolve_fragment}; \ - case "$$check" in \ - *.a) continue;; \ - Missing\ library|Error:*) \ - echo 1>&2 "Can't resolve libspec $$d"; \ - exit 1;; \ - *) \ - echo "$$shprefix$$check";; \ - esac -. endfor -.endif +.endfor # recursively build a list of dirs for package running, ready for tsort _recurse-run-dir-depends: @@ -2723,4 +2727,4 @@ uninstall deinstall: _internal-newlib-depends-check \ _internal-manpages-check _internal-plist _internal-update-plist \ _internal-update update print-plist print-plist-contents \ - _list-port-libs _print-package-signature-helper + _list-port-libs _print-package-signature-lib _print-package-signature-run