diff --git a/man8/pkgmeek.8 b/man8/pkgmeek.8 index 72726e4..0a1e8b9 100644 --- a/man8/pkgmeek.8 +++ b/man8/pkgmeek.8 @@ -49,12 +49,6 @@ Update footprint and treat last build as successful. .B "\-us, \-\-update\-signature" Update port signature and sha256sums. .TP -.B "\-um, \-\-update\-md5sum" -Update md5sum using the current source files. -.TP -.B "\-im, \-\-ignore\-md5sum" -Build package without checking md5sum first. -.TP .B "\-cs, \-\-check\-signature" Check the validity of Pkgfile, footprint, and sources using the signature and sha256sums. .TP @@ -94,6 +88,17 @@ Print version and exit. .TP .B "\-h, \-\-help" Print help and exit. +.SH DEPRECATED OPTIONS +.TP +.B "\-cm, \-\-check-md5sum, \-im, \-\-ignore-md5sum, \-um, \-\-update-md5sum" +The code to generate a listing of md5 hashes for the source files has been removed +from \fBpkgmeek\fP. Verifying the integrity of source files should be handled +by \fBsignify(1)\fP instead. If you are maintaining a personal port collection and +have not saved the signify public key in /etc/ports (corresponding private key in +~/.ssh or /etc/ports), then other users of your port collection will have no way +to ensure the integrity of the downloaded source files. Read the CRUX wiki to learn +how to configure a signify public key for your port collection. + .SH FILES .TP .B "Pkgfile" @@ -103,9 +108,6 @@ Package build description. Package footprint (a listing of all the files that would be unpacked from the compressed archive, complete with owner/group/permissions and full paths) .TP -.B ".md5sum" -MD5 checksum of each file listed in the \fIsource\fP array of the Pkgfile. -.TP .B ".signature" SHA256 checksum of footprint, Pkgfile, and each source file, and a signify checksum. .TP @@ -132,7 +134,7 @@ An error occured during the download of source files. An error occured during unpacking of source files. .TP .B 6 -A mismatch in the footprint or the md5sum was detected. +A footprint mismatch was detected. .TP .B 7 An error occured while running the build function. @@ -145,15 +147,15 @@ An error occured while verifying the signature. .SH PROGRAM DESIGN -\fBpkgmeek\fP aims to provide the same feature set that users of \fBpkgmk\fP have -come to rely upon, but without the sprawling code base that discourages would-be -contributors from stepping in to help. Its core function of building a package is -based on \fBupkgmk\fP, a 100-line bash script written by Fun. Upon this foundation, -new code in the same pithy style was added, but not haphazardly. A deliberately linear -sequence was imposed, letting new users discern a narrative arc to the process of building -a binary package, and each exit ramp along that story line is clearly marked. By moving the -linear sequence of main() to the top of the file, \fBpkgmeek\fP provides new users an overview of -the entire process, so they can quickly identify where a new feature is best inserted. +\fBpkgmeek\fP aims to provide the same feature set as the original \fBpkgmk\fP by +Per Liden, but in a more compact code base to allow for extensibility and easier auditing. +Its core function of building a package is based on \fBupkgmk\fP, a 100-line bash script +written by Fun. Upon this foundation, new code in the same pithy style was added. +Reviewers of the \fBpkgmeek\fP script should be able to discern a linear narrative leading +to the desired binary package, with each exit ramp along that story line clearly marked. +By moving the linear sequence of main() to the top of the file, \fBpkgmeek\fP provides +new users an overview of the entire process, so they can quickly identify where a new +feature is best inserted. When the first version of \fBpkgmk\fP was written, it had a modest goal: read the Pkgfile and build a compressed archive. As Fun demonstrated, this goal can be achieved in 100 lines @@ -180,11 +182,12 @@ of writing, patch submissions on our bug tracker are nowhere near the volume the The patch submissions that do arrive are not dealt with in a timely manner, which is more easily explained by a reduction in the number of core developers than by stipulating that CRUX package tools have reached full maturity and cannot gain much benefit from new patches. The next most -probable explanations for the slow response to bug reports are: 1, interest in CRUX is waning (and -could stand to be reinvigorated), or 2, the would-be contributors start paging through a legacy code -base and lose confidence in their ability to help out. If either of these latter explanations has -any merit, \fBpkgmeek\fP is offered as a partial remedy to the problem that "we don't have people -that actually sign up to care about our core tools" (FS#1410). +probable explanations for the slow response to bug reports are: 1, an accelerated pace of upstream +development keeps core maintainers too busy bumping the version numbers in the ports tree, so +patches to system tools cannot be reviewed as quickly as in 2017, or 2, would-be contributors start +paging through a legacy code base and lose confidence in their ability to help out. If either of +these latter explanations has any merit, \fBpkgmeek\fP is offered as a partial remedy to the problem +that "we don't have people that actually sign up to care about our core tools" (FS#1410). .SH SEE ALSO pkgmk.conf(5), Pkgfile(5), pkgadd(8), pkgrm(8), pkginfo(8), rejmerge(8), signify(1), curl(1), wget(1) diff --git a/scripts/pkgmeek b/scripts/pkgmeek index 07afad0..55d0a3a 100755 --- a/scripts/pkgmeek +++ b/scripts/pkgmeek @@ -46,12 +46,19 @@ fi package="${name}#${version}-${release}.pkg.tar.${PKGMK_COMPRESSION_MODE}" declare -a _local_ for (( s=0; s<${#source[@]}; s++ )); do - [[ ${source[$s]} =~ ^(http|https|ssh|ftp|git)://.*/(.+)$ ]] && \ - _local_[$s]="${BASH_REMATCH[2]%.git*}" || _local_[$s]="${source[$s]}" + case "${source[$s]}" in + http://*|https://*|ftp://*|__git__*) + _local_[$s]="${source[$s]##*/}" # strip the leading path + # and for git sources, extract the project name: + [[ "${source[$s]}" =~ ^__git__ ]] && { _local_[$s]="${_local_[$s]%\#*}"; + _local_[$s]="${_local_[$s]%.git}"; } ;; + *) + _local_[$s]="${source[$s]}" ;; + esac [ -z "${renames[$s]}" ] || [ "${renames[$s]}" = "SKIP" ] || \ _local_[$s]="${renames[$s]}" done -# Example: source = ( https://gitlab.com/demo-user/cool-project.git#0.4.9 +# Example: source = ( __git__https://gitlab.com/demo-user/cool-project.git#0.4.9 # https://dev.big-corp.com/src/needed-library.tgz # random.patch ) # _local_ = ( cool-project @@ -78,54 +85,46 @@ done ; } # Can stop here if the user asked for '-do', but honor any requests for '-um', '-uf', # '-us', '-eo', or '-utd' by proceeding to those steps -[ "$PKGMK_DOWNLOAD_ONLY" = "no" ] || [ "$PKGMK_UPDATE_MD5" = "yes" ] || \ - [ "$PKGMK_UPDATE_FOOTPRINT" = "yes" ] || [ "$PKGMK_UPDATE_SIG" = "yes" ] || \ - [ "$PKGMK_EXTRACT_ONLY" = "yes" ] || [ "$PKGMK_MTIME_ONLY" = "yes" ] || exit 0 +[ "$PKGMK_DOWNLOAD_ONLY" = "no" ] || [ "$PKGMK_UPDATE_FOOTPRINT" = "yes" ] \ + || [ "$PKGMK_UPDATE_SIG" = "yes" ] || [ "$PKGMK_EXTRACT_ONLY" = "yes" ] \ + || [ "$PKGMK_MTIME_ONLY" = "yes" ] || exit 0 # If the user only asked for '-utd', perform the check using the sources that do exist. check_pkg_mtime; pkg_utd=$? [ "$PKGMK_MTIME_ONLY" = "no" ] || [ "$PKGMK_CHECK_SIG" = "yes" ] || \ - [ "$PKGMK_FORCE" = "yes" ] || exit $pkg_utd + [ "$PKGMK_FORCE" = "yes" ] || { cleanup_work; exit $pkg_utd; } # Take into account all the actions that can be done with a previously built package, # or with a full set of sources -[ "$pkg_utd" = 0 ] || [ "$PKGMK_FORCE" = "yes" ] || [ "$PKGMK_UPDATE_FOOTPRINT" = "yes" ] \ - || [ "$PKGMK_UPDATE_SIG" = "yes" ] || [ "$PKGMK_CHECK_SIG" = "yes" ] \ - || [ "$PKGMK_UPDATE_MD5" = "yes" ] || [ "$PKGMK_EXTRACT_ONLY" = "yes" ] \ - || { info "$package is up to date, use '-f' to force a rebuild."; exit 0; } +[ "$pkg_utd" = 0 ] || [ "$PKGMK_FORCE" = "yes" ] || [ "$PKGMK_EXTRACT_ONLY" = "yes" ] \ + || [[ "$PKGMK_UPDATE_FOOTPRINT $PKGMK_CHECK_SIG $PKGMK_UPDATE_SIG" =~ yes ]] \ + || { info "$package is up to date, use '-f' to force a rebuild."; cleanup_work; exit 0; } # Silence the progress report if the user never intended to proceed with unpacking [ "$pkg_utd" = 1 ] || [[ "$PKGMK_CHECK_SIG $PKGMK_UPDATE_SIG" =~ yes ]] || \ - [[ "$PKGMK_UPDATE_FOOTPRINT $PKGMK_UPDATE_MD5" =~ yes ]] || \ - [[ "$PKGMK_IGNORE_SIG" = "yes" ]] || echo "Checking signatures before unpacking..." + [[ "$PKGMK_IGNORE_SIG $PKGMK_UPDATE_FOOTPRINT" =~ yes ]] \ + || echo "Checking signatures before unpacking..." # The option -uf is meant to be used AFTER a previous invocation of pkgmeek has -# alerted the user to a footprint mismatch. The options -us|-um are likewise meant -# to update the signature|md5sum, which only requires the sources and the footprint. +# alerted the user to a footprint mismatch. Updating signatures (option -us) likewise +# needs only the sources and the footprint. if [ "$PKGMK_UPDATE_FOOTPRINT" = "yes" ]; then [ -f "$pkg_dir$package" ] || \ { error "unable to update footprint. File '$package' not found."; - exit "$E_MANIFEST"; } + exit "$E_FOOTPRINT"; } [ "$pkg_utd" = 1 ] || [ "$PKGMK_FORCE" = "yes" ] || \ { error "outdated package. Use '-f' to force the footprint update."; - exit "$E_MANIFEST"; } - cat_manifest footprint > "$PKGMK_ROOT/.footprint" && info "footprint created." \ + exit "$E_FOOTPRINT"; } + cat_footprint > "$PKGMK_ROOT/.footprint" && info "footprint created." \ || { error "Failed to write the footprint."; exit "$E_DIRPERM"; } fi -[ "$PKGMK_UPDATE_SIG" = "no" ] || \ - { cat_signature > "$PKGMK_ROOT/.signature" && - info "Signature successfully created."; } || \ - { cat_manifest md5sum > "$PKGMK_ROOT"/.md5sum && - warning "Signature creation failed, falling back to md5sums."; } \ - || { info "Could not create signatures or md5sums."; exit "$E_DIRPERM"; } -[ "$PKGMK_UPDATE_MD5" = "no" ] || \ - { cat_manifest md5sum > "$PKGMK_ROOT"/.md5sum && info "md5sum updated."; } || \ - { info "Could not create requested md5sums."; exit "$E_DIRPERM"; } +if [ "$PKGMK_UPDATE_SIG" = "yes" ]; then + cat_signature > "$PKGMK_ROOT/.signature" && info "Signature successfully created." \ + || { info "Could not create signature."; exit "$E_DIRPERM"; } +fi # Exit after fulfilling any *explicit* requests for (signed) manifests -# (decision regarding the work directory is retained from the first run of pkgmeek) -[ "$PKGMK_UPDATE_FOOTPRINT" = "yes" ] || [ "$PKGMK_UPDATE_SIG" = "yes" ] || \ - [ "$PKGMK_UPDATE_MD5" = "yes" ] && exit 0 +[[ "$PKGMK_UPDATE_FOOTPRINT $PKGMK_UPDATE_SIG" =~ yes ]] && { cleanup_work; exit 0; } # All the sources should be here by now, let's verify that we can trust them. readonly cs_fail_msg="Use '--ignore-signature' to override, if you have determined integrity by other means." @@ -135,7 +134,7 @@ case $? in 1) error "Signature file missing or corrupted." ; echo "$cs_fail_msg" ; exit $E_SIGNATURE ;; 2) error "Failed to authenticate remote sources using signify." ; echo "$cs_fail_msg" ; exit $E_SIGNATURE ;; esac; } -[ "$PKGMK_CHECK_SIG" = "no" ] || exit 0 # no need to continue if the user only requested -cs +[ "$PKGMK_CHECK_SIG" = "no" ] || { cleanup_work; exit 0; } # no need to continue if the user only requested -cs if [ "$pkg_utd" = 0 ] || [ "$PKGMK_FORCE" = "yes" ]; then for (( u=0; u<${#_local_[@]}; u++ )) ; do @@ -149,17 +148,14 @@ if [ "$pkg_utd" = 0 ] || [ "$PKGMK_FORCE" = "yes" ]; then done [ $errUZ = 0 ] && info "Sources successfully unpacked." || \ - { error "Failed to unpack all sources."; exit "$E_UNPACK"; } + { error "Failed to unpack all sources."; exit "$E_UNPACK"; } [ "$PKGMK_EXTRACT_ONLY" = "no" ] || exit 0 # The actual build step! (use fakeroot when building daemon ports as an ordinary user, # otherwise the owner and group might not be correct) (SRC=$(pwd)/src; PKG=$(pwd)/pkg; cd src; set -e -x; build) - if [ $? = 0 ]; then - echo "Build successful. Moving on to compression." - else - error "Unsuccessful build!"; cleanup_work; exit "$E_BUILD" - fi + [ $? = 0 ] && echo "Build successful. Moving on to compression." \ + || { error "Unsuccessful build!"; cleanup_work; exit "$E_BUILD"; } [ -f "$PKGMK_ROOT/.nostrip" ] && ns_filter="| grep -v -f $PKGMK_ROOT/.nostrip" find pkg -type f $ns_filter | while read -r f; do @@ -185,22 +181,25 @@ if [ "$pkg_utd" = 0 ] || [ "$PKGMK_FORCE" = "yes" ]; then fi # Check the footprint of the built package, unless '-if' was given - [ "$PKGMK_IGNORE_FOOTPRINT" = "yes" ] || check_manifest footprint + [ "$PKGMK_IGNORE_FOOTPRINT" = "yes" ] || check_footprint - # Clean up the work directory - find . -maxdepth 1 -mindepth 1 -type l -delete; cleanup_work -fi # Continue from here if the extract and build were skipped +fi # Continue from here if extract and build were skipped -# Install if requested. For non-root builds, only sudo and doas are supported. -# Avoid falling back on su -c! It gobbles up the options intended for pkgadd. -[ $UID = 0 ] || PKGMK_SU="sudo"; -[ -z "$PKGMK_SU" ] || [ -x "$(command -v $PKGMK_SU)" ] || PKGMK_SU="/usr/bin/doas"; -[ -z "$PKGMK_SU" ] || [ -x "$(command -v $PKGMK_SU)" ] \ +# Clean up the work directory. +find . -maxdepth 1 -mindepth 1 -type l -delete; cleanup_work + +# Proceed to install/upgrade if requested. +if [ -n "$PKGMK_INSTALL_COMMAND" ]; then + [ $UID = 0 ] || PKGMK_SU="sudo"; + [ -z "$PKGMK_SU" ] || [ -x "$(command -v $PKGMK_SU)" ] || PKGMK_SU="/usr/bin/doas"; + [ -z "$PKGMK_SU" ] || [ -x "$(command -v $PKGMK_SU)" ] \ || { error "Cannot run pkgadd as a non-root user."; exit "$E_INSTALL"; } -[ -z "$PKGMK_INSTALL_COMMAND" ] || { $PKGMK_SU $PKGMK_INSTALL_COMMAND "$pkg_dir$package" \ - && info "$(basename $PKGMK_INSTALL_COMMAND) $package succeeded."; } || \ - { error "$(basename $PKGMK_INSTALL_COMMAND) $package failed."; exit "$E_INSTALL"; } + $PKGMK_SU $PKGMK_INSTALL_COMMAND "$pkg_dir$package" \ + && info "$(basename $PKGMK_INSTALL_COMMAND) $package succeeded." || \ + { error "$(basename $PKGMK_INSTALL_COMMAND) $package failed."; + exit "$E_INSTALL"; } +fi # Done! } @@ -210,7 +209,7 @@ E_PKGFILE=2 # invalid Pkgfile E_DIRPERM=3 # (source/build) directory missing or missing read/write permission E_DOWNLOAD=4 # error during download E_UNPACK=5 # error during unpacking of source file(s) -E_MANIFEST=6 # footprint or md5sum failure +E_FOOTPRINT=6 # footprint mismatch E_BUILD=7 # error while running 'build()' E_INSTALL=8 # error while installing the package via 'pkgadd' E_SIGNATURE=9 # error verifying the signature @@ -223,14 +222,11 @@ readonly PRTWASH_COMMAND="/usr/bin/prtwash" PKGMK_DOWNLOAD_PROG="/usr/bin/curl" PKGMK_GIT_COMMAND="/usr/bin/git" PKGMK_CONF="/etc/pkgmk.conf" - PKGMK_SOURCE_DIR="$PWD"; PKGMK_WORK_DIR="$PWD/work" PKGMK_PACKAGE_DIR="$PWD"; PKGMK_COMPRESSION_MODE="gz" - PKGMK_INSTALL_COMMAND=""; PKGMK_FORCE="no" PKGMK_CLEAN="no"; PKGMK_KEEP_WORK="no" -PKGMK_RECURSIVE="no"; PKGMK_UPDATE_MD5="no" -PKGMK_DOWNLOAD_ONLY="no"; PKGMK_IGNORE_MD5="no" +PKGMK_DOWNLOAD_ONLY="no"; PKGMK_RECURSIVE="no" PKGMK_EXTRACT_ONLY="no"; PKGMK_UPDATE_SIG="no" PKGMK_MTIME_ONLY="no"; PKGMK_IGNORE_SIG="no" PKGMK_UPDATE_FOOTPRINT="no"; PKGMK_REFRESH_SIG="no" @@ -243,20 +239,20 @@ parse_options() { case $1 in -r|--recursive) PKGMK_RECURSIVE="yes" ;; -c|--clean) [ -x "$PRTWASH_COMMAND" ] && PKGMK_CLEAN="yes" || \ - { error "option '-c' not supported ($PRTWASH_COMMAND not installed)"; + { error "option '$1' not supported ($PRTWASH_COMMAND not installed)"; exit 1; } ;; -uf|--update-footprint) PKGMK_UPDATE_FOOTPRINT="yes" ;; - -um|--update-md5sum) PKGMK_UPDATE_MD5="yes" ;; -us|--update-signature) PKGMK_UPDATE_SIG="yes" ;; -rs|--refresh-signature) PKGMK_REFRESH_SIG="yes" ;; -cs|--check-signature) PKGMK_CHECK_SIG="yes" ;; - -d|-cm|--download|--check-md5sum) ;; # deprecated flags, but at least they won't trigger an error + --check-md5sum|--ignore-md5sum) ;; # deprecated flags + -d|-cm|-im|--download) ;; # but don't throw an error for them + -um|--update-md5sum) warn "updating md5sums is deprecated, ignoring option '$1'." ;; -do|--download-only) PKGMK_DOWNLOAD_ONLY="yes" ;; -eo|--extract-only) PKGMK_EXTRACT_ONLY="yes" ;; -utd|--up-to-date) PKGMK_MTIME_ONLY="yes" ;; -if|--ignore-footprint) PKGMK_IGNORE_FOOTPRINT="yes" ;; -in|--ignore-new) PKGMK_IGNORE_NEW="yes" ;; - -im|--ignore-md5sum) PKGMK_IGNORE_MD5="yes" ;; -is|--ignore-signature) PKGMK_IGNORE_SIG="yes" ;; -ns|--no-strip) echo ".*" >> "$PKGMK_ROOT/.nostrip" ;; -f|--force) PKGMK_FORCE="yes" ;; @@ -297,12 +293,10 @@ print_help() { echo " -do, --download-only stop after downloading all the necessary source file(s)" echo " -eo, --extract-only stop after downloading and extracting source file(s)" echo " -utd, --up-to-date report whether the built package is up to date, then exit" - echo " -um, --update-md5sum update md5sum of the downloaded sources, do not build" echo " -us, --update-signature update signature of Pkgfile and sources, do not build" echo " -cs, --check-signature verify the signatures, do not build" echo " -if, --ignore-footprint build package without checking footprint" echo " -in, --ignore-new build package, ignoring new files in a footprint mismatch" - echo " -im, --ignore-md5sum build package without checking md5sum" echo " -is, --ignore-signature build package without checking signature" echo " -uf, --update-footprint update footprint using result from last build" echo " -rs, --refresh-signature create new signature and keep existing sha256 checksums" @@ -352,7 +346,7 @@ check_pkg_mtime() { # can be called even if some sources are missing [ "$pkgdir$package" -nt $(realpath "${_local_[$li]}") ] || utd=0 li=$(( li+1 )) done - [ ! -e Pkgfile ] || [ "$pkg_dir$package" -nt Pkgfile ] || utd=0 + [ ! -e "$PKGMK_ROOT/Pkgfile" ] || [ "$pkg_dir$package" -nt "$PKGMK_ROOT/Pkgfile" ] || utd=0 fi [ $utd = 0 ] || msg="$package is up to date." @@ -360,98 +354,89 @@ check_pkg_mtime() { # can be called even if some sources are missing } fetch_url() { - local u="$1"; local h="$2"; local finished=0; local gitsrc tag CLONE_ARGS + local u="$1"; local h="$2"; local finished=0; local giturl tag CLONE_ARGS local REPO SAVE_AS OCONTINUE OOUT; local m=0 - [ -x "$PKGMK_GIT_COMMAND" ] || PKGMK_GIT_COMMAND="/bin/false" - # Is this a url that requires git? - if [[ $u =~ ^(https|http|ssh|git)://.+/(.+)\.git($|#.*) ]]; then - # Did the port maintainer specify a branch other than 'master'? - tag=${BASH_REMATCH[3]#\#}; gitsrc="${u%.git*}.git" - [ -z "$tag" ] || CLONE_ARGS="--branch $tag" - - # If git is not installed, this source cannot be obtained at present - [ "$PKGMK_GIT_COMMAND" != "/bin/false" ] || return 1; - - # Is it our first attempt to download this repository? - if [ ! -d "$src_dir/$h.partial" ]; then - "$PKGMK_GIT_COMMAND" clone "$gitsrc" $CLONE_ARGS "$h.partial" - finished=$? - [ "$src_dir" = "" ] || { mv "$h.partial" "$src_dir"; - ln -s "$src_dir/$h.partial" "$h"; } - else - ln -s "$src_dir/$h.partial" "$h"; cd "$h" - "$PKGMK_GIT_COMMAND" pull $gitsrc $tag - finished=$?; cd .. - fi - return $finished; + if [[ "$u" =~ ^__git__ ]]; then + # git must be installed in order to obtain such a source + [ -x "$PKGMK_GIT_COMMAND" ] || return 1; + # + giturl="${u#__git__}" + # Did the port maintainer specify a branch other than 'master'? + tag="${giturl##*\#}" + [ -z "$tag" ] || { giturl="${giturl%\#*}"; CLONE_ARGS="--branch $tag"; } + # + # Has this project been downloaded before? + if [ -d "$src_dir/$h.partial" ]; then + ln -s "$src_dir/$h.partial" "$h"; cd "$h" + "$PKGMK_GIT_COMMAND" pull $giturl $tag + finished=$?; cd .. + else + "$PKGMK_GIT_COMMAND" clone "$giturl" $CLONE_ARGS "$h.partial" + finished=$?; [ "$src_dir" = "" ] || { mv "$h.partial" "$src_dir"; + ln -s "$src_dir/$h.partial" "$h"; } + fi + return $finished; + else + case "$PKGMK_DOWNLOAD_PROG" in + *wget) OCONTINUE="-c" + OOUT=("--compression=none" "--passive-ftp" "--no-directories" + "--tries=3" "--waitretry=3" $PKGMK_WGET_OPTIONS -O) ;; + *curl) OCONTINUE="-C -" + OOUT=("-L" "-#" "--fail" "--ftp-pasv" + --retry 3 --retry-delay 3 $PKGMK_CURL_OPTIONS -o) ;; + *) SAVE_AS="/bin/false" ;; + esac + # + # start with the mirrors defined in pkgmk.conf, then go to the url found in the Pkgfile + while [ $m -le ${#PKGMK_SOURCE_MIRRORS[@]} ] && [ $finished = 0 ] \ + && [[ ! $SAVE_AS =~ false$ ]]; do + [ "${PKGMK_SOURCE_MIRRORS[m]}" = "" ] && um=$u || \ + { REPO=${PKGMK_SOURCE_MIRRORS[m]%/}; um=$REPO/${u##*/}; } + m=$(( m+1 )) + # interrupted downloads from a previous run should be put where wget or curl will find them + [ -f "$src_dir/$h.partial" ] && { ln -s "$src_dir/$h.partial" . ; + SAVE_AS="$PKGMK_DOWNLOAD_PROG $um $OCONTINUE ${OOUT[@]}"; } \ + || SAVE_AS="$PKGMK_DOWNLOAD_PROG $um ${OOUT[@]}" + if $SAVE_AS "$h.partial"; then + finished=1 + [ "$src_dir" = "" ] || [ ! -w "$src_dir"/ ] || \ + { mv "$h.partial" "$src_dir/$h"; ln -sf "$src_dir/$h" . ; } + else # an interrupted download should not have its efforts destroyed by cleanup_work() + [ ! -s "$h.partial" ] || [ "$src_dir" = "" ] || \ + [ ! -w "$src_dir"/ ] || mv "$h.partial" "$src_dir" + fi + done fi - - # haven't returned yet, so a different transport protocol must be in effect - [[ "$PKGMK_DOWNLOAD_PROG" =~ wget$ ]] && { OCONTINUE="-c"; OOUT="--compression=none --passive-ftp --no-directories --tries=3 --waitretry=3 $PKGMK_WGET_OPTIONS -O"; } - [[ "$PKGMK_DOWNLOAD_PROG" =~ curl$ ]] && { OCONTINUE="-C -"; OOUT="-L -# --fail --ftp-pasv --retry 3 --retry-delay 3 $PKGMK_CURL_OPTIONS -o"; } - [[ "$PKGMK_DOWNLOAD_PROG" =~ (wget|curl)$ ]] || SAVE_AS=/bin/false - - # start with the mirrors defined in pkgmk.conf, then go to the url found in the Pkgfile - while [ $m -le ${#PKGMK_SOURCE_MIRRORS[@]} ] && [ $finished = 0 ] && [[ ! $SAVE_AS =~ false$ ]]; do - [ "${PKGMK_SOURCE_MIRRORS[m]}" = "" ] && um=$u || \ - { REPO=${PKGMK_SOURCE_MIRRORS[m]%/}; um=$REPO/${u##*/}; } - m=$(( m+1 )) - - # interrupted downloads from a previous run should be put where wget or curl will find them - [ -f "$src_dir/$h.partial" ] && { ln -s "$src_dir/$h.partial" . ; - SAVE_AS="$PKGMK_DOWNLOAD_PROG $um $OCONTINUE $OOUT"; } \ - || SAVE_AS="$PKGMK_DOWNLOAD_PROG $um $OOUT" - - if $SAVE_AS "$h.partial"; then - finished=1 - [ "$src_dir" = "" ] || [ ! -w "$src_dir"/ ] || \ - { mv "$h.partial" "$src_dir/$h"; ln -sf "$src_dir/$h" . ; } - else # an interrupted download should not have its efforts destroyed by cleanup_work() - [ ! -s "$h.partial" ] || [ "$src_dir" = "" ] || [ ! -w "$src_dir"/ ] \ - || mv "$h.partial" "$src_dir" - fi - done } -cat_manifest() { - case "$1" in - footprint) - pkginfo --footprint "$pkg_dir$package" \ - | sed "s|\tlib/modules/$(uname -r)/|\tlib/modules//|g" \ - | sort -k 3 - ;; - md5sum) - md5sum "${_local_[@]}" 2>&1 \ - | grep -v "is a directory" | sed 's, .*/, ,' | sort -k 2 - ;; - esac +cat_footprint() { + pkginfo --footprint "$pkg_dir$package" \ + | sed "s|\tlib/modules/$(uname -r)/|\tlib/modules//|g" \ + | sort -k 3 } -check_manifest() { - local FILTER CN CM - local TRUTH="$PKGMK_ROOT/.$1"; local diffs=0; local severity=error; - [ -f "$pkg_dir$package" ] || [ "$1" = "md5sum" ] || \ - { error "$package not found. Cannot check $1."; exit "$E_MANIFEST"; } - [ "$1" = "md5sum" ] && FILTER="-k 3" || FILTER="" +check_footprint() { + local CN CM + local TRUTH="$PKGMK_ROOT/.footprint"; local diffs=0; local severity=error; + [ -f "$pkg_dir$package" ] || \ + { error "$package not found. Cannot check footprint."; exit "$E_FOOTPRINT"; } if [ -f "$TRUTH" ]; then - diff -w -t -U 0 <(sort $FILTER "$TRUTH") <(cat_manifest $1 | sort $FILTER) | \ - sed '/^@@/d; /^+++/d; /^---/d; s/^+/NEW /g; s/^-/MISSING /g' > ".$1.diff" - if [ -s ".$1.diff" ]; then - CN=$(grep -c ^NEW ".$1.diff"); CM=$(grep -c ^MISSING ".$1.diff") - if [ "$1" = "footprint" ]; then - [ "$PKGMK_IGNORE_MISSING" = "yes" ] || diffs=$CM - [ "$PKGMK_IGNORE_NEW" = "yes" ] || diffs=$(( diffs+CN )) - [ $diffs = 0 ] && severity=warning - fi - $severity "$1 mismatch found:"; cat ".$1.diff" >&2 + diff -w -t -U 0 <(sort "$TRUTH") <(cat_footprint | sort) | \ + sed '/^@@/d; /^+++/d; /^---/d; s/^+/NEW /g; s/^-/MISSING /g' > ".footprint.diff" + if [ -s ".footprint.diff" ]; then + CN=$(grep -c ^NEW ".$1.diff"); CM=$(grep -c ^MISSING ".footprint.diff") + [ "$PKGMK_IGNORE_MISSING" = "yes" ] || diffs=$CM + [ "$PKGMK_IGNORE_NEW" = "yes" ] || diffs=$(( diffs+CN )) + [ $diffs = 0 ] && severity=warning + $severity "footprint mismatch found:"; cat ".footprint.diff" >&2 fi - rm ".$1.diff" + rm ".footprint.diff" else - warning ".$1 not found, creating new."; cat_manifest $1 > "$TRUTH" + warning "footprint not found, creating new."; cat_footprint > "$TRUTH" fi - [ $diffs = 0 ] || exit $E_MANIFEST + [ $diffs = 0 ] || exit $E_FOOTPRINT } parse_signify_output() { # chomps the output of check_signature() @@ -476,16 +461,13 @@ parse_signify_output() { # chomps the output of check_signature() } check_signature() { # called from $PKGMK_ROOT in the case "when"="pre-Pkgfile", - # otherwise called from within $work. Pass control to - # check_manifest() if the signature is missing and "ignore-md5" - # has not been requested. + # otherwise called from within $work. local reqfiles=(Pkgfile); local s=0; local when="$1"; if [ -f "$PKGMK_ROOT/.signature" ]; then [ "$when" = "pre-Pkgfile" ] || reqfiles=(.footprint) while [ "$when" = "pre-build" ] && [ "$s" -lt ${#_local_[@]} ]; do - [[ "${source[$s]}" =~ ^(http|https|ssh|git)://(.+)\.git ]] || \ - reqfiles+=("${_local_[$s]}") + [[ "${source[$s]}" =~ ^__git__ ]] || reqfiles+=("${_local_[$s]}") s=$(( s+1 )) done for FILE in "${reqfiles[@]}"; do @@ -499,10 +481,8 @@ check_signature() { # called from $PKGMK_ROOT in the case "when"="pre-Pkgfile", "${reqfiles[@]}" 2>&1 fi else - [ "$when" != "pre-Pkgfile" ] || echo "Pkgfile verification failed" - [ "$when" = "pre-Pkgfile" ] || [ "$PKGMK_IGNORE_MD5" = "yes" ] || \ - { info "Signature not found, falling back to old md5sum checking."; - check_manifest md5sum ; } + [ "$when" = "pre-Pkgfile" ] && echo "Pkgfile verification failed" + [ "$when" != "pre-Pkgfile" ] && echo "signature verification failed" fi } @@ -511,25 +491,26 @@ cat_signature() { [ -e "$PKGMK_ROOT/.footprint" ] || warning "Footprint not found, signature will be incomplete." for key in ~/.ssh/*.sec /etc/ports/*.sec; do [ -e "$key" ] || continue # workaround for brain-dead shell globbing - pub=/etc/ports/$(basename "${key/%.sec/.pub}") + pub="/etc/ports/$(basename "${key%.sec}.pub")" if [ -e "$pub" ]; then pub=$(readlink -f "$pub") for f in "$PKGMK_ROOT/Pkgfile" "$PKGMK_ROOT/.footprint"; do - [ -e $f ] && ordered+=( "$f" ) + [ -e "$f" ] && ordered+=( "$f" ) done for ((si=0; si < ${#source[@]}; si++)); do - [[ ${source[$si]} =~ ^(http|https|ssh|git)://.+/.+\.git($|#.*) ]] || ordered+=("${_local_[$si]}"); done - sha256sum --tag "${ordered[@]}" \ + # ignore git directories when writing the signature + [[ "${source[$si]}" =~ ^__git__ ]] || ordered+=("${_local_[$si]}"); done + sha256sum --tag "${ordered[@]}" \ | sed 's|^SHA256 (.*/\(.*\))\(.* = .*\)|SHA256 (\1)\2|' \ - | /usr/bin/signify -S -e -x - -q -s "$key" -m - \ - | sed "s|${key/%.sec/.pub}|$pub|" + | /usr/bin/signify -S -e -x - -q -s "$key" -m - \ + | sed "s|${key%.sec}.pub|$pub|" break fi done } refresh_signature() { - if [ -e ".signature" ] && [ ! -w ".signature" ]; then + if [ -e ".signature" ] && [ ! -w ".signature" ]; then echo "error: .signature not writable."; return $E_DIRPERM fi @@ -585,7 +566,7 @@ error() { ######################## end of subroutines ########################### ## Now ensure that they cannot be overwritten when sourcing Pkgfile ## readonly -f main info warning error print_help parse_options validate_pkgfile \ - check_reqvars check_pkg_mtime fetch_url cat_manifest check_manifest \ + check_reqvars check_pkg_mtime fetch_url cat_footprint check_footprint \ cat_signature check_signature parse_signify_output refresh_signature \ cleanup_work recursive trap "interrupted" SIGHUP SIGINT SIGQUIT SIGTERM