diff --git a/Tools/portbuild/scripts/dopackages b/Tools/portbuild/scripts/dopackages index ad7480294ef1..87cbe231abb9 100755 --- a/Tools/portbuild/scripts/dopackages +++ b/Tools/portbuild/scripts/dopackages @@ -20,7 +20,7 @@ errorexit () { } usage () { - echo "usage: [-restart] [-ftp] [-cdrom] [-nodummy] [-nobuild] [-noindex] [-noduds] [-nocvsup] [-nocvs] [-noportscvs] [-norestr] [-plistcheck] branch date" + echo "usage: [-continue] [-nofinish] [-finish] [-restart] [-ftp] [-cdrom] [-nodummy] [-nobuild] [-noindex] [-noduds] [-nocvsup] [-nocvs] [-noportscvs] [-norestr] [-plistcheck] branch date" errorexit 1 } @@ -86,6 +86,7 @@ makeduds () { echo "duds diff:" diff ${pb}/${arch}/${branch}/duds.old ${pb}/${arch}/${branch}/duds cp -p ${pb}/${arch}/${branch}/duds ${pb}/${arch}/${branch}/duds.old + cp -p ${pb}/${arch}/${branch}/duds ${pb}/${arch}/${branch}/duds.orig } # usage: setupnode pb arch scripts branch me node md5 tmpdir @@ -151,10 +152,10 @@ archiveports () { arch=$2 branch=$3 - echo "started archive of /${branch}/ports at $(date)" + echo "started archive of ${pb}/${arch}/${branch}/ports at $(date)" cd ${pb}/${arch}/${branch} tar --exclude CVS -czf ${pb}/${arch}/${branch}/tarballs/ports.tar.gz ports - echo "ended archive of /${branch}/ports at $(date)" + echo "ended archive of ${pb}/${arch}/${branch}/ports at $(date)" } # usage: generatemd5 pb branch @@ -200,6 +201,9 @@ nodummy=0 cdrom=0 ftp=0 restart=0 +cont=0 +finish=0 +nofinish=0 # optional arguments while [ $# -gt 2 ]; do @@ -237,9 +241,19 @@ while [ $# -gt 2 ]; do x-ftp) ftp=1 ;; + x-continue) + cont=1 + ;; x-restart) restart=1 ;; + x-nofinish) + nofinish=1 + ;; + x-finish) + nobuild=1 + finish=1 + ;; *) usage ;; @@ -247,6 +261,12 @@ while [ $# -gt 2 ]; do shift done +if [ "$restart" = 1 -o "$cont" = 1 -o "$finish" = 1 ]; then + skipstart=1 +else + skipstart=0 +fi + # mandatory arguments branch=$1 date=$2 @@ -276,7 +296,7 @@ fi export DISTDIR=${pb}/${arch}/${branch}/distfiles export PACKAGES=${pb}/${arch}/${branch}/packages -if [ "$restart" = 0 ]; then +if [ "$skipstart" = 0 ]; then if [ "$noportscvs" = 0 ]; then echo "================================================" echo "running cvs update -PAd on ${PORTSDIR}" @@ -319,16 +339,18 @@ if [ "$restart" = 0 ]; then fi fi -md5=$(/sbin/md5 ${pb}/${arch}/${branch}/tarballs/bindist.tar | awk '{print $4}') -echo "================================================" -echo "setting up nodes" -echo "================================================" -for node in $(awk '{print $1}' ${pb}/${arch}/mlist); do - setupnode ${pb} ${arch} ${scripts} ${branch} ${me} ${node} ${md5} ${scratchdir} & - sleep 2 -done +if [ "$nobuild" = 0 ]; then + md5=$(/sbin/md5 ${pb}/${arch}/${branch}/tarballs/bindist.tar | awk '{print $4}') + echo "================================================" + echo "setting up nodes" + echo "================================================" + for node in $(awk '{print $1}' ${pb}/${arch}/mlist); do + setupnode ${pb} ${arch} ${scripts} ${branch} ${me} ${node} ${md5} ${scratchdir} & + sleep 2 + done +fi -if [ "$restart" = 0 ]; then +if [ "$skipstart" = 0 ]; then restrictedlist ${pb} ${arch} ${scripts} ${branch} & sleep 2 @@ -339,7 +361,7 @@ fi wait -if [ "$restart" = 0 ]; then +if [ "$skipstart" = 0 ]; then checkindex ${pb} ${arch} ${branch} ${pb}/scripts/makeparallel ${arch} ${branch} @@ -356,6 +378,7 @@ if [ "$restart" = 0 ]; then fi # hack to extend length of dependency chain to build quickports first + # XXX Does this actually work? - kk for i in ${quickports}; do if [ -d $i ]; then quickpkg="$(cd $i; make package-name)${PKGSUFFIX}" @@ -370,8 +393,8 @@ if [ "$restart" = 0 ]; then cd ${pb}/${arch} if [ "$nobuild" = 0 ]; then - mkdir -p ${branch}/distfiles - chown -R ${user} ${branch}/distfiles + #mkdir -p ${branch}/distfiles + #chown -R ${user} ${branch}/distfiles cd ${pb}/${arch}/${branch}/bak rm -rf errors logs packages old-errors @@ -387,6 +410,7 @@ if [ "$restart" = 0 ]; then cd ${pb}/${arch}/${branch} mkdir -p packages/All + mkdir -p ${pb}/${arch}/archive/errorlogs rm -rf ${pb}/${arch}/archive/errorlogs/e.${branch}.${date} ${pb}/${arch}/archive/errorlogs/e.${branch}.${shortdate} mkdir -p ${pb}/${arch}/archive/errorlogs/e.${branch}.${date} @@ -397,7 +421,7 @@ if [ "$restart" = 0 ]; then mkdir -p ${pb}/${arch}/archive/errorlogs/a.${branch}.${date} ln -sf ${pb}/${arch}/archive/errorlogs/a.${branch}.${date} ${pb}/${arch}/${branch}/logs ln -sf a.${branch}.${date} ${pb}/${arch}/archive/errorlogs/a.${branch}.${shortdate} - chown -RL ${user} errors logs packages +# chown -RL ${user} errors logs packages echo "error logs in ${pb}/${arch}/archive/errorlogs/e.${branch}.${date}" cp -p ${pb}/${arch}/cvsdone ${pb}/${arch}/archive/errorlogs/e.${branch}.${date}/cvsdone @@ -419,7 +443,15 @@ if [ "$restart" = 0 ]; then fi if [ "$nobuild" = 0 ]; then - find /tmp -name 'make*' -mmin +60 | xargs rm -f + cd ${pb}/${arch}/${branch} + + if [ "$cont" = 1 ]; then + find errors/ -name \*.log | sed -e 's,\.log$,,' -e 's,^errors/,,' >> duds.errors + cat duds duds.errors | sort -u > duds.new + mv duds.new duds + else + cp duds.orig duds + fi count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/${arch}/mlist | sed -e 's/\..*$//') cd ${pb}/${arch}/${branch}/packages/All @@ -474,122 +506,88 @@ if [ "$nobuild" = 0 ]; then phase2end=$(date +%s) echo "phase 2 took $(date -u -j -r $(($phase2end - $phase2start)) | awk '{print $4}')" - rm Makefile -# if [ "$nodummy" = 0 ]; then -# rm ${dummypkgs} -# fi - - ${pb}/scripts/chopindex ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/${branch}/packages > \ - ${pb}/${arch}/${branch}/packages/INDEX echo $(ls -1 ${pb}/${arch}/${branch}/packages/All | wc -l) "packages built" echo $(cat ${pb}/${arch}/${branch}/packages/${INDEXFILE} | wc -l) "lines in INDEX" echo $(echo $(du -sk ${pb}/${arch}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages" echo $(echo $(du -sk ${pb}/${arch}/${branch}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles" +fi + +cd ${pb}/${arch}/${branch} +if grep -qE '(ptimeout|pnohang): killing' make.1; then + echo "The following port(s) timed out:" + grep -E '(ptimeout|pnohang): killing' make.1 | sed -e 's/^.*ptimeout:/ptimeout:/' -e 's/^.*pnohang:/pnohang:/' +fi + +# Clean up temporary duds file +if [ "$cont" = 1 ]; then + cp duds.orig duds +fi + +cd ${pb}/${arch}/${branch}/packages/All +if [ "$nofinish" = 0 ]; then + rm -f Makefile # Always delete restricted packages/distfiles since they're published on the # website echo "deleting restricted ports" sh ${pb}/${arch}/${branch}/restricted.sh - ${pb}/scripts/chopindex ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/${branch}/packages > \ - ${pb}/${arch}/${branch}/packages/INDEX - echo $(ls -1 ${pb}/${arch}/${branch}/packages/All | wc -l) "packages built" - echo $(cat ${pb}/${arch}/${branch}/packages/${INDEXFILE} | wc -l) "lines in INDEX" - echo $(echo $(du -sk ${pb}/${arch}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages" - echo $(echo $(du -sk ${pb}/${arch}/${branch}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles" - rm -rf ${pb}/${arch}/${branch}/bad - mkdir -p ${pb}/${arch}/${branch}/bad - echo "checking packages" - for i in *${PKGSUFFIX}.; do - if ! ${PKGZIPCMD} -t $i; then - echo "Warning: package $i is bad, moving to ${pb}/${arch}/${branch}/bad" - # the latest link will be left behind... - mv $i ${pb}/${arch}/${branch}/bad - rm ../*/$i - fi - done - - cd ${pb}/${arch}/${branch} - if grep -qE '(ptimeout|pnohang): killing' make.1; then - echo "The following port(s) timed out:" - grep -E '(ptimeout|pnohang): killing' make.1 | sed -e 's/^.*ptimeout:/ptimeout:/' -e 's/^.*pnohang:/pnohang:/' + if [ "$cdrom" = 1 ]; then + echo "deleting cdrom restricted ports" + sh ${pb}/${arch}/${branch}/cdrom.sh fi + # Remove packages not listed in INDEX + ${pb}/scripts/prunepkgs ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/${branch}/packages +fi + +#rm -rf ${pb}/${arch}/${branch}/bad +#mkdir -p ${pb}/${arch}/${branch}/bad +#echo "checking packages" +#for i in *${PKGSUFFIX}; do +# if ! ${PKGZIPCMD} -t $i; then +# echo "Warning: package $i is bad, moving to ${pb}/${arch}/${branch}/bad" +# # the latest link will be left behind... +# mv $i ${pb}/${arch}/${branch}/bad +# rm ../*/$i +# fi +#done + +if [ "$nofinish" = 0 ]; then + # Remove INDEX entries for packages that do not exist + ${pb}/scripts/chopindex ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/${branch}/packages > ${pb}/${arch}/${branch}/packages/INDEX + cd ${pb}/${arch}/${branch}/old-errors - new="" for i in *.log; do - if [ ! -f ../errors/$i ]; then - new="$new $(basename $i .log)" - cp -p ${i} ${pb}/${arch}/${branch}/errors/old-errors - fi + cp -p ${i} ${pb}/${arch}/${branch}/errors/old-errors; done - if [ "x$new" != "x" ]; then - echo "The following port(s) didn't build the first time around: $new" - fi + archiveports ${pb} ${arch} ${branch} & generatemd5 ${pb} ${arch} ${branch} & - sleep 5 - - echo "================================================" - echo "new failures" - echo "================================================" - cd ${pb}/${arch}/${branch}/errors - for i in *.log; do - if [ ! -f ${pb}/${arch}/${branch}/bak/errors/$i ]; then - echo -n " $(basename $i .log)" - fi - done - - if cd ${pb}/${arch}/${branch}/bak/packages/All; then - echo - echo "================================================" - echo "old packages" - echo "================================================" - for i in *${PKGSUFFIX}; do - if [ ! -f ${pb}/${arch}/${branch}/packages/All/$i ]; then - echo -n " $(basename $i ${PKGSUFFIX})" - fi - done - fi - - if cd ${pb}/${arch}/${branch}/bak/errors; then - echo - echo "================================================" - echo "old failures" - echo "================================================" - for i in *.log; do - if [ ! -f ${pb}/${arch}/${branch}/errors/$i ]; then - echo -n " $(basename $i .log)" - fi - done - fi - - echo - echo "================================================" - echo "new packages" - echo "================================================" - cd ${pb}/${arch}/${branch}/packages/All - for i in *${PKGSUFFIX}; do - if [ ! -f ${pb}/${arch}/${branch}/bak/packages/All/$i ]; then - echo -n " $(basename $i ${PKGSUFFIX})" - fi - done - echo wait ls -asFlrt ${pb}/${arch}/${branch}/packages/All > ${pb}/${arch}/${branch}/logs/ls-lrt cp -p ${pb}/${arch}/${branch}/make.[01] ${pb}/${arch}/${branch}/logs + # Always delete restricted distfiles + echo "deleting restricted distfiles" + sh ${pb}/${arch}/${branch}/restricted.sh + + if [ "$cdrom" = 1 ]; then + echo "deleting cdrom restricted distfiles" + sh ${pb}/${arch}/${branch}/cdrom.sh + fi + echo "================================================" echo "copying distfiles" echo "================================================" echo "started at $(date)" cd ${pb}/${arch} - rm -rf ${branch}/bak/distfiles - mv -f ${branch}/distfiles ${branch}/bak - ${scripts}/dodistfiles ${pb}/${arch}/${branch}/bak/distfiles - rm -rf ${pb}/${arch}/${branch}/bak/distfiles/.btmp - rm -f ${pb}/${arch}/${branch}/bak/distfiles/.done + #rm -rf ${branch}/bak/distfiles + #mv -f ${branch}/distfiles ${branch}/bak + ${scripts}/dodistfiles ${pb}/${arch}/${branch}/distfiles + rm -rf ${pb}/${arch}/${branch}/distfiles/.pbtmp + rm -f ${pb}/${arch}/${branch}/distfiles/.done if [ "$branch" != "4-exp" ]; then # Currently broken - kk @@ -599,7 +597,7 @@ if [ "$nobuild" = 0 ]; then echo "================================================" echo "copying packages" echo "================================================" - ${scripts}/docppackages ${branch} + ${scripts}/docppackages ${arch} ${branch} fi fi fi