trying to use time

because i sometimes run them on weak computers

HISTORY

@@ -1,10 +1,18 @@
 HISTORY
 =======
 
-Version 0.1.0-rc1
+Version 0.1.0
 ---------------------
 This release includes breaking changes.
 
+### Cross-OS testing
+I have started testing Urchin across multiple operating systems.
+This gives access to more shells, as some shels are easier to install on
+certain operating systems.
+
+With this cross-OS test suite, I have extended support to more shells.
+A later version of Urchin could include a remote testing feature.
+
 ### Test root directory
 We introduce a concept of the root directory of a test suite.
 Such a concept is important in case you want to run subsets of your
@@ -72,7 +80,7 @@ might look for the dependency and then skip if it does not see the dependency.
 It might look like this.
 
     #!/bin/sh
-    if which inkscape; then
+    if ! which inkscape; then
       exit 3 # status code 3 for skip
     fi
     inkscape blah blah ...
@@ -171,10 +179,6 @@ set as an environment variable, and the latter was set with the -s flag..
 Urchin now uses the -s flag for both of these settings, and it mostly ignores
 the exported TEST_SHELL variable.
 
-Urchin also inspects the shebang line differently. Previously, Urchin would
-vary the shells with which a test is run if the shebang line either was absent
-or was #!/bin/sh. Now it varies the shell only if the shebang line is absent.
-
 If you pass -n/--disable-cycling, Urchin will invoke tests ordinarily and will
 only set the TEST_SHELL variable if it does not exist. If the TEST_SHELL
 variable is absent, it will be set to /bin/sh.

Makefile

@@ -0,0 +1,8 @@
+.PHONY: test install
+
+test:
+	./urchin tests
+	./urchin -s sh -v ./cross-os-tests
+
+install:
+	cp ./urchin /usr/bin

TODO

@@ -63,3 +63,54 @@ bash
 Alternatives
 --------------
 JSON.sh test suite
+
+Running in multiple environments
+-----------------------------------
+Setup for other environments includes the following.
+
+* Installing packages
+* `touch .zshrc`
+* Copy urchin and tests
+
+Bugs
+-------
+
+Both md5sum and md5 should be supported.
+
+Trouble logging in to hpux, irix, miros, netbsd, tru64, qnx, ....
+
+    $ rsync -e 'ssh -p 785' urchin tlevine@hpux.polarhome.com:.blah
+    HP-UX hpux.polarhome.com B.11.11 U 9000/785 (ta)
+    Welcome to HPUX/PA... member of polarhome.com realm
+
+    bash: rsync: command not found
+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+    rsync error: remote command not found (code 127) at io.c(226)
+    [sender=3.1.1]
+
+
+OpenIndiana grep does not support -q
+
+I get `/urchin: syntax error at line 84: \`}' unexpected` on
+unixware and solaris.
+
+mktemp
+
+    > tlevine@hpux.polarhome.com -p 785
+    F sh (8 seconds)
+      | 
+      | HP-UX hpux.polarhome.com B.11.11 U 9000/785 (ta)
+      | Welcome to HPUX/PA... member of polarhome.com realm
+      | 
+      | 
+      | HP-UX hpux.polarhome.com B.11.11 U 9000/785 (ta)
+      | Welcome to HPUX/PA... member of polarhome.com realm
+      | 
+      | mktemp: option requires an argument -- d
+      | ./urchin[96]: /tmp/tlevinea21441/log: Cannot create the specified file.
+
+date
+
+    tlevine@hpux64$ ./urchin tests/ -n -vv
+    date: bad format character - s
+

cross-os-tests/.run

@@ -0,0 +1,13 @@
+params="$(basename "${0}")"
+
+hostname="$(echo "${params}" | cut -d\  -f1)"
+if echo "${params}" | grep -q \ ; then
+  flags="$(echo "${params}" | cut -d\  -f2-)"
+fi
+
+urchin_dir=.urchin-cross-shell-test
+
+rsync --archive -e "ssh ${flags}" $RSYNC_FLAGS \
+  ../urchin ../tests "${hostname}":"${urchin_dir}"
+ssh "${hostname}" ${flags} \
+  "cd ${urchin_dir} && ./urchin --run-in-series tests"

cross-os-tests/tlevine@centos.polarhome.com -p 925

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@dada.pink

@@ -0,0 +1,3 @@
+#!/bin/sh
+# apt-get install bash dash ksh posh pdksh mksh yash zsh
+. ./.run

cross-os-tests/tlevine@debian-ppc.polarhome.com -p 625

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@debian.polarhome.com -p 755

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@dragonfly.polarhome.com -p 915

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@freebsd.polarhome.com -p 715

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@hpux-ia64.polarhome.com -p 965

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@hpux.polarhome.com -p 785

@@ -0,0 +1,3 @@
+#!/bin/sh
+RSYNC_FLAGS='--rsync-path=/usr/local/bin/rsync'
+. ./.run

cross-os-tests/tlevine@irix.polarhome.com -p 825

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@miros.polarhome.com -p 935

@@ -0,0 +1,3 @@
+#!/bin/sh
+# SSH public key needs to be in ~/.etc/ssh/authorized_keys
+. ./.run

cross-os-tests/tlevine@netbsd.polarhome.com -p 745

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@openbsd.polarhome.com -p 735

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@openindiana.polarhome.com -p 845

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@pidora.polarhome.com -p 615

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@qnx.polarhome.com -p 815

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@raspbian.polarhome.com -p 975

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@redhat.polarhome.com -p 795

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@scosysv.polarhome.com -p 895

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@solaris-x86.polarhome.com -p 865

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@solaris.polarhome.com -p 725

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@tru64.polarhome.com -p 835

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@ubuntu.polarhome.com -p 885

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

cross-os-tests/tlevine@unixware.polarhome.com -p 905

@@ -0,0 +1,2 @@
+#!/bin/sh
+. ./.run

packages/.gitignore

@@ -0,0 +1 @@
+*.tar.gz

packages/nongnu.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+name=urchin-$(../urchin --version)
+
+tmp=$(mktemp -d)
+mkdir $tmp/$name
+cp ../urchin ../readme.md ../AUTHORS ../COPYING $tmp/$name
+cd $tmp
+tar czf $name.tar.gz $name
+cd - > /dev/null
+mv $tmp/$name.tar.gz .
+rm -R $tmp

readme.md

@@ -51,7 +51,7 @@ Urchin is contained in a single file, so you can install it by copying it to a
 directory in your `PATH`. For example, you can run the following as root.
 
     cd /usr/local/bin
-    wget https://raw.githubusercontent.com/tlevine/urchin/v0.0.6/urchin
+    wget https://raw.githubusercontent.com/tlevine/urchin/v0.1.0-rc3/urchin
     chmod +x urchin
 
 Urchin can be installed with npm too.

tests/Flags/--timeout error message

@@ -0,0 +1,2 @@
+../../urchin -T aoeu .testsuite 2>&1 | grep Bad
+../../urchin -T .testsuite 2>&1 | grep Bad

tests/Flags/--timeout fail

@@ -1,3 +1 @@
-set -e
-$TEST_SHELL ../../urchin --shell sh .slow-tests
 ! $TEST_SHELL ../../urchin --shell sh --timeout 0.3 .slow-tests

tests/Flags/--timeout output

@@ -0,0 +1,4 @@
+$TEST_SHELL ../../urchin --shell sh --timeout 0.3 .slow-tests 2>&1 |
+  grep -v -- --timeout |
+  grep timeout
+test $? = 1

tests/Flags/--timeout success

@@ -0,0 +1 @@
+$TEST_SHELL ../../urchin --shell sh .slow-tests --timeout 1000

urchin

@@ -51,7 +51,17 @@ set -e
 # Kill subprocesses on interrupt.
 trap "kill -$$; exit" HUP INT TERM
 
-DEFAULT_SHELLS='sh bash dash mksh zsh'
+DEFAULT_SHELLS='
+sh
+bash
+dash
+ksh
+posh
+pdksh
+mksh
+yash
+zsh
+'
 if [ -n "${ZSH_VERSION}" ]; then
   # avoid "no matches found: *" error when directories are empty
   setopt NULL_GLOB
@@ -81,20 +91,44 @@ validate_test_arg() {
 }
 
 # All temporary files go here
-urchin_tmp=$(mktemp -d)
+urchin_tmp=$(mktemp)
+
+# Support HP-UX mktemp that has wrong exit codes and
+# can't make directories.
+if test -f "${urchin_tmp}"; then
+  rm "${urchin_tmp}"
+fi
+mkdir "${urchin_tmp}"
+
 > "${urchin_tmp}/log"
+
 urchin_exit() {
   rm -Rf "${urchin_tmp}"
   exit "$@"
 }
 
+if which md5 1> /dev/null 2> /dev/null; then
+  urchin_md5=md5
+elif which md5sum 1> /dev/null 2> /dev/null; then
+  urchin_md5=md5sum
+else
+  echo Could not find MD5 hash command >&2
+  urchin_exit 1
+fi
+
+
 stdout_file() {
   the_test="${1}"
   the_shell="${2}"
 
   x="${urchin_tmp}/stdout$(fullpath "$the_test")"
   mkdir -p "${x}"
-  echo "${x}/$(echo "${the_shell}" | md5sum | cut -d\  -f1)"
+  case "${urchin_md5}" in
+    md5sum) y=$(echo "${the_shell}" | md5sum | cut -d\  -f1) ;;
+    md5) y=$(echo "${the_shell}" | md5 -q) ;;
+    *) echo md5 command is not configured >&2; urchin_exit 1;;
+  esac
+  echo "${x}/${y}"
 }
 
 # Expand relative paths
@@ -137,7 +171,7 @@ urchin_root() {
 }
 
 # Urchin version number
-VERSION=0.1.0-rc2
+VERSION=0.1.1-unstable
 
 indent() {
   level="${1}"
@@ -152,6 +186,10 @@ recurse() {
   cycle_shell="${3}"
   TEST_SHELL="${4}"
 
+  if $print_debug; then
+    echo Entered directory "${PWD}"
+  fi
+
   for ignore in setup_dir teardown_dir setup teardown; do
     if test "$(basename "${potential_test}")" = "${ignore}"; then
       return
@@ -181,7 +219,12 @@ recurse() {
         run_in_series_dir=false
       fi
 
-      if test -f setup_dir; then . ./setup_dir; fi
+      if test -f setup_dir; then
+        if $print_debug; then
+          echo Sourcing "${PWD}/setup_dir"
+        fi
+        . ./setup_dir
+      fi
 
       for test in *; do
         if test "${test}" = '*' && ! test -e "${test}"; then
@@ -195,22 +238,41 @@ recurse() {
         if "${run_in_series}" || "${run_in_series_dir}"; then
           if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
           if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
-            if test -f teardown_dir; then . ./teardown_dir; fi
+            if test -f teardown_dir; then
+              if $print_debug; then
+                echo Sourcing "${PWD}/teardown_dir"
+              fi
+              . ./teardown_dir
+            fi
             return 1
           fi
         fi
       done
       wait
-      if test -f teardown_dir; then . ./teardown_dir; fi
+      if test -f teardown_dir; then
+        if $print_debug; then
+          echo Sourcing "${PWD}/teardown_dir"
+        fi
+        . ./teardown_dir
+      fi
     )
     elif [ -f "${potential_test}" ]; then
       cd -- "$(dirname -- "${potential_test}")"
 
+      if $print_debug; then
+        echo Running "${potential_test}"
+      fi
+
       # Determine the environment variable to define for test scripts
       # that reflects the specified or implied shell to use for shell-code tests.
       while read the_test_shell; do
         (
-          if test -f setup; then . ./setup; fi
+          if test -f setup; then
+            if $print_debug; then
+              echo Sourcing "${PWD}/setup"
+            fi
+            . ./setup
+          fi
 
           # Run the test
           start=$(date +%s)
@@ -237,7 +299,12 @@ recurse() {
           set -e
           finish=$(date +%s)
 
-          if test -f teardown; then . ./teardown; fi
+          if test -f teardown; then
+            if $print_debug; then
+              echo Sourcing "${PWD}/teardown"
+            fi
+            . ./teardown
+          fi
 
           if [ "${exit_code}" -eq 0 ]; then
             result=ok
@@ -256,14 +323,19 @@ recurse() {
         if "${run_in_series}"; then
           if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
           if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
-            if test -f teardown_dir; then . ./teardown_dir; fi
+            if test -f teardown_dir; then
+              if $print_debug; then
+                echo Sourcing "${PWD}/teardown_dir"
+              fi
+              . ./teardown_dir
+            fi
             return 1
           fi
         fi
       done < "${shell_list}"
       wait
     else
-      echo "${potential_test}: Neither file nor directory!?" > /dev/stderr
+      echo "${potential_test}: Neither file nor directory!?" >&2
     fi
   else
     # Shell is ''
@@ -275,11 +347,9 @@ report_outcome() {
   root="${1}"
   tap_format="${2}"
   log_file="${3}"
-  start="${4}"
-  finish="${5}"
+  elapsed="${4}"
 
   escaped_root="$(fullpath "${root}" | sed 's/\//\\\//g')"
-  elapsed=$(($finish - $start))
 
   if "${tap_format}"; then
     printf \#\ 
@@ -295,7 +365,7 @@ report_outcome() {
   # Use a temporary file rather than a pipe because a pipe starts a sub-shell
   # and thus makes the above variables local.
   sorted_log_file=$(mktemp)
-  cat "${log_file}" | LC_ALL=POSIX sort > "${sorted_log_file}"
+  cat "${log_file}" | LC_COLLATE=C sort > "${sorted_log_file}"
 
   while read line; do
     abspath=$(echo "${line}" | cut -f1)
@@ -450,8 +520,8 @@ The following flags affect how Urchin processes tests.
 -b, --run-in-series      Run tests in series. The default is to run tests
                          in parallel where possible.
 -e, --exit-on-fail       Stop running if any single test fails.
-                         This is useful if you are running something
-                         configuration files with Urchin.
+                         This can be useful if you are running something
+                         other than test files with Urchin.
 -T, --timeout <seconds>  Kill a test if it runs for longer than the
                          specified duration. The default is no timeout.
 -f, --force              Force running even if the test directory's name
@@ -464,6 +534,8 @@ effect when combined with --tap.
 -p, --pretty             Print results in color and with fancy symbols.
 -t, --tap                Format output in Test Anything Protocol (TAP)
 
+And these options affect how much is printed.
+
 -q, --quiet              Print nothing to stdout;
                          the only output is the exit code.
 (default verbosity)      Print names of failed tests and counts
@@ -471,14 +543,14 @@ effect when combined with --tap.
 -v                       Print stdout from failing tests.
 -vv                      Print names of passed tests.
 -vvv, --verbose          Print stdout from all tests.
--vvvv, --debug           Print debugging messages (XXX not implemented)
+-vvvv, --debug           Print debugging messages.
 
 The remaining flags provide information about urchin.
 
 -h, --help               Display this help.
 --version                Display the version number.
 
-Go to https://github.com/tlevine/urchin for documentation on writing tests.
+Go to https://thomaslevine.com/!/urchin/ for documentation on writing tests.
 
 EOF
 }
@@ -518,6 +590,7 @@ main() {
   print_not_ok=true
   print_ok_stdout=false
   print_not_ok_stdout=false
+  print_debug=false
   while [ "${#}" -gt 0 ]
   do
       case "${1}" in
@@ -537,14 +610,23 @@ main() {
             validate_strings "${shell_for_sh_tests}" 'Shell paths'
             if echo "${shell_for_sh_tests}" | grep -q \ ; then
               echo "Warning: It is best if shell paths contain no spaces so that
-  you don't need to quote the TEST_SHELL variable." > /dev/stderr
+  you don't need to quote the TEST_SHELL variable." >&2
             fi
 
             echo "${shell_for_sh_tests}" >> "${shell_list}"
             ;;
           -n|--disable-cycling) cycle_shell=false;; 
           -t|--tap) tap_format=true;;
-          -T|--timeout) shift; urchin_timeout="${1}" ;;
+          -T|--timeout)
+            shift
+            urchin_timeout="${1}" 
+            if ! {
+              echo "${urchin_timeout}" |
+              grep '[0-9][0-9.]*\(s\|m\|h\|d\|\)'
+            }; then
+              echo Bad timeout argument: "${urchin_timeout}" >&2
+              urchin_exit 1
+            fi ;;
           -p|--pretty) print_in_color=true;;
 
           -q|--quiet)    print_not_ok=false
@@ -555,7 +637,10 @@ main() {
           -vvv|--verbose)print_not_ok_stdout=true
                          print_ok=true;
                          print_ok_stdout=true;;
-          -vvvv|--debug) echo 'Not implemented' > /dev/stderr && exit 1;;
+          -vvvv|--debug) print_not_ok_stdout=true
+                         print_ok=true;
+                         print_ok_stdout=true
+                         print_debug=true;;
 
           -h|--help) urchin_help
                      urchin_exit 0;;
@@ -588,7 +673,7 @@ main() {
   if ! test -f "${shell_list}"; then
     if $cycle_shell; then
       for shell in $DEFAULT_SHELLS; do
-        if which $shell > /dev/null; then
+        if which $shell 1> /dev/null 2> /dev/null; then
           echo $shell >> "$shell_list"
         fi
       done
@@ -597,11 +682,16 @@ main() {
     fi
   fi
 
+  if $print_debug; then
+    echo Cycling with the following shells:
+    cat "${shell_list}"
+  fi
+
   if test -n "${urchin_timeout}"; then
     # Choose the timeout command
-    if timeout -t 0 true; then
+    if timeout -t 0 true 2> /dev/null; then
       TIMEOUT="timeout -t ${urchin_timeout}"
-    elif timeout 0 true; then
+    elif timeout 0 true 2> /dev/null; then
       TIMEOUT="timeout ${urchin_timeout}"
     else
       echo I couldn\'t figure out how to use your version of timeout >&2
@@ -615,22 +705,23 @@ main() {
   fi
 
   # --------------------   REALLY RUN   -------------------- #
-  start=$(date +%s)
+  elapsed=$({
+    time {
+      # 1 test file or folder to run
+      # 2 urchin root
+      # 3 Should we cycle shells?
+      # 4 TEST_SHELL
+      while read seed; do
+        recurse "$(fullpath "${seed}")" "${root}" "${cycle_shell}" \
+          "${TEST_SHELL}" || break
+      done < "${test_arg_list}"
+    }
+  }  3>&1 1>&2 2>&3 3>& | grep ^real | cut -f 2)
 
-  # 1 test file or folder to run
-  # 2 urchin root
-  # 3 Should we cycle shells?
-  # 4 TEST_SHELL
-  while read seed; do
-    recurse "$(fullpath "${seed}")" "${root}" "${cycle_shell}" \
-      "${TEST_SHELL}" || break
-  done < "${test_arg_list}"
-  finish=$(date +%s)
-
-  test $(cat "${urchin_tmp}"/log | wc -l) -gt 0 || {
-    echo 'No tests found' > /dev/stderr
+  if test $(cat "${urchin_tmp}"/log | wc -l) -eq 0; then
+    echo 'No tests found' >&2
     urchin_exit 1
-  }
+  fi
     
   report_outcome "${root}" "${tap_format}" "${urchin_tmp}"/log "${start}" \
     "${finish}"