Automatic commit with j

HISTORY

@@ -7,6 +7,22 @@ Version 0.0.7
 The Molly-guard is now more accepting. For example, you no longer need to
 pass -f in this case: https://github.com/creationix/nvm/issues/357
 
+Previously, tests were run if they were executable and were otherwise marked
+as skipped. Now, an executable script can indicate that it is skipped by
+exiting with code 3.  For example, if a test requires some dependancy, it
+might look for the dependency and then skip if it does not see the dependency.
+It might look like this.
+
+    #!/bin/sh
+    if which inkscape; then
+      exit 3 # status code 3 for skip
+    fi
+    inkscape blah blah ...
+
+I chose status code 3 sort of arbitrarily at first, but it turns out that it
+would the appropriate status code if these tests were Nagios plugins, as the
+concept of skipping a test is similar to the Nagios concept of unknown service
+status (https://nagios-plugins.org/doc/guidelines.html#AEN78).
 
 Version 0.0.6
 ---------------------

TODO

@@ -1,18 +1,6 @@
 Things I want
 =============
 
-Skip tests
--------------
-I want tests to be able to decide to skip themselves. For example, if a test
-requires some dependancy, it might look for the dependency and then skip if it
-does not see the dependency. It might look like this.
-
-    #!/bin/sh
-    if which parallel > /dev/null; then
-      exit 3 # status code 3 for skip
-    fi
-    parallel blah blah ...
-
 Test speed
 -------------
 Make tests run faster.
@@ -31,6 +19,11 @@ Hmm or maybe there's a compromise: Tell people to mount /tmp as a tmpfs so
 that temp files are fast. Maybe allow people to set some other directory as
 the temporary file place, in case they want a different tmpfs location.
 
+In order to run things in parallel, we have to change how we do the
+stdout_file. I think it's easiest to create separate files for each test and
+to save them in testroot/.urchin/stdout/$filename. The test root would be
+defined as the closest ancestor containing a .urchin directory.
+
 Options
 -------------
 I want long options. For example, there's presently -f and -e.
@@ -151,6 +144,13 @@ on the nvm tests. I wound up running this instead.
 The Molly guard would be assessed, and the corresponding setup, setup_dir,
 teardown, and teardown_dir files would be run in the appropriate order.
 
+In order to know how far up the tree to evaluate the setup, &c. files,
+I think it would make sense to require that a ".urchin" file be placed in the
+root of the tests. Urchin would keep going up until it sees this file, and it
+would evaluate the appropriate setup, &c. files from there down to the
+particular test file of interest. We would also use this for testing
+directtories more correctly.
+
 Running automated tasks
 -------------------------
 Urchin might be appropriate for if you have lots of tasks that you want to run

tests/.urchin/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

tests/Exit on fail if -e is passed.

@@ -1,5 +1,5 @@
 tmp=$(mktemp)
-../urchin -e -b -f ./.die-on-fail > $tmp
+../urchin -e -f ./.die-on-fail > $tmp
 result=$?
 
 grep '1 should run.' $tmp

tests/TAP/Running with -t should produce TAP output.

@@ -1,4 +1,4 @@
 tmp=$(mktemp)
 
-../../urchin -b -t .testsuite/ | sed 1d > $tmp
+../../urchin -t .testsuite/ | sed 1d > $tmp
 diff $tmp .expected-output

tests/Test root contains .urchin./.meta-assess

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+observed=$(../../urchin "$1" .testsuite/a/b)
+expected=.testsuite/a/b/../..
+
+test "$observed" = "$expected"

tests/Test root contains .urchin./.testsuite/.urchin/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

tests/Test root contains .urchin./.testsuite/a/b/c/testcase

@@ -0,0 +1 @@
+#!/usr/bin/env true

tests/Test root contains .urchin./Assess root on a file.

@@ -0,0 +1,3 @@
+#!/bin/sh
+#!/bin/sh
+../../urchin --root .testsuite/a/b/c/testcase

tests/Test root contains .urchin./Assess root with --root.

@@ -0,0 +1,2 @@
+#!/bin/sh
+./.meta-assess --root

tests/Test root contains .urchin./Assess root with -r.

@@ -0,0 +1,2 @@
+#!/bin/sh
+./.meta-assess -r

tests/Test root contains .urchin./No root set

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+tmp=$(mktemp -d)
+../../urchin --root $tmp 2>&1 | grep suppress
+../../urchin --root $tmp 2>&1 | grep "'$tmp/.urchin'"
+../../urchin --root $tmp
+rm -R "$tmp"

tests/Test root contains .urchin./Wrong filename

@@ -0,0 +1,2 @@
+#!/bin/sh
+! ../../urchin --root .testsuite/a/not-a-file

tests/skip/.test/fail

@@ -0,0 +1 @@
+exit 1

tests/skip/.test/skip

@@ -0,0 +1 @@
+exit 3

tests/skip/.test/succeed

@@ -0,0 +1 @@
+exit 0

tests/skip/Skip on exit code 3.

@@ -0,0 +1 @@
+test $(../../urchin -t .test/ | grep -c SKIP) -eq 1

urchin

@@ -7,7 +7,8 @@
 # No part of urchin, including this file, may be copied, modified, propagated,
 # or distributed except according to the terms contained in the COPYING file.
 
-
+# Stop on error.
+set -e
 
 # Make sure that CDPATH isn't set, as it causes `cd` to behave unpredictably -
 # notably, it can produce output, which breaks fullpath().
@@ -16,11 +17,37 @@ unset CDPATH
 # Urchin version number
 VERSION=0.0.6
 
-fullpath() {
-  (
-    cd -- "$1"
-    pwd
-  )
+urchin_root() {
+  # Call recursively but remember the original argument.
+  current="$1"
+  if test -n "$2"; then
+    orig="$2"
+  else
+    orig="$1"
+  fi
+
+  if ! test -e "$current"; then
+    echo "$current: No such file or directory">&2
+    exit 1
+  elif test -f "$current"; then
+    urchin_root "$(dirname "$current")" "$orig"
+  elif test -d "$current"/.urchin; then
+    echo "$current"
+  elif test "$(readlink -f "$current")" = /; then
+    if test -d "$orig"; then
+      origdir="$orig"
+    else
+      origdir=$(dirname "$orig")
+    fi
+    echo "You must create a .urchin directory to specify the urchin root.
+Run something like this, and then run urchin again.
+
+  mkdir '$origdir/.urchin'
+" >&2
+    exit 1
+  else
+    urchin_root "$current"/.. "$orig"
+  fi
 }
 
 indent() {
@@ -30,16 +57,36 @@ indent() {
   fi
 }
 
+escape_slashes() {
+  echo "$1" | sed s+/+\\\\/+g
+}
+
+get_stdout_file() {
+  root="$(readlink -f "$1")"
+  test_file="$(readlink -f "$2")"
+  eroot="$(escape_slashes "$root")"
+  base="$(echo "$test_file" | sed "s/^$eroot/$eroot\/.urchin/")"
+  if test -d "$1"; then
+    echo "$base"/.index
+  else
+    echo "$base"
+  fi
+}
+
 recurse() {
   potential_test="$1"
   indent_level="$2"
   shell_for_sh_tests="$3"
+  root="$4"
 
   [ "$potential_test" = 'setup_dir' ] && return
   [ "$potential_test" = 'teardown_dir' ] && return
   [ "$potential_test" = 'setup' ] && return
   [ "$potential_test" = 'teardown' ] && return
 
+  stdout_file="$(get_stdout_file "$root" "$potential_test")"
+  mkdir -p "$(dirname "$stdout_file")"
+
   [ $indent_level -eq 0 ] && : > "$stdout_file"
 
   if [ -d "$potential_test" ]
@@ -67,23 +114,17 @@ recurse() {
         [ -f setup ] && [ -x setup ] && ./setup >> "$stdout_file"
 
         # $2 instead of $indent_level so it doesn't clash
-        (
-          recurse "${test}" $(( $2 + 1 )) "$shell_for_sh_tests"
-          exit_code=$?
-          [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
-          exit $exit_code
-        ) &
+        recurse "${test}" "$(( $2 + 1 ))" "$shell_for_sh_tests" "$root"
+        exit_code=$?
 
-        if $single_process; then
-          wait $!
-          exit_code=$?
-          if $exit_on_fail && test $exit_code -ne 0; then
-            [ -f teardown_dir ] && [ -x teardown_dir ] && ./teardown_dir >> "$stdout_file"
-            return 1
-          fi
+        if $exit_on_fail && test $exit_code -ne 0; then
+          [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
+          [ -f teardown_dir ] && [ -x teardown_dir ] && ./teardown_dir >> "$stdout_file"
+          return 1
         fi
+
+        [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
       done
-      wait
       [ -f teardown_dir ] && [ -x teardown_dir ] && ./teardown_dir >> "$stdout_file"
     )
     if $tap_format; then
@@ -104,12 +145,13 @@ recurse() {
       else
         TEST_SHELL="$TEST_SHELL" ./"$potential_test" > "$stdout_file" 2>&1
       fi
-
       exit_code="$?"
 
       [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
       if [ $exit_code -eq 0 ]; then
         result=success
+      elif [ $exit_code -eq 3 ]; then
+        result=skip
       else
         result=fail
       fi
@@ -166,7 +208,6 @@ recurse() {
       return 1
     fi
   fi
-  [ $indent_level -eq 0 ] && rm "$stdout_file"
 }
 
 has_sh_or_no_shebang_line() {
@@ -189,8 +230,6 @@ $USAGE
 
 -s <shell>  Invoke test scripts that either have no shebang line at all or
             have shebang line "#!/bin/sh" with the specified shell.
--b, --series   Run all tests in series in a single process. The default
-               is to run tests from the same directory in parallel.
 -e          Stop running if any single test fails. This is helpful if you want
             to use Urchin to run things other than tests, such as a set of
             configuration scripts.
@@ -198,6 +237,7 @@ $USAGE
             contain the word "test".
 -t          Format output in Test Anything Protocol (TAP)
 -h, --help  This help.
+-r, --root  Print the Urchin root for a particular file or directory.
 -v          Display the version number.
 
 Go to https://github.com/tlevine/urchin for documentation on writing tests.
@@ -217,28 +257,88 @@ plural () {
   fi
 }
 
-urchin_go() {
-  rm -f "$logfile"
+urchin_molly_guard() {
+  {
+    echo
+    echo 'The name of the directory on which you are running urchin'
+    echo 'does not contain the word "test", so I am not running,'
+    echo 'in case that was an accident. Use the -f flag if you really'
+    echo 'want to run urchin on that directory.'
+    echo
+  } >&2
+  exit 1
+}
+
+shell_for_sh_tests=
+force=false
+exit_on_fail=false
+tap_format=false
+while [ $# -gt 0 ]
+do
+    case "$1" in
+        -e) exit_on_fail=true;;
+        -f) force=true;;
+        -r|--root)
+          shift
+          urchin_root "$1"
+          exit;;
+        -s)
+          shift
+          shell_for_sh_tests=$1
+          which "$shell_for_sh_tests" > /dev/null || {
+            echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2
+            urchin_help >&2
+            exit 11
+          } ;;
+        -t) tap_format=true;;
+        -h|--help) urchin_help
+          exit 0;;
+        -v) echo "$VERSION"
+            exit;;
+        -*) urchin_help >&2
+            exit 1;;
+        *)  break;;
+    esac
+    shift
+done
+
+# Verify argument for main stuff
+if [ "$#" != '1' ] || [ ! -d "$1" ]
+  then
+  [ -n "$1" ] && [ ! -d "$1" ] && echo "Not a directory: '$1'" >&2
+  echo "$USAGE" >&2
+  exit 11
+fi
+
+# Run or present the Molly guard.
+root="$(urchin_root "$1")"
+if basename "$(readlink -f "$root")" |
+  grep -Fi 'test' > /dev/null || $force; then
+  logfile="$(readlink -f "$root")/.urchin/.log"
+  echo > "$logfile"
+
   if "$tap_format"; then
     printf \#\ 
   fi
   echo Running tests at $(date +%Y-%m-%dT%H:%M:%S)
-  start=$(date +%s)
+  start="$(date +%s)"
 
   # Determine the environment variable to define for test scripts
   # that reflects the specified or implied shell to use for shell-code tests.
   #  - Set it to the shell specified via -s, if any.
   #  - Otherwise, use its present value, if non-empty.
   #  - Otherwise, default to '/bin/sh'.
-  if [ -n "$2" ]
-    then
-    TEST_SHELL="$2"
-  elif [ -z "$TEST_SHELL" ]
-    then
+  if [ -n "$shell_for_sh_tests" ]; then
+    TEST_SHELL="$shell_for_sh_tests"
+  elif [ -z "$TEST_SHELL" ]; then
     TEST_SHELL='/bin/sh'
   fi
 
-  recurse "$1" 0 "$2"  # test folder -- indentation level -- [shell to invoke test scripts with]
+  # 1 test folder
+  # 2 indentation level
+  # 3 shell to invoke test scripts with
+  # 4 urchin root
+  recurse "$1" 0 "$shell_for_sh_tests" "$root"
 
   finish=$(date +%s)
   elapsed=$(($finish - $start))
@@ -262,72 +362,7 @@ urchin_go() {
   fi
   rm -f "$logfile"
   test -z "$failed" || test "$failed" -eq '0'
-}
 
-urchin_molly_guard() {
-  {
-    echo
-    echo 'The name of the directory on which you are running urchin'
-    echo 'does not contain the word "test", so I am not running,'
-    echo 'in case that was an accident. Use the -f flag if you really'
-    echo 'want to run urchin on that directory.'
-    echo
-  } >&2
-  exit 1
-}
-
-single_process=false
-shell_for_sh_tests=
-force=false
-exit_on_fail=false
-tap_format=false
-while [ $# -gt 0 ]
-do
-    case "$1" in
-        -b|--series) single_process=true;;
-        -e) exit_on_fail=true;;
-        -f) force=true;;
-        -s)
-          shift
-          shell_for_sh_tests=$1
-          which "$shell_for_sh_tests" > /dev/null || {
-            echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2
-            urchin_help >&2
-            exit 3
-          } ;;
-        -t) tap_format=true;;
-        -h|--help) urchin_help
-          exit 0;;
-        -v) echo "$VERSION"
-            exit;;
-        -*) urchin_help >&2
-            exit 1;;
-        *)  break;;
-    esac
-    shift
-done
-
-# Verify argument for main stuff
-if [ "$#" != '1' ] || [ ! -d "$1" ]
-  then
-  [ -n "$1" ] && [ ! -d "$1" ] && echo "Not a directory: '$1'" >&2
-  echo "$USAGE" >&2
-  exit 3
-fi
-
-if $exit_on_fail && ! $single_process; then
-  echo 'You must specify -b/--series in order to use -e.' >&2
-  exit 3
-fi
-
-# Constants
-logfile=$(fullpath "$1")/.urchin.log
-stdout_file=$(fullpath "$1")/.urchin_stdout
-
-# Run or present the Molly guard.
-if fullpath "$1" | grep -Fi 'test' > /dev/null || $force
-  then
-  urchin_go "$1" "$shell_for_sh_tests"
 else
   urchin_molly_guard
 fi