introduction paragraph

* move shall reference to the right place
* further explain cross-shell testing methods

.gitignore

@@ -1 +0,0 @@
-.urchin_stdout

HISTORY

@@ -1,18 +1,31 @@
-History
-======
+HISTORY
+-------
 
+Version 0.0.5
+---------------------
+* urchin now unsets `CDPATH`.
+* The documentation for `urchin -x` was removed because it was confusing.
 
-Not done
-------
-We made the colors brighter.
+Version 0.0.4
+---------------------
+* Switch urchin -x to urchin -sh and fix some problems with it
+* Documentation
 
-We allowed for multiple tests to be written in a file
+Version 0.0.3
+---------------------
+General tidying
 
-We added flags for verbosity and for Molly guards
+Run with different shells in three ways
 
-Done
------
+* urchin -s
+* $TEST_SHELL variable with $TEST_SHELL
+* $TEST_SHELL variable with urchin -sh
 
-We adjusted the input parameters so it is harder to accidentally run all executible files in your home directory.
+Set NULL_GLOB so zsh doesn't print a warning.
+
+Before version 0.0.3
+----------------------
+We adjusted the input parameters so it is harder to accidentally run all executable files in you
+r home directory.
 
 We added directory-based indents.

alternatives

@@ -0,0 +1,2 @@
+Totally different syntax and similar features, plus TAP output
+https://github.com/sstephenson/bats

cross-shell-tests

@@ -6,7 +6,7 @@ for shell in dash bash ksh zsh; do
   if which $shell > /dev/null 2> /dev/null; then
     echo
     echo Running urchin tests in $shell
-    $shell urchin tests | tail -n 2
+    $shell urchin tests | tail -n 3
   else
     echo
     echo Skipping $shell because it is not in the PATH

package.json

@@ -1,26 +1,32 @@
 {
   "name": "urchin",
-  "version": "0.0.2",
+  "version": "0.0.5",
   "description": "Test framework for shell",
   "main": "urchin",
   "directories": {
     "test": "tests"
   },
   "scripts": {
-    "test": "./urchin tests"
+    "test": "export PATH=\"$PWD:$PATH\" && urchin tests"
   },
   "bin": "./urchin",
   "repository": {
     "type": "git",
-    "url": "git://github.com/scraperwiki/urchin.git"
+    "url": "git://github.com/tlevine/urchin.git"
   },
   "keywords": [
     "shell",
     "test",
     "testing"
   ],
-  "author": "ScraperWiki Limited <feedback@scraperwiki.com>",
-  "license": "BSD",
-  "readmeFilename": "readme.md",
-  "gitHead": "015052dcfd7152a0df037bef1bc5452e0c956695"
+  "author": "Thomas Levine <_@thomaslevine.com>",
+  "contributors": [
+    {"name": "Thomas Levine", "email": "_@thomaslevine.com"},
+    {"name": "David Jones", "email": "drj@pobox.com"},
+    {"name": "Francis Irving", "email": "francis@flourish.org"},
+    {"name": "Zarino Zappia", "email": "mail@zarino.co.uk"},
+    {"name": "Tom Mortimer-Jones", "email": "tom@morty.co.uk"},
+    {"name": "Michael Klement", "email": "mklement0@gmail.com"}
+  ],
+  "license": "BSD"
 }

readme.md

@@ -4,37 +4,38 @@
     / /_/ / /  / /__/ / / / / / / /
     \__,_/_/   \___/_/ /_/_/_/ /_/ 
 
-Urchin is a test framework for shell. It is implemented in
-portable /bin/sh and should work on GNU/Linux, Mac OS X, and
-other Unix platforms.
+Urchin is a file-based test harness, normally used for testing shell programs.
+It is written in portable shell and should thus work on GNU/Linux, BSD
+(including Mac OS X), and other Unix-like platforms.
 
 ## Try it out
 Urchin's tests are written in Urchin, so you can run them to see what Urchin
 is like. Clone the repository
 
-    git clone git://github.com/scraperwiki/urchin.git
+    git clone git://github.com/tlevine/urchin.git
 
 Run the tests
 
     cd urchin
     ./urchin tests
 
-The above command will run the tests in your systems default
+The above command will run the tests in your system's default
 shell, /bin/sh (on recent Ubuntu this is dash, but it could be
-ksh or bash on other systems); to test cross-shell compatibility,
+ksh or bash on other systems); to test urchin's cross-shell compatibility,
 run this:
 
     cd urchin
     ./cross-shell-tests
 
-## Globally
-Download Urchin like so (as root) (or use npm, below):
+## Install
+Urchin is contained in a single file, so you can install it by copying it to a
+directory in your `PATH`. For example, you can run the following as root.
 
     cd /usr/local/bin
-    wget https://raw.github.com/scraperwiki/urchin/master/urchin
+    wget https://raw.github.com/tlevine/urchin/master/urchin
     chmod +x urchin
 
-Can be installed with npm too:
+Urchin can be installed with npm too.
 
     npm install -g urchin
 
@@ -42,6 +43,8 @@ Now you can run it.
 
     urchin <test directory>
 
+Run `urchin -h` to get command-line help.
+
 ## Writing tests
 Make a root directory for your tests. Inside it, put executable files that
 exit `0` on success and something else on fail. Non-executable files and hidden
@@ -83,3 +86,88 @@ within the particular directory, and the `teardown` file is run right after.
 Files are only run if they are executable, and files beginning with `.` are
 ignored. Thus, fixtures and libraries can be included sloppily within the test
 directory tree. The test passes if the file exits 0; otherwise, it fails.
+
+Tests files and subdirectories are run in ASCIIbetical order within each
+directory; that is,
+urchin looks for files within a directory in the following manner.
+
+    for file in *; do
+      do_something_with_test_file $file
+    done
+
+### Writing cross-shell compatibility tests for testing shell code
+
+While you could write your test scripts to explicitly invoke the functionality
+to test with various shells, Urchin facilitates a more flexible approach.
+
+The specific approach depends on your test scenario:
+
+* (a) Your test scripts _invoke_ scripts containing portable shell code.
+* (b) Your scripts _source_ scripts containing portable shell code.
+
+#### (a) Cross-shell tests with test scripts that _invoke_ shell scripts
+Urchin sets the `TEST_SHELL` environment variable so that you may change the
+shell with which your tests call other shell programs. To run your test
+scripts in multiple shells you must call `$TEST_SHELL` in your tests and then
+run urchin with the appropriate option.
+
+In your test scripts, invoke the shell scripts to test via the shell
+specified in environment variable `TEST_SHELL` rather than directly;
+e.g.: `$TEST_SHELL ../foo bar` (rather than just `../foo bar`).  
+
+On invocation of Urchin, prepend a definition of environment variable
+`TEST_SHELL` specifying the shell to test with, e.g.,
+
+    TEST_SHELL=zsh urchin ./tests
+
+To test with multiple shells in sequence, use something like:
+
+    for shell in sh bash ksh zsh; do
+      TEST_SHELL=$shell urchin ./tests
+    done
+
+If `TEST_SHELL` has no value, Urchin defines it as `/bin/sh`, so the test
+scripts can rely on `$TEST_SHELL` always containing a value when Urchin runs
+them.
+
+That said, we still recommand that you account for the possibility that
+`$TEST_SHELL` does not contain a value so that you may run your test scripts
+without Urchin. Supporting this case is very simple; when you invoke scripts
+that happen to be in the current directory, be sure to use the prefix `./`,
+e.g., `$TEST_SHELL ./baz` rather than `$TEST_SHELL baz`.
+
+#### (b) Cross-shell tests with test scripts that _source_ shell scripts
+If you _source_ shell code in your test scripts, it is the test scripts
+themselves that must be run with the shell specified.
+
+Urchin supports the `-s <shell>` option, which instructs
+Urchin to invoke the test scripts with the specified shell; e.g., `-s bash`.  
+(In addition, Urchin sets environment variable `TEST_SHELL` to the specified
+shell.)
+
+Note that only test scripts that either have no shebang line at all or
+have shebang line `#!/bin/sh` are invoked with the specified shell.
+This allows non-shell test scripts or test scripts for specific
+shells to coexist with those whose invocation should be controlled by `-s`.
+
+To test with multiple shells in sequence, use something like:
+
+    for shell in sh bash ksh zsh; do
+      urchin -s $shell ./tests
+    done
+
+Also consider using [shall](https://github.com/mklement0/shall).
+It does something similar, but the interface may be more intuitive.
+
+    #!/usr/bin/env shall
+    echo This is a test file.
+
+## Alternatives to Urchin
+Alternatives to Urchin are discussed in
+[this blog post](https://blog.scraperwiki.com/2012/12/how-to-test-shell-scripts/).
+
+## Ideas for new features
+
+* Support [Nagios plugins](https://nagios-plugins.org/doc/guidelines.html)
+* Stop running if a test fails so one can use Urchin as a
+    [setup framework](https://github.com/tlevine/urchin/issues/16).

tests/.gitignore

@@ -1 +1,2 @@
 .urchin.log
+.urchin_stdout

tests/.print-arg-3

@@ -0,0 +1 @@
+echo $3

tests/A nonempty CDPATH should not break urchin.

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+cd ..
+export CDPATH=$PWD
+./urchin -f 'tests/urchin exit code' >/dev/null
+

tests/Command-line help contents/The -s option should be documented.

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+../../urchin -h | grep -- -s

tests/Command-line help contents/XSD output (-xsd flag) should be documented.

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-../../urchin -h|grep -i xsd

tests/Cross-shell test support/.test-TEST_SHELL-passed-through/$TEST_SHELL should be bash.

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# Assuming that urchin was invoked with `TEST_SHELL=bash urchin ...`, $TEST_SHELL should contain 'bash'.
+
+echo "\$TEST_SHELL: $TEST_SHELL"
+
+[ "$TEST_SHELL" = 'bash' ]

tests/Cross-shell test support/.test-TEST_SHELL-undefined_or_empty/Invoking a command with $TEST_SHELL should work

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Invoke a simple test command with $TEST_SHELL as the executable.
+[ "$($TEST_SHELL -c 'echo $0')" = "$TEST_SHELL" ]

tests/Cross-shell test support/.test-run-by-specified-shell/With -s, a non-sh test script should be invoked directly.

@@ -0,0 +1,5 @@
+#!/usr/bin/awk -f
+
+# This script will only succeed if it is indeed processed by awk.
+
+BEGIN { print "ok" }

tests/Cross-shell test support/.test-run-by-specified-shell/With -s, a sh test script should be invoked with the specified shell.

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# Assuming that urchin was invoked with `-s bash`, this script should be being run with bash.
+
+this_shell=$(ps -o comm= -p $$ && :)
+
+echo "Running shell: $this_shell"
+
+[  "$this_shell" = 'bash' ]
+

tests/Cross-shell test support/.test-run-by-specified-shell/With -s, a shebang-less test script should be invoked with the specified shell.

@@ -0,0 +1,10 @@
+# By design, this file has no shebang line.
+
+# Assuming that urchin was invoked with `-s bash`, this script should be being run with bash.
+
+this_shell=$(ps -o comm= -p $$ && :)
+
+echo "Running shell: $this_shell"
+
+[  "$this_shell" = 'bash' ]
+

tests/Cross-shell test support/.test-run-by-specified-shell/With -s, the TEST_SHELL environment variable should be set to the specified shell too.

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# Assuming that urchin was invoked with `-s bash`, $TEST_SHELL should contain 'bash'.
+
+echo "Running shell: $(ps -o comm= -p $$ && :)"
+
+echo "\$TEST_SHELL: $TEST_SHELL"
+
+[  "$TEST_SHELL" = 'bash' ]

tests/Cross-shell test support/Environment variable TEST_SHELL should be passed through.

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# Tests support for either passing through or defining a default value for environment variable TEST_SHELL.
+# (for test scripts that want to invoke shell scripts with a specified shell).
+
+which bash 2>/dev/null || { echo "Cannot test -s option: bash cannot be located." >&2; exit 1; }
+
+# Test if $TEST_SHELL, when placed in urchin's environment, is passed through to the test scripts.
+TEST_SHELL=bash ../../urchin ./.test-TEST_SHELL-passed-through

tests/Cross-shell test support/Environment variable TEST_SHELL should default to sh if empty.

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Tests support for either passing through or defining a default value for environment variable TEST_SHELL.
+# (for test scripts that want to invoke shell scripts with a specified shell).
+
+# Test if $TEST_SHELL - if *defined, but empty* - is exported with value '/bin/sh' by urchin
+# and thus has that value inside the scripts.
+TEST_SHELL= ../../urchin ./.test-TEST_SHELL-undefined_or_empty

tests/Cross-shell test support/Environment variable TEST_SHELL should default to sh if unset.

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# Tests support for either passing through or defining a default value for environment variable TEST_SHELL.
+# (for test scripts that want to invoke shell scripts with a specified shell).
+
+# Test if $TEST_SHELL - if *undefined* - is exported with value '/bin/sh' by urchin
+# and thus has that value inside test scripts.
+unset -v TEST_SHELL
+../../urchin ./.test-TEST_SHELL-undefined_or_empty

tests/Cross-shell test support/The -s option should invoke tests with the specified shell.

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Tests the `-s <shell> option, which invokes shebang-less and sh-shebang-line test scripts with the specified shell (for testing *sourced* shell code).
+
+which bash >/dev/null || { echo "Cannot test -s option: bash cannot be located." >&2; exit 2; }
+which /usr/bin/awk >/dev/null || { echo "Cannot test -s option: /usr/bin/awk not found." >&2; exit 2; }
+
+../../urchin -s bash ./.test-run-by-specified-shell

tests/Urchin should do a sanity check so that directories like the user home are not specified as tests./Urchin should warn if it is run on something that does not look like a test.

@@ -1,4 +1,3 @@
 #!/bin/sh
 
-mkdir /tmp/urchintmp
-! ../../urchin /tmp/urchintmp
+! ../../urchin ./.chainsaw

tests/XSD output should work./.fixture/fail

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exit 1

tests/XSD output should work./.fixture/pass

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exit

tests/XSD output should work./XSD output should produce XML.

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-../../urchin .fixture | grep 'http://www.w3.org/2001/XMLSchema'

tests/setup

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-export variable_set_in_setup='value'

tests/urchin -x should start the $TEST_SHELL.

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+test c = $(../urchin -x .print-arg-3 a 'b b b b' c d e)

tests/urchin exit code/non zero exit code when a test fails

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# Borrow this test which has a failure in it (3 in fact), and
+# run it using urchin:
+../../urchin "../Counts should be kept of successes and failures./.test" > /dev/null
+
+# Now invert the result, we want _this_ test to fail if the exit
+# code was 0.
+case $? in
+  0) exit 8;;
+  *) exit 0;;
+esac
+

tests/urchin exit code/zero exit code when tests all pass

@@ -0,0 +1 @@
+../../urchin .test > /dev/null

urchin

@@ -1,5 +1,9 @@
 #!/bin/sh
 
+# Make sure that CDPATH isn't set, as it causes `cd` to behave unpredictably - notably, it can produce output,
+# which breaks fullpath().
+unset CDPATH
+
 fullpath() {
   (
     cd -- "$1"
@@ -15,13 +19,14 @@ indent() {
 recurse() {
   potential_test="$1"
   indent_level="$2"
+  shell_for_sh_tests="$3"
 
   [ "$potential_test" = 'setup_dir' ] && return
   [ "$potential_test" = 'teardown_dir' ] && return
   [ "$potential_test" = 'setup' ] && return
   [ "$potential_test" = 'teardown' ] && return
 
-  echo > "$stdout_file"
+  [ $indent_level -eq 0 ] && : > "$stdout_file"
 
   if [ -d "$potential_test" ]
     then
@@ -30,12 +35,18 @@ recurse() {
       echo "  ${potential_test}"
       cd -- "$potential_test"
       [ -f setup_dir ] && [ -x setup_dir ] && ./setup_dir >> "$stdout_file"
+
+      if [ -n "$ZSH_VERSION" ]; then
+        # avoid "no matches found: *" error when directories are empty
+        setopt NULL_GLOB
+      fi
+
       for test in *
         do
         [ -f setup ] && [ -x setup ] && ./setup >> "$stdout_file"
 
         # $2 instead of $indent_level so it doesn't clash
-        recurse "${test}" $(( $2 + 1 ))
+        recurse "${test}" $(( $2 + 1 )) "$shell_for_sh_tests"
 
         [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
       done
@@ -48,15 +59,21 @@ recurse() {
     [ -f setup ] && [ -x setup ] && ./setup >> "$stdout_file"
 
     # Run the test
-    ./"$potential_test" > "$stdout_file" 2>&1
+    if [ -n "$shell_for_sh_tests" ] && has_sh_or_no_shebang_line ./"$potential_test"
+      then    
+      TEST_SHELL="$TEST_SHELL" "$shell_for_sh_tests" ./"$potential_test" > "$stdout_file" 2>&1
+    else
+      TEST_SHELL="$TEST_SHELL" ./"$potential_test" > "$stdout_file" 2>&1
+    fi
     exit_code="$?"
 
+
     [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
 
     indent $indent_level
-    if [ "$exit_code" = '0' ]
+    if [ $exit_code -eq 0 ]
       then
-      # On success, print a '✓'
+      # On success, print a green '✓'
       printf '\033[32m✓ \033[0m'
       printf '%s\n' "${potential_test}"
       printf '%s\n' "${potential_test} passed" >> "$logfile"
@@ -65,25 +82,40 @@ recurse() {
       printf '\033[31m✗ \033[0m'
       printf '%s\n' "${potential_test}"
       printf '%s\n' "${potential_test} failed" >> "$logfile"
+      printf '\033[31m' # Print output captured from failed test in red.
       cat "$stdout_file"
+      printf '\033[0m'
     fi
-    rm "$stdout_file"
   fi
+  [ $indent_level -eq 0 ] && rm "$stdout_file"
 }
 
-USAGE="usage: $0 <test directory>"
+has_sh_or_no_shebang_line() {
+  head -n 1 "$1" | grep -vqE '^#!' && return 0 # no shebang line at all
+  head -n 1 "$1" | grep -qE '^#![[:blank:]]*/bin/sh($|[[:blank:]])' && return 0  # shebang line is '#!/bin/sh' or legal variations thereof
+  return 1
+}
+
+USAGE="usage: $0 [<options>] <test directory>"
 
 urchin_help() {
-  echo
-  echo "$USAGE"
-  echo
-  echo '-f      Force urchin to run on directories not named "test".'
-  echo '-h      This help'
-  echo
-  echo '--xsd   Output xUnit XML schema for an integration server.'
-  echo
-  echo 'Go to http://www.urchin.sh for documentation on writing tests.'
-  echo
+  cat <<EOF
+
+$USAGE
+
+-s <shell>  Invoke test scripts that either have no shebang line at all or
+            have shebang line "#!/bin/sh" with the specified shell.
+-f          Force running even if the test directory's name does not
+            contain the word "test".
+-h          This help.
+
+Go to https://github.com/tlevine/urchin for documentation on writing tests.
+
+EOF
+  # [Experimental -x option left undocumented for now.]
+  # -x          [Experimental; not meant for direct invocation, but for use in
+  #             the shebang line of test scripts]
+  #             Run with "\$TEST_SHELL", falling back on /bin/sh.
 }
 
 plural () {
@@ -99,61 +131,88 @@ plural () {
 }
 
 urchin_go() {
-  echo Running tests at $(date +%Y-%m-%dT%H:%M:%S)
+  echo Running tests at $(date +%Y-%m-%dT%H:%M:%S) | tee "$logfile"
   start=$(date +%s)
 
-  echo > "$logfile"
-  recurse "$1" 0
+  # Determine the environment variable to define for test scripts
+  # that reflects the specified or implied shell to use for shell-code tests.
+  #  - Set it to the shell specified via -s, if any.
+  #  - Otherwise, use its present value, if non-empty.
+  #  - Otherwise, default to '/bin/sh'.
+  if [ -n "$2" ]
+    then
+    TEST_SHELL="$2"
+  elif [ -z "$TEST_SHELL" ]
+    then
+    TEST_SHELL='/bin/sh'
+  fi
+
+  recurse "$1" 0 "$2"  # test folder -- indentation level -- [shell to invoke test scripts with]
 
   finish=$(date +%s)
   elapsed=$(($finish - $start))
-  echo Done, took $elapsed $(plural second $elapsed)
+  echo "Done, took $elapsed $(plural second $elapsed)."
   set -- $(grep -e 'passed$' "$logfile"|wc -l) $(grep -e 'failed$' "$logfile"|wc -l)
   printf '%s\n' "$1 $(plural test "$1") passed."
+  [ $2 -gt 0 ] && printf '\033[31m' || printf '\033[32m' # If tests failed, print the message in red, otherwise in green.
   printf '%s\n' "$2 $(plural test "$2") failed."
+  printf '\033[m'
+  return "$2"
 }
 
 urchin_molly_guard() {
-  echo
-  echo 'The directory on which you are running urchin is not'
-  echo 'called "test", so I am not running in case that'
-  echo 'was an accident. Use the -f flag if you really want'
-  echo 'to run urchin on that directory.'
-  echo
+  {
+    echo
+    echo 'The name of the directory on which you are running urchin'
+    echo 'does not contain the word "test", so I am not running,'
+    echo 'in case that was an accident. Use the -f flag if you really'
+    echo 'want to run urchin on that directory.'
+    echo
+  } >&2
   exit 1
 }
 
-FORCE=false
+shell_for_sh_tests=
+force=false
 while [ $# -gt 0 ]
 do
     case "$1" in
-        -f) FORCE=true;;
+        -f) force=true;;
+        -s)
+          shift
+          shell_for_sh_tests=$1
+          which "$shell_for_sh_tests" >/dev/null || { echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2; urchin_help >&2; exit 2; }
+          ;;
+        -x) # [EXPERIMENTAL; UNDOCUMENTED FOR NOW] `urchin -x <test-script>` in a test script's shebang line is equivalent to invoking that script with `"$TEST_SHELL" <test-script>`
+          shift
+          urchinsh=${TEST_SHELL:-/bin/sh}
+          "$urchinsh" "$@"
+          exit $?;;
         -h|--help) urchin_help
           exit 0;;
-#       --xsd) action=testsuite;;
-#       --) shift; break;;
-        -*) urchin_help 1>&2
+        -*) urchin_help >&2
             exit 1;;
         *)  break;;
     esac
     shift
 done
 
+# Verify argument for main stuff
+if [ "$#" != '1' ] || [ ! -d "$1" ]
+  then
+  [ -n "$1" ] && [ ! -d "$1" ] && echo "Not a directory: '$1'" >&2
+  echo "$USAGE" >&2
+  exit 2
+fi
+
 # Constants
 logfile=$(fullpath "$1")/.urchin.log
 stdout_file=$(fullpath "$1")/.urchin_stdout
 
-# Verify argument for main stuff
-if [ "$#" != '1' ] && [ ! -d "$1" ]
-  then
-  echo "$USAGE"
-  exit 1
-fi
-
 # Run or present the Molly guard.
-if basename "$(fullpath "$1")" | grep test > /dev/null || $FORCE
+if basename "$(fullpath "$1")" | grep -Fi 'test' > /dev/null || $force
   then
-  urchin_go "$1"
+  urchin_go "$1" "$shell_for_sh_tests"
 else
   urchin_molly_guard
 fi