diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..d066057 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,9 @@ +Authors +------- +David Jones +Michael Klement +Thomas Levine + +Maintainer +------- +Thomas Levine <_@thomaslevine.com> diff --git a/HISTORY b/HISTORY index a9bf61a..172e137 100644 --- a/HISTORY +++ b/HISTORY @@ -1,5 +1,23 @@ HISTORY -------- +======= + +Version 0.0.6 +--------------------- + +* Produce TAP output with the -t flag. +* Add a + sign in front of directories in the normal output so that they + line up with non-directories. +* Display skipped tests in the normal output and in the TAP output. +* Correct some things in the documentation. +* Rearrange things in the documentation to be more clear. +* Pass the -e flag to exit urchin if any single test fails. +* Remove the undocumented, experimental -x flag now that shall exists. +* Display version number with the -v flag. +* Document why Urchin is called "Urchin" + +These changes are made somewhat separately in the branches "exit-on-fail", +"remove-urchin-x", "tap", and "update-readme". They are rebased into one +branch, "tlevine-2016-02", for merging into "master". Version 0.0.5 --------------------- diff --git a/cross-shell-tests b/cross-shell-tests index 5f66f09..a169ad8 100755 --- a/cross-shell-tests +++ b/cross-shell-tests @@ -6,7 +6,7 @@ for shell in dash bash ksh zsh; do if which $shell > /dev/null 2> /dev/null; then echo echo Running urchin tests in $shell - $shell urchin tests | tail -n 3 + $shell urchin -s $shell tests | tail -n 4 else echo echo Skipping $shell because it is not in the PATH diff --git a/readme.md b/readme.md index caeb6a6..722c6e5 100644 --- a/readme.md +++ b/readme.md @@ -4,9 +4,13 @@ / /_/ / / / /__/ / / / / / / / \__,_/_/ \___/_/ /_/_/_/ /_/ -Urchin is a test framework for shell. It is implemented in -portable /bin/sh and should work on GNU/Linux, Mac OS X, and -other Unix platforms. +Urchin is a file-based test harness, normally used for testing shell programs. +It is written in portable shell and should thus work on GNU/Linux, BSD +(including Mac OS X), and other Unix-like platforms. + +Urchin is called "Urchin" because +[sea urchins](https://en.wikipedia.org/wiki/Sea_urchin) +have shells called "tests". ## Try it out Urchin's tests are written in Urchin, so you can run them to see what Urchin @@ -27,14 +31,15 @@ run this: cd urchin ./cross-shell-tests -## Globally -Download Urchin like so (as root) (or use npm, below): +## Install +Urchin is contained in a single file, so you can install it by copying it to a +directory in your `PATH`. For example, you can run the following as root. cd /usr/local/bin wget https://raw.github.com/tlevine/urchin/master/urchin chmod +x urchin -Can be installed with npm too: +Urchin can be installed with npm too. npm install -g urchin @@ -86,15 +91,14 @@ Files are only run if they are executable, and files beginning with `.` are ignored. Thus, fixtures and libraries can be included sloppily within the test directory tree. The test passes if the file exits 0; otherwise, it fails. -In case you care about the order in which your tests execute, consider that +Tests files and subdirectories are run in ASCIIbetical order within each +directory; that is, urchin looks for files within a directory in the following manner. for file in *; do do_something_with_test_file $file done -Tests within a directory are executed in whatever order `*` returns. - ### Writing cross-shell compatibility tests for testing shell code While you could write your test scripts to explicitly invoke the functionality @@ -106,24 +110,20 @@ The specific approach depends on your test scenario: * (b) Your scripts _source_ scripts containing portable shell code. #### (a) Cross-shell tests with test scripts that _invoke_ shell scripts +Urchin sets the `TEST_SHELL` environment variable so that you may change the +shell with which your tests call other shell programs. To run your test +scripts in multiple shells you must call `$TEST_SHELL` in your tests and then +run urchin with the appropriate option. -First, consider using [shall](https://github.com/mklement0/shall). - - #!/usr/bin/env shall - echo This is a test file. - -Alternatively, you can use urchin's built-in recognition of the -`TEST_SHELL` environment variable. In your test scripts, invoke the shell scripts to test via the shell specified in environment variable `TEST_SHELL` rather than directly; e.g.: `$TEST_SHELL ../foo bar` (rather than just `../foo bar`). -Note that if you alsow want your test scripts to work when run directly, -outside of Urchin, be sure to target scripts that happen to be in the -current directory with prefix `./`; e.g., `$TEST_SHELL ./baz` -(rather than `$TEST_SHELL baz`). -Then, on invocation of Urchin, prepend a definition of environment variable -`TEST_SHELL` specifying the shell to test with, e.g.: `TEST_SHELL=zsh urchin ./tests`. +On invocation of Urchin, prepend a definition of environment variable +`TEST_SHELL` specifying the shell to test with, e.g., + + TEST_SHELL=zsh urchin ./tests + To test with multiple shells in sequence, use something like: for shell in sh bash ksh zsh; do @@ -131,14 +131,20 @@ To test with multiple shells in sequence, use something like: done If `TEST_SHELL` has no value, Urchin defines it as `/bin/sh`, so the test -scripts can rely on `$TEST_SHELL` always containing a value. +scripts can rely on `$TEST_SHELL` always containing a value when Urchin runs +them. + +That said, we still recommand that you account for the possibility that +`$TEST_SHELL` does not contain a value so that you may run your test scripts +without Urchin. Supporting this case is very simple; when you invoke scripts +that happen to be in the current directory, be sure to use the prefix `./`, +e.g., `$TEST_SHELL ./baz` rather than `$TEST_SHELL baz`. #### (b) Cross-shell tests with test scripts that _source_ shell scripts - If you _source_ shell code in your test scripts, it is the test scripts themselves that must be run with the shell specified. -To that end, Urchin supports the `-s ` option, which instructs +Urchin supports the `-s ` option, which instructs Urchin to invoke the test scripts with the specified shell; e.g., `-s bash`. (In addition, Urchin sets environment variable `TEST_SHELL` to the specified shell.) @@ -154,21 +160,12 @@ To test with multiple shells in sequence, use something like: urchin -s $shell ./tests done - ## Alternatives to Urchin Alternatives to Urchin are discussed in [this blog post](https://blog.scraperwiki.com/2012/12/how-to-test-shell-scripts/). diff --git a/tests/.die-on-fail/1 should run. b/tests/.die-on-fail/1 should run. new file mode 100755 index 0000000..e69de29 diff --git a/tests/.die-on-fail/2 should run. b/tests/.die-on-fail/2 should run. new file mode 100755 index 0000000..c508d53 --- /dev/null +++ b/tests/.die-on-fail/2 should run. @@ -0,0 +1 @@ +false diff --git a/tests/.die-on-fail/3 should not run. b/tests/.die-on-fail/3 should not run. new file mode 100755 index 0000000..c508d53 --- /dev/null +++ b/tests/.die-on-fail/3 should not run. @@ -0,0 +1 @@ +false diff --git a/tests/.die-on-fail/4 should not run. b/tests/.die-on-fail/4 should not run. new file mode 100755 index 0000000..c508d53 --- /dev/null +++ b/tests/.die-on-fail/4 should not run. @@ -0,0 +1 @@ +false diff --git a/tests/Cross-shell test support/.test-run-by-specified-shell/With -s, a non-sh test script should be invoked directly. b/tests/Cross-shell test support/.test-run-by-specified-shell/With -s, a non-sh test script should be invoked directly. index 2dd5303..358713c 100755 --- a/tests/Cross-shell test support/.test-run-by-specified-shell/With -s, a non-sh test script should be invoked directly. +++ b/tests/Cross-shell test support/.test-run-by-specified-shell/With -s, a non-sh test script should be invoked directly. @@ -1,5 +1,3 @@ -#!/usr/bin/awk -f - -# This script will only succeed if it is indeed processed by awk. - -BEGIN { print "ok" } +#!/usr/bin/env true +true will processed the contents of this script, but that +means that nothing will happen and the script will exit 0 diff --git a/tests/Cross-shell test support/The -s option should invoke tests with the specified shell. b/tests/Cross-shell test support/The -s option should invoke tests with the specified shell. index 62e1264..00a74ac 100755 --- a/tests/Cross-shell test support/The -s option should invoke tests with the specified shell. +++ b/tests/Cross-shell test support/The -s option should invoke tests with the specified shell. @@ -3,6 +3,5 @@ # Tests the `-s option, which invokes shebang-less and sh-shebang-line test scripts with the specified shell (for testing *sourced* shell code). which bash >/dev/null || { echo "Cannot test -s option: bash cannot be located." >&2; exit 2; } -which /usr/bin/awk >/dev/null || { echo "Cannot test -s option: /usr/bin/awk not found." >&2; exit 2; } ../../urchin -s bash ./.test-run-by-specified-shell diff --git a/tests/Exit on fail if -e is passed. b/tests/Exit on fail if -e is passed. new file mode 100755 index 0000000..d9c3d22 --- /dev/null +++ b/tests/Exit on fail if -e is passed. @@ -0,0 +1,11 @@ +tmp=$(mktemp) +../urchin -e -f ./.die-on-fail > $tmp +result=$? + +grep '1 should run.' $tmp +grep '2 should run.' $tmp +grep -v '3 should not run.' $tmp +grep -v '4 should not run.' $tmp + +rm $tmp +exit $result diff --git a/tests/Print version on -v. b/tests/Print version on -v. new file mode 100755 index 0000000..6a21714 --- /dev/null +++ b/tests/Print version on -v. @@ -0,0 +1,2 @@ +#!/bin/sh +../urchin -v | grep '[0-9.]\{3,\}' diff --git a/tests/TAP/.expected-output b/tests/TAP/.expected-output new file mode 100644 index 0000000..69364fc --- /dev/null +++ b/tests/TAP/.expected-output @@ -0,0 +1,10 @@ +# Begin - .testsuite/ +not ok 1 - a +# ------------ Begin output ------------ +# This is stdout from a. +# ------------ End output ------------ +ok 2 - b +ok 3 - # SKIP c +# End - .testsuite/ +# Took 0 seconds. +1..3 diff --git a/tests/TAP/.testsuite/a b/tests/TAP/.testsuite/a new file mode 100755 index 0000000..c68153e --- /dev/null +++ b/tests/TAP/.testsuite/a @@ -0,0 +1,4 @@ +#!/bin/sh +echo This is stderr from a. > /dev/stderr +echo This is stdout from a. > /dev/stdout +false diff --git a/tests/TAP/.testsuite/b b/tests/TAP/.testsuite/b new file mode 100755 index 0000000..4dd2598 --- /dev/null +++ b/tests/TAP/.testsuite/b @@ -0,0 +1,4 @@ +#!/bin/sh +echo This is stderr from b. > /dev/stderr +echo This is stdout from b. > /dev/stdout +true diff --git a/tests/TAP/.testsuite/c b/tests/TAP/.testsuite/c new file mode 100644 index 0000000..619220c --- /dev/null +++ b/tests/TAP/.testsuite/c @@ -0,0 +1 @@ +This should not be run. diff --git a/tests/TAP/Running with -t should produce TAP output. b/tests/TAP/Running with -t should produce TAP output. new file mode 100755 index 0000000..32912fa --- /dev/null +++ b/tests/TAP/Running with -t should produce TAP output. @@ -0,0 +1,4 @@ +tmp=$(mktemp) + +../../urchin -t .testsuite/ | sed 1d > $tmp +diff $tmp .expected-output diff --git a/tests/urchin -x should start the $TEST_SHELL. b/tests/urchin -x should start the $TEST_SHELL. deleted file mode 100755 index e137df2..0000000 --- a/tests/urchin -x should start the $TEST_SHELL. +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -test c = $(../urchin -x .print-arg-3 a 'b b b b' c d e) diff --git a/urchin b/urchin index 60b7d84..aa10e5d 100755 --- a/urchin +++ b/urchin @@ -4,6 +4,9 @@ # which breaks fullpath(). unset CDPATH +# Urchin version number +VERSION=0.0.6 + fullpath() { ( cd -- "$1" @@ -30,9 +33,16 @@ recurse() { if [ -d "$potential_test" ] then - ( + + if $tap_format; then + indent $indent_level | sed 's/ /#/g' + echo "# Begin - ${potential_test}" + else indent $indent_level - echo " ${potential_test}" + echo "+ ${potential_test}" + fi + + ( cd -- "$potential_test" [ -f setup_dir ] && [ -x setup_dir ] && ./setup_dir >> "$stdout_file" @@ -47,44 +57,92 @@ recurse() { # $2 instead of $indent_level so it doesn't clash recurse "${test}" $(( $2 + 1 )) "$shell_for_sh_tests" + exit_code=$? + + if $exit_on_fail && test $exit_code -ne 0; then + [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file" + [ -f teardown_dir ] && [ -x teardown_dir ] && ./teardown_dir >> "$stdout_file" + return 1 + fi [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file" done [ -f teardown_dir ] && [ -x teardown_dir ] && ./teardown_dir >> "$stdout_file" - echo ) - elif [ -x "$potential_test" ] - then - - [ -f setup ] && [ -x setup ] && ./setup >> "$stdout_file" - - # Run the test - if [ -n "$shell_for_sh_tests" ] && has_sh_or_no_shebang_line ./"$potential_test" - then - TEST_SHELL="$TEST_SHELL" "$shell_for_sh_tests" ./"$potential_test" > "$stdout_file" 2>&1 + if $tap_format; then + indent $indent_level | sed 's/ /#/g' + echo "# End - ${potential_test}" else - TEST_SHELL="$TEST_SHELL" ./"$potential_test" > "$stdout_file" 2>&1 + echo fi - exit_code="$?" - - - [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file" - - indent $indent_level - if [ $exit_code -eq 0 ] + else + if [ -x "$potential_test" ] then - # On success, print a green '✓' - printf '\033[32m✓ \033[0m' - printf '%s\n' "${potential_test}" - printf '%s\n' "${potential_test} passed" >> "$logfile" + [ -f setup ] && [ -x setup ] && ./setup >> "$stdout_file" + + # Run the test + if [ -n "$shell_for_sh_tests" ] && has_sh_or_no_shebang_line ./"$potential_test" + then + TEST_SHELL="$TEST_SHELL" "$shell_for_sh_tests" ./"$potential_test" > "$stdout_file" 2>&1 + else + TEST_SHELL="$TEST_SHELL" ./"$potential_test" > "$stdout_file" 2>&1 + fi + exit_code="$?" + + [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file" + if [ $exit_code -eq 0 ]; then + result=success + else + result=fail + fi else - # On fail, print a red '✗' - printf '\033[31m✗ \033[0m' - printf '%s\n' "${potential_test}" - printf '%s\n' "${potential_test} failed" >> "$logfile" - printf '\033[31m' # Print output captured from failed test in red. - cat "$stdout_file" - printf '\033[0m' + result=skip + fi + + echo "${result}" >> "$logfile" + if $tap_format; then + n=$(grep -ce '^\(success\|fail\|skip\)' "$logfile") + + if [ "$result" == fail ]; then + not='not ' + else + not='' + fi + if [ "$result" == skip ]; then + skip='# SKIP ' + else + skip='' + fi + echo "${not}ok $n - ${skip}${potential_test}" + if [ "$result" == fail ]; then + echo '# ------------ Begin output ------------' + sed 's/^/# /' "$stdout_file" + echo '# ------------ End output ------------' + fi + else + indent $indent_level + case "$result" in + success) + # On success, print a green '✓' + printf '\033[32m✓ \033[0m' + printf '%s\n' "${potential_test}" + ;; + fail) + # On fail, print a red '✗' + printf '\033[31m✗ \033[0m' + printf '%s\n' "${potential_test}" + printf '\033[31m' # Print output captured from failed test in red. + cat "$stdout_file" + printf '\033[0m' + ;; + skip) + printf ' %s\n' "${potential_test}" + ;; + esac + fi + + if $exit_on_fail && test 0 -ne $exit_code; then + return 1 fi fi [ $indent_level -eq 0 ] && rm "$stdout_file" @@ -105,17 +163,18 @@ $USAGE -s Invoke test scripts that either have no shebang line at all or have shebang line "#!/bin/sh" with the specified shell. +-e Stop running if any single test fails. This is helpful if you want + to use Urchin to run things other than tests, such as a set of + configuration scripts. -f Force running even if the test directory's name does not contain the word "test". --h This help. +-t Format output in Test Anything Protocol (TAP) +-h, --help This help. +-v Display the version number. Go to https://github.com/tlevine/urchin for documentation on writing tests. EOF - # [Experimental -x option left undocumented for now.] - # -x [Experimental; not meant for direct invocation, but for use in - # the shebang line of test scripts] - # Run with "\$TEST_SHELL", falling back on /bin/sh. } plural () { @@ -131,7 +190,11 @@ plural () { } urchin_go() { - echo Running tests at $(date +%Y-%m-%dT%H:%M:%S) | tee "$logfile" + rm -f "$logfile" + if "$tap_format"; then + printf \#\ + fi + echo Running tests at $(date +%Y-%m-%dT%H:%M:%S) start=$(date +%s) # Determine the environment variable to define for test scripts @@ -151,13 +214,23 @@ urchin_go() { finish=$(date +%s) elapsed=$(($finish - $start)) - echo "Done, took $elapsed $(plural second $elapsed)." - set -- $(grep -e 'passed$' "$logfile"|wc -l) $(grep -e 'failed$' "$logfile"|wc -l) - printf '%s\n' "$1 $(plural test "$1") passed." - [ $2 -gt 0 ] && printf '\033[31m' || printf '\033[32m' # If tests failed, print the message in red, otherwise in green. - printf '%s\n' "$2 $(plural test "$2") failed." - printf '\033[m' - return "$2" + + passed=$(grep -c '^success' "$logfile") + failed=$(grep -c '^fail' "$logfile") + skipped=$(grep -c '^skip' "$logfile") + if $tap_format; then + echo "# Took $elapsed $(plural second $elapsed)." + echo 1..$(($passed + $failed + $skipped)) + else + echo "Done, took $elapsed $(plural second $elapsed)." + printf '%s\n' "$passed $(plural test "$passed") passed." + printf '%s\n' "$skipped $(plural test "$skipped") skipped." + [ $failed -gt 0 ] && printf '\033[31m' || printf '\033[32m' # If tests failed, print the message in red, otherwise in green. + printf '%s\n' "$failed $(plural test "$failed") failed." + printf '\033[m' + fi + rm -f "$logfile" + test -z "$failed" || test "$failed" -eq '0' } urchin_molly_guard() { @@ -174,22 +247,23 @@ urchin_molly_guard() { shell_for_sh_tests= force=false +exit_on_fail=false +tap_format=false while [ $# -gt 0 ] do case "$1" in + -e) exit_on_fail=true;; -f) force=true;; -s) shift shell_for_sh_tests=$1 which "$shell_for_sh_tests" >/dev/null || { echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2; urchin_help >&2; exit 2; } ;; - -x) # [EXPERIMENTAL; UNDOCUMENTED FOR NOW] `urchin -x ` in a test script's shebang line is equivalent to invoking that script with `"$TEST_SHELL" ` - shift - urchinsh=${TEST_SHELL:-/bin/sh} - "$urchinsh" "$@" - exit $?;; + -t) tap_format=true;; -h|--help) urchin_help exit 0;; + -v) echo "$VERSION" + exit;; -*) urchin_help >&2 exit 1;; *) break;;