Compare commits

...

21 Commits
master ... tap

Author SHA1 Message Date
Thomas Levine
5e3c918675 add + indent symbol so directories line up 2016-01-28 15:12:34 +00:00
Thomas Levine
f5f98c8c2f fix fixture to not expect stderr 2016-01-28 15:11:27 +00:00
Thomas Levine
ce175cc3df test stdout stderr for TAP 2016-01-28 14:55:11 +00:00
Thomas Levine
cabaaa7ba4 TAP test 2016-01-28 14:51:43 +00:00
Thomas Levine
a663085069 remove awk dependency in test suite
I'm on NixOS, so awk isn't in /usr/bin. I can search for it with
/usr/bin/env, but then I can't use -f in the shebang line.
2016-01-27 10:51:04 +00:00
Thomas Levine
6bb606a3cc test plan 2016-01-27 10:39:07 +00:00
Thomas Levine
1c93e9a5c2 logfile 2016-01-27 10:31:51 +00:00
Thomas Levine
508e695dc3 error on fail 2016-01-27 10:25:21 +00:00
Thomas Levine
a51d96631f print skip count at end 2016-01-27 10:24:28 +00:00
Thomas Levine
1cd9991587 print stdout for tap and not-tap 2016-01-27 10:17:33 +00:00
Thomas Levine
0f1c2848b4 tap indentation comments for directories 2016-01-27 10:14:22 +00:00
Thomas Levine
9d10e12633 assorted tap stuff 2016-01-27 09:48:30 +00:00
Thomas Levine
40979f6e18 more tap 2016-01-27 01:13:55 +00:00
Thomas Levine
33e158e8f7 more tap 2016-01-27 01:08:04 +00:00
Thomas Levine
7ecacad132 oops 2016-01-27 01:05:43 +00:00
Thomas Levine
c8df46014d test test successes better 2016-01-27 01:05:08 +00:00
Thomas Levine
de2da89169 convert indents to comments 2016-01-27 00:50:55 +00:00
Thomas Levine
a6d6730e74 start writing non-tap cases 2016-01-27 00:32:51 +00:00
Thomas Levine
d01e993041 call urchin -s in cross-shell tests 2016-01-25 14:00:47 +00:00
Thomas Levine
b5c6464eab remove "urchin -x" test 2016-01-25 13:57:14 +00:00
Thomas Levine
329fc27929 remove "urchin -x"
unnecessary now that shall exists
2016-01-25 13:35:54 +00:00
10 changed files with 129 additions and 58 deletions

View File

@ -6,7 +6,7 @@ for shell in dash bash ksh zsh; do
if which $shell > /dev/null 2> /dev/null; then if which $shell > /dev/null 2> /dev/null; then
echo echo
echo Running urchin tests in $shell echo Running urchin tests in $shell
$shell urchin tests | tail -n 3 $shell urchin -s $shell tests | tail -n 3
else else
echo echo
echo Skipping $shell because it is not in the PATH echo Skipping $shell because it is not in the PATH

View File

@ -1,5 +1,3 @@
#!/usr/bin/awk -f #!/usr/bin/env true
true will processed the contents of this script, but that
# This script will only succeed if it is indeed processed by awk. means that nothing will happen and the script will exit 0
BEGIN { print "ok" }

View File

@ -3,6 +3,5 @@
# Tests the `-s <shell> option, which invokes shebang-less and sh-shebang-line test scripts with the specified shell (for testing *sourced* shell code). # Tests the `-s <shell> option, which invokes shebang-less and sh-shebang-line test scripts with the specified shell (for testing *sourced* shell code).
which bash >/dev/null || { echo "Cannot test -s option: bash cannot be located." >&2; exit 2; } which bash >/dev/null || { echo "Cannot test -s option: bash cannot be located." >&2; exit 2; }
which /usr/bin/awk >/dev/null || { echo "Cannot test -s option: /usr/bin/awk not found." >&2; exit 2; }
../../urchin -s bash ./.test-run-by-specified-shell ../../urchin -s bash ./.test-run-by-specified-shell

View File

@ -0,0 +1,10 @@
# Begin - .testsuite/
not ok 1 - a
# ------------ Begin output ------------
# This is stdout from a.
# ------------ End output ------------
ok 2 - b
ok 3 - # SKIP c
# End - .testsuite/
# Took 0 seconds.
1..3

4
tests/TAP/.testsuite/a Executable file
View File

@ -0,0 +1,4 @@
#!/bin/sh
echo This is stderr from a. > /dev/stderr
echo This is stdout from a. > /dev/stdout
false

4
tests/TAP/.testsuite/b Executable file
View File

@ -0,0 +1,4 @@
#!/bin/sh
echo This is stderr from b. > /dev/stderr
echo This is stdout from b. > /dev/stdout
true

1
tests/TAP/.testsuite/c Normal file
View File

@ -0,0 +1 @@
This should not be run.

View File

@ -0,0 +1,4 @@
tmp=$(mktemp)
../../urchin -t .testsuite/ | sed 1d > $tmp
diff $tmp .expected-output

View File

@ -1,3 +0,0 @@
#!/bin/sh
test c = $(../urchin -x .print-arg-3 a 'b b b b' c d e)

150
urchin
View File

@ -30,9 +30,16 @@ recurse() {
if [ -d "$potential_test" ] if [ -d "$potential_test" ]
then then
(
if $tap_format; then
indent $indent_level | sed 's/ /#/g'
echo "# Begin - ${potential_test}"
else
indent $indent_level indent $indent_level
echo " ${potential_test}" echo "+ ${potential_test}"
fi
(
cd -- "$potential_test" cd -- "$potential_test"
[ -f setup_dir ] && [ -x setup_dir ] && ./setup_dir >> "$stdout_file" [ -f setup_dir ] && [ -x setup_dir ] && ./setup_dir >> "$stdout_file"
@ -51,40 +58,79 @@ recurse() {
[ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file" [ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
done done
[ -f teardown_dir ] && [ -x teardown_dir ] && ./teardown_dir >> "$stdout_file" [ -f teardown_dir ] && [ -x teardown_dir ] && ./teardown_dir >> "$stdout_file"
echo
) )
elif [ -x "$potential_test" ] if $tap_format; then
then indent $indent_level | sed 's/ /#/g'
echo "# End - ${potential_test}"
[ -f setup ] && [ -x setup ] && ./setup >> "$stdout_file"
# Run the test
if [ -n "$shell_for_sh_tests" ] && has_sh_or_no_shebang_line ./"$potential_test"
then
TEST_SHELL="$TEST_SHELL" "$shell_for_sh_tests" ./"$potential_test" > "$stdout_file" 2>&1
else else
TEST_SHELL="$TEST_SHELL" ./"$potential_test" > "$stdout_file" 2>&1 echo
fi fi
exit_code="$?" else
if [ -x "$potential_test" ]
[ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
indent $indent_level
if [ $exit_code -eq 0 ]
then then
# On success, print a green '✓'
printf '\033[32m✓ \033[0m' [ -f setup ] && [ -x setup ] && ./setup >> "$stdout_file"
printf '%s\n' "${potential_test}"
printf '%s\n' "${potential_test} passed" >> "$logfile" # Run the test
if [ -n "$shell_for_sh_tests" ] && has_sh_or_no_shebang_line ./"$potential_test"
then
TEST_SHELL="$TEST_SHELL" "$shell_for_sh_tests" ./"$potential_test" > "$stdout_file" 2>&1
else
TEST_SHELL="$TEST_SHELL" ./"$potential_test" > "$stdout_file" 2>&1
fi
exit_code="$?"
[ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
if [ $exit_code -eq 0 ]; then
result=success
else
result=fail
fi
else else
# On fail, print a red '✗' result=skip
printf '\033[31m✗ \033[0m' fi
printf '%s\n' "${potential_test}"
printf '%s\n' "${potential_test} failed" >> "$logfile" echo "${result}" >> "$logfile"
printf '\033[31m' # Print output captured from failed test in red. if $tap_format; then
cat "$stdout_file" n=$(grep -ce '^\(success\|fail\|skip\)' "$logfile")
printf '\033[0m'
if [ "$result" == fail ]; then
not='not '
else
not=''
fi
if [ "$result" == skip ]; then
skip='# SKIP '
else
skip=''
fi
echo "${not}ok $n - ${skip}${potential_test}"
if [ "$result" == fail ]; then
echo '# ------------ Begin output ------------'
sed 's/^/# /' "$stdout_file"
echo '# ------------ End output ------------'
fi
else
indent $indent_level
case "$result" in
success)
# On success, print a green '✓'
printf '\033[32m✓ \033[0m'
printf '%s\n' "${potential_test}"
;;
fail)
# On fail, print a red '✗'
printf '\033[31m✗ \033[0m'
printf '%s\n' "${potential_test}"
printf '\033[31m' # Print output captured from failed test in red.
cat "$stdout_file"
printf '\033[0m'
;;
skip)
printf ' %s\n' "${potential_test}"
;;
esac
fi fi
fi fi
[ $indent_level -eq 0 ] && rm "$stdout_file" [ $indent_level -eq 0 ] && rm "$stdout_file"
@ -107,15 +153,12 @@ $USAGE
have shebang line "#!/bin/sh" with the specified shell. have shebang line "#!/bin/sh" with the specified shell.
-f Force running even if the test directory's name does not -f Force running even if the test directory's name does not
contain the word "test". contain the word "test".
-t Format output in Test Anything Protocol (TAP)
-h This help. -h This help.
Go to https://github.com/tlevine/urchin for documentation on writing tests. Go to https://github.com/tlevine/urchin for documentation on writing tests.
EOF EOF
# [Experimental -x option left undocumented for now.]
# -x [Experimental; not meant for direct invocation, but for use in
# the shebang line of test scripts]
# Run with "\$TEST_SHELL", falling back on /bin/sh.
} }
plural () { plural () {
@ -131,7 +174,11 @@ plural () {
} }
urchin_go() { urchin_go() {
echo Running tests at $(date +%Y-%m-%dT%H:%M:%S) | tee "$logfile" rm -f "$logfile"
if "$tap_format"; then
printf \#\
fi
echo Running tests at $(date +%Y-%m-%dT%H:%M:%S)
start=$(date +%s) start=$(date +%s)
# Determine the environment variable to define for test scripts # Determine the environment variable to define for test scripts
@ -151,13 +198,23 @@ urchin_go() {
finish=$(date +%s) finish=$(date +%s)
elapsed=$(($finish - $start)) elapsed=$(($finish - $start))
echo "Done, took $elapsed $(plural second $elapsed)."
set -- $(grep -e 'passed$' "$logfile"|wc -l) $(grep -e 'failed$' "$logfile"|wc -l) passed=$(grep -c '^success' "$logfile")
printf '%s\n' "$1 $(plural test "$1") passed." failed=$(grep -c '^fail' "$logfile")
[ $2 -gt 0 ] && printf '\033[31m' || printf '\033[32m' # If tests failed, print the message in red, otherwise in green. skipped=$(grep -c '^skip' "$logfile")
printf '%s\n' "$2 $(plural test "$2") failed." if $tap_format; then
printf '\033[m' echo "# Took $elapsed $(plural second $elapsed)."
return "$2" echo 1..$(($passed + $failed + $skipped))
else
echo "Done, took $elapsed $(plural second $elapsed)."
printf '%s\n' "$passed $(plural test "$passed") passed."
printf '%s\n' "$skipped $(plural test "$skipped") skipped."
[ $failed -gt 0 ] && printf '\033[31m' || printf '\033[32m' # If tests failed, print the message in red, otherwise in green.
printf '%s\n' "$failed $(plural test "$failed") failed."
printf '\033[m'
fi
rm -f "$logfile"
test -z "$failed" || test "$failed" -eq '0'
} }
urchin_molly_guard() { urchin_molly_guard() {
@ -174,6 +231,7 @@ urchin_molly_guard() {
shell_for_sh_tests= shell_for_sh_tests=
force=false force=false
tap_format=false
while [ $# -gt 0 ] while [ $# -gt 0 ]
do do
case "$1" in case "$1" in
@ -183,11 +241,7 @@ do
shell_for_sh_tests=$1 shell_for_sh_tests=$1
which "$shell_for_sh_tests" >/dev/null || { echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2; urchin_help >&2; exit 2; } which "$shell_for_sh_tests" >/dev/null || { echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2; urchin_help >&2; exit 2; }
;; ;;
-x) # [EXPERIMENTAL; UNDOCUMENTED FOR NOW] `urchin -x <test-script>` in a test script's shebang line is equivalent to invoking that script with `"$TEST_SHELL" <test-script>` -t) tap_format=true;;
shift
urchinsh=${TEST_SHELL:-/bin/sh}
"$urchinsh" "$@"
exit $?;;
-h|--help) urchin_help -h|--help) urchin_help
exit 0;; exit 0;;
-*) urchin_help >&2 -*) urchin_help >&2