516 lines
14 KiB
Bash
Executable File
516 lines
14 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
# This file is part of urchin. It is subject to the license terms in the
|
|
# COPYING file found in the top-level directory of this distribution or at
|
|
# https://raw.githubusercontent.com/tlevine/urchin/master/COPYING
|
|
|
|
# No part of urchin, including this file, may be copied, modified, propagated,
|
|
# or distributed except according to the terms contained in the COPYING file.
|
|
|
|
set -e
|
|
|
|
DEFAULT_SHELLS='sh bash dash mksh zsh'
|
|
|
|
# Make sure that CDPATH isn't set, as it causes `cd` to behave unpredictably -
|
|
# notably, it can produce output.
|
|
unset CDPATH
|
|
|
|
# All temporary files go here
|
|
urchin_tmp=$(mktemp -d)
|
|
> $urchin_tmp/log
|
|
urchin_exit() {
|
|
rm -Rf "$urchin_tmp"
|
|
exit "$@"
|
|
}
|
|
|
|
stdout_file() {
|
|
the_test="$1"
|
|
the_shell="$2"
|
|
|
|
x="$urchin_tmp/stdout$(fullpath "$the_test")"
|
|
mkdir -p "$x"
|
|
echo "$x/$(echo "$the_shell" | md5sum | cut -d\ -f1)"
|
|
}
|
|
|
|
# Expand relative paths
|
|
alias fullpath='readlink -f --'
|
|
|
|
remove_trailing_slash() {
|
|
echo "$1" | sed s/\\/$//
|
|
}
|
|
|
|
urchin_root() {
|
|
# Call recursively but remember the original argument.
|
|
current="$(remove_trailing_slash "$1")"
|
|
if test -n "$2"; then
|
|
orig="$2"
|
|
else
|
|
orig="$1"
|
|
fi
|
|
|
|
if test "$(readlink -f -- "$1")" = /; then
|
|
# Stop traversing upwards at /
|
|
if test -d "$orig"; then
|
|
echo "$orig"
|
|
else
|
|
dirname -- "$orig"
|
|
fi
|
|
elif ! test -e "$current"; then
|
|
echo "$current: No such file or directory">&2
|
|
return 1
|
|
elif test -f "$current"; then
|
|
urchin_root "$(dirname -- "$current")" "$orig"
|
|
elif test -f "$current"/.urchin; then
|
|
remove_trailing_slash "$current"
|
|
else
|
|
urchin_root "$current"/.. "$orig"
|
|
fi
|
|
}
|
|
|
|
# Urchin version number
|
|
VERSION=0.1.0-rc1
|
|
|
|
indent() {
|
|
level="$1"
|
|
if test "$level" -gt 0; then
|
|
printf "%$((2 * ${level}))s"
|
|
fi
|
|
}
|
|
|
|
recurse() {
|
|
set -e
|
|
requested_path="$1"
|
|
potential_test="$(fullpath "$2")"
|
|
cycle_shell="$3"
|
|
TEST_SHELL="$4"
|
|
|
|
for ignore in setup_dir teardown_dir setup teardown; do
|
|
if test "$(basename "$potential_test")" = $ignore; then
|
|
return
|
|
fi
|
|
done
|
|
|
|
echo "$requested_path" | grep "^$potential_test" > /dev/null ||
|
|
echo "$potential_test" | grep "^$requested_path" > /dev/null ||
|
|
return 0
|
|
|
|
validate_strings "$potential_test" 'Test file names'
|
|
|
|
if [ -d "$potential_test" ]; then
|
|
(
|
|
cd -- "$potential_test" > /dev/null
|
|
if test -f setup_dir; then . ./setup_dir; fi
|
|
|
|
if [ -n "$ZSH_VERSION" ]; then
|
|
# avoid "no matches found: *" error when directories are empty
|
|
setopt NULL_GLOB
|
|
fi
|
|
|
|
for test in *; do
|
|
if test "$test" = '*' && ! test -e $test; then
|
|
# The directory is empty.
|
|
break
|
|
fi
|
|
|
|
(
|
|
if test -f setup; then . ./setup; fi
|
|
set +e
|
|
recurse "$requested_path" "$test" "$cycle_shell" "$TEST_SHELL"
|
|
exit_code=$?
|
|
set -e
|
|
if test -f teardown; then . ./teardown; fi
|
|
exit $exit_code
|
|
) &
|
|
|
|
if $run_in_series; then
|
|
wait $!
|
|
exit_code=$?
|
|
if $exit_on_not_ok && test $exit_code -ne 0; then
|
|
if test -f teardown_dir; then . ./teardown_dir; fi
|
|
urchin_exit 1
|
|
fi
|
|
fi
|
|
done
|
|
wait
|
|
if test -f teardown_dir; then . ./teardown_dir; fi
|
|
)
|
|
else
|
|
if [ -x "$potential_test" ]; then
|
|
cd -- "$(dirname -- "$potential_test")"
|
|
|
|
# Determine the environment variable to define for test scripts
|
|
# that reflects the specified or implied shell to use for shell-code tests.
|
|
while read the_test_shell; do
|
|
if test -z "$the_test_shell"; then
|
|
# Shell cycling is disabled with -n; use the present value of
|
|
# TEST_SHELL or default to /bin/sh
|
|
if [ -n "$TEST_SHELL" ]; then
|
|
the_test_shell="$TEST_SHELL"
|
|
else
|
|
the_test_shell=/bin/sh
|
|
fi
|
|
fi
|
|
(
|
|
if test -f setup; then . ./setup; fi
|
|
|
|
# Run the test
|
|
start=$(date +%s)
|
|
set +e
|
|
{
|
|
if $cycle_shell &&
|
|
has_sh_or_no_shebang_line "$potential_test"; then
|
|
TEST_SHELL="$the_test_shell" "$the_test_shell" \
|
|
"$potential_test"
|
|
else
|
|
TEST_SHELL="$the_test_shell" "$potential_test"
|
|
fi
|
|
} > "$(stdout_file "$potential_test" "$the_test_shell")" 2>&1
|
|
exit_code="$?"
|
|
set -e
|
|
finish=$(date +%s)
|
|
|
|
if test -f teardown; then . ./setup; fi
|
|
|
|
if [ $exit_code -eq 0 ]; then
|
|
result=ok
|
|
elif [ $exit_code -eq 3 ]; then
|
|
result=skip
|
|
else
|
|
result=not_ok
|
|
fi
|
|
|
|
elapsed=$(($finish - $start))
|
|
printf "${potential_test}\t${the_test_shell}\t${result}\t${elapsed}\n" \
|
|
>> $urchin_tmp/log
|
|
exit "$exit_code"
|
|
) &
|
|
|
|
if $run_in_series; then
|
|
wait $!
|
|
exit_code=$?
|
|
if $exit_on_not_ok && test $exit_code -ne 0; then
|
|
if test -f teardown_dir; then . ./teardown_dir; fi
|
|
return 1
|
|
fi
|
|
fi
|
|
done < $shell_list
|
|
wait
|
|
else
|
|
# Shell is ''
|
|
printf "${potential_test}\t\tskip\t0\n" >> $urchin_tmp/log
|
|
fi
|
|
fi
|
|
}
|
|
|
|
report_outcome() {
|
|
root="$1"
|
|
tap_format="$2"
|
|
log_file="$3"
|
|
start="$4"
|
|
finish="$5"
|
|
|
|
escaped_root="$(fullpath "$root" | sed 's/\//\\\//g')"
|
|
elapsed=$(($finish - $start))
|
|
|
|
if "$tap_format"; then
|
|
printf \#\
|
|
fi
|
|
echo Running tests at $(date +%Y-%m-%dT%H:%M:%S)
|
|
|
|
for number in n oks skips not_oks; do
|
|
eval "$number=0"
|
|
done
|
|
|
|
# Use a temporary file rather than a pipe because a pipe starts a sub-shell
|
|
# and thus makes the above variables local.
|
|
sorted_log_file=$(mktemp)
|
|
sort "$log_file" > $sorted_log_file
|
|
while read line; do
|
|
abspath=$(echo "$line" | cut -f1)
|
|
path=$(echo "$abspath" | sed "s/$escaped_root\/\?//")
|
|
the_shell=$(echo "$line" | cut -f2)
|
|
result=$(echo "$line" | cut -f3)
|
|
file_elapsed=$(echo "$line" | cut -f4)
|
|
|
|
prevdir=$currentdir
|
|
currentdir="$(dirname -- "$path")"
|
|
|
|
# Number of files that have run, including this one
|
|
n=$(($n + 1))
|
|
|
|
# Number of files that have been ok, not ok, and skipped
|
|
eval "old_count=${result}s"
|
|
eval "${result}s=$(($old_count+1))"
|
|
|
|
if $tap_format; then
|
|
if [ "$result" = not_ok ]; then
|
|
not='not '
|
|
else
|
|
not=''
|
|
fi
|
|
if [ "$result" = skip ]; then
|
|
skip='# SKIP'
|
|
else
|
|
skip=''
|
|
fi
|
|
|
|
if test -z "$the_shell"; then
|
|
the_shell='File is not executable.'
|
|
fi
|
|
echo "${not}ok $n - ${path} ($the_shell) ${skip}"
|
|
if [ "$result" = not_ok ]; then
|
|
echo '# ------------ Begin output ------------'
|
|
sed 's/^/# /' "$(stdout_file "$abspath" "$the_shell")"
|
|
echo '# ------------ End output ------------'
|
|
fi
|
|
echo "# Previous test took $file_elapsed seconds."
|
|
else
|
|
if test "$prevdir" != "$currentdir"; then
|
|
echo
|
|
fi
|
|
if test "$prevpath" != "$path"; then
|
|
printf "$(dirname -- "$path")/\n> $(basename -- "$path")\n"
|
|
fi
|
|
case "$result" in
|
|
ok)
|
|
# On success, print a green '✓'
|
|
printf '\033[32m✓ \033[0m'
|
|
echo "${the_shell} ($file_elapsed $(plural second $file_elapsed))"
|
|
;;
|
|
not_ok)
|
|
# On not_ok, print a red '✗'
|
|
printf '\033[31m✗ \033[0m'
|
|
echo "${the_shell} ($file_elapsed $(plural second $file_elapsed))"
|
|
|
|
# Print output captured from not_ok test in red.
|
|
printf '\033[31m'
|
|
|
|
sed 's/^/ # /' "$(stdout_file "$abspath" "$the_shell")"
|
|
printf '\033[0m'
|
|
;;
|
|
skip)
|
|
if test -z "$the_shell"; then
|
|
echo ' (File is not executable.)'
|
|
else
|
|
echo " ${the_shell} ($file_elapsed $(plural second $file_elapsed))"
|
|
fi
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
prevpath="$path"
|
|
done < $sorted_log_file
|
|
rm $sorted_log_file
|
|
|
|
set +e
|
|
if $tap_format; then
|
|
echo "# Full test suite took $elapsed $(plural second $elapsed)."
|
|
echo 1..$n
|
|
else
|
|
echo
|
|
echo "Done, took $elapsed $(plural second $elapsed)."
|
|
printf '%s\n' "$oks $(plural test "$oks") passed."
|
|
printf '%s\n' "$skips $(plural test "$skips") skipped."
|
|
|
|
# If any tests are not ok, print the message in red.
|
|
if [ $not_oks -gt 0 ] ; then
|
|
printf '\033[31m'
|
|
fi
|
|
printf '%s\n' "$not_oks $(plural test "$not_oks") failed."
|
|
printf '\033[m\n'
|
|
fi
|
|
test "$not_oks" -eq '0'
|
|
}
|
|
|
|
has_sh_or_no_shebang_line() {
|
|
# no shebang line at all
|
|
head -n 1 "$1" | grep -vqE '^#!' && return 0
|
|
|
|
# shebang line is '#!/bin/sh' or legal variations thereof
|
|
head -n 1 "$1" | grep -qE '^#![[:blank:]]*/bin/sh($|[[:blank:]])' &&
|
|
return 0
|
|
|
|
return 1
|
|
}
|
|
|
|
USAGE="usage: $0 [<options>] <test directory>"
|
|
|
|
urchin_help() {
|
|
cat <<EOF
|
|
|
|
$USAGE
|
|
|
|
By default, Urchin checks for the following shells and runs every
|
|
particular test file once per shell.
|
|
|
|
$(echo "$DEFAULT_SHELLS" | sed 's/ /\n /g')
|
|
|
|
On each run,
|
|
|
|
1. The TEST_SHELL environment variable is set to the particular shell.
|
|
2. If the test file has a shebang line of "#!/bin/sh" or no shebang
|
|
line at all, the test script is also executed in that shell.
|
|
|
|
The following flags affect how this multiple-shell testing is handled.
|
|
|
|
-s, --shell <shell> Tell Urchin to use a different list of shells.
|
|
(You can pass this flag multiple times.)
|
|
|
|
-n, --disable-cycling Disable the cycling of shells; Urchin will execute
|
|
test files directly, and it will not manipulate the
|
|
TEST_SHELL environment variable.
|
|
|
|
The following flags affect how Urchin processes tests.
|
|
|
|
-b, --run-in-series Run tests in series. The default is to run tests
|
|
in parallel where possible.
|
|
-e, --exit-on-fail Stop running if any single test fails.
|
|
This is useful if you are running something
|
|
configuration files with Urchin.
|
|
-T, --timeout <seconds> Kill a test if it runs for longer than the
|
|
specified duration. The default is no timeout.
|
|
XXX Not yet implemented
|
|
-f, --force Force running even if the test directory's name
|
|
does not contain the word "test".
|
|
|
|
These options affect how results are formatted.
|
|
|
|
-t, --tap Format output in Test Anything Protocol (TAP)
|
|
-v, --verbose Print stdout from all tests, not just failed tests.
|
|
XXX Not yet implemented
|
|
|
|
The remaining flags provide information about urchin.
|
|
|
|
-h, --help Display this help.
|
|
--version Display the version number.
|
|
|
|
Go to https://github.com/tlevine/urchin for documentation on writing tests.
|
|
|
|
EOF
|
|
}
|
|
|
|
plural () {
|
|
# Make $1 a plural according to the number $2.
|
|
# If $3 is supplied, use that instead of "${1}s".
|
|
# Result is written to stdout.
|
|
if [ "$2" = 1 ]
|
|
then
|
|
printf '%s\n' "$1"
|
|
else
|
|
printf '%s\n' "${3-${1}s}"
|
|
fi
|
|
}
|
|
|
|
validate_strings() {
|
|
test $(echo "$1" | wc -l) -eq 1 || {
|
|
echo '$1 may not contain tab or newline characters.' >&2
|
|
echo 'If this is really a problem, tell me, and I may fix it.' >&2
|
|
urchin_exit 11
|
|
}
|
|
}
|
|
|
|
cycle_shell=true
|
|
shell_list=$urchin_tmp/shell_list
|
|
run_in_series=false
|
|
force=false
|
|
exit_on_not_ok=false
|
|
tap_format=false
|
|
while [ $# -gt 0 ]
|
|
do
|
|
case "$1" in
|
|
-b|--run-in-series) run_in_series=true;;
|
|
-e|--exit-on-fail) exit_on_not_ok=true;;
|
|
-f) force=true;;
|
|
-s)
|
|
shift
|
|
shell_for_sh_tests=$1
|
|
|
|
which "$shell_for_sh_tests" > /dev/null || {
|
|
echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2
|
|
urchin_help >&2
|
|
urchin_exit 11
|
|
}
|
|
|
|
validate_strings "$shell_for_sh_tests" 'Shell paths'
|
|
if echo "$shell_for_sh_tests" | grep \ > /dev/null; then
|
|
echo "Warning: It is best if shell paths contain no spaces so that
|
|
you don't need to quote the TEST_SHELL variable." > /dev/stderr
|
|
fi
|
|
|
|
echo "$shell_for_sh_tests" >> "$shell_list"
|
|
;;
|
|
-n) cycle_shell=false;;
|
|
-t) tap_format=true;;
|
|
-h|--help) urchin_help
|
|
exit 0;;
|
|
--version) echo "$VERSION"
|
|
urchin_exit;;
|
|
-*) urchin_help >&2
|
|
exit 1;;
|
|
*) break;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
if ! $cycle_shell && test -f "$shell_list"; then
|
|
echo "The -n/--disable-cycling and -s/--shell options clash with each other." >&2
|
|
urchin_exit 11
|
|
fi
|
|
|
|
# If -s was not passed, use the available default shells.
|
|
if ! test -f "$shell_list"; then
|
|
if $cycle_shell; then
|
|
for shell in $DEFAULT_SHELLS; do
|
|
if which $shell > /dev/null; then
|
|
echo $shell >> "$shell_list"
|
|
fi
|
|
done
|
|
else
|
|
echo > "$shell_list"
|
|
fi
|
|
fi
|
|
|
|
# Verify argument for main stuff
|
|
if [ "$#" != '1' ] || [ ! -e "$1" ]; then
|
|
if [ -n "$1" ] && [ ! -e "$1" ]; then
|
|
echo "No such file or directory: '$1'" >&2
|
|
fi
|
|
echo "$USAGE" >&2
|
|
urchin_exit 11
|
|
fi
|
|
|
|
if $exit_on_not_ok && ! $run_in_series; then
|
|
echo 'You must also pass -b/--series in order to use -e/--exit-on-fail.' >&2
|
|
urchin_exit 11
|
|
fi
|
|
|
|
# Run or present the Molly guard.
|
|
root="$(urchin_root "$1")"
|
|
if basename "$(fullpath "$root")" |
|
|
grep -i 'test' > /dev/null || $force; then
|
|
|
|
start=$(date +%s)
|
|
set +e
|
|
|
|
# 1 test file or folder to run
|
|
# 2 urchin root
|
|
# 3 Should we cycle shells?
|
|
# 4 TEST_SHELL
|
|
recurse "$(fullpath "$1")" "$root" "$cycle_shell" "$TEST_SHELL"
|
|
exit_code=$?
|
|
|
|
set -e
|
|
finish=$(date +%s)
|
|
|
|
report_outcome "$root" $tap_format $urchin_tmp/log $start $finish
|
|
|
|
urchin_exit $exit_code
|
|
else
|
|
echo 'The name of the directory on which you are running urchin
|
|
does not contain the word "test", so I am not running,
|
|
in case that was an accident. Use the -f flag if you really
|
|
want to run urchin on that directory.' >&2
|
|
exit_code=1
|
|
fi
|
|
urchin_exit $exit_code
|