urchin/urchin
2016-03-06 09:44:10 +00:00

602 lines
18 KiB
Bash
Executable File

#!/bin/sh
# ----------------------------------------------------------------------
# Copyright (c) 2013, 2014, 2015, 2016 Thomas Levine
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# Copyright (c) 2014, Michael Klement
# Copyright (c) 2012, ScraperWiki Limited
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------
set -e
DEFAULT_SHELLS='sh bash dash mksh zsh'
if [ -n "${ZSH_VERSION}" ]; then
# avoid "no matches found: *" error when directories are empty
setopt NULL_GLOB
emulate sh
fi
# All temporary files go here
urchin_tmp=$(mktemp -d)
> "${urchin_tmp}/log"
urchin_exit() {
rm -Rf "${urchin_tmp}"
exit "$@"
}
stdout_file() {
the_test="${1}"
the_shell="${2}"
x="${urchin_tmp}/stdout$(fullpath "$the_test")"
mkdir -p "${x}"
echo "${x}/$(echo "${the_shell}" | md5sum | cut -d\ -f1)"
}
# Expand relative paths
fullpath() {
readlink -f -- "${1}"
}
remove_trailing_slash() {
echo "$1" | sed s/\\/$//
}
urchin_root() {
# Call recursively but remember the original argument.
current="$(remove_trailing_slash "${1}")"
if test -n "${2}"; then
orig="${2}"
else
orig="${1}"
fi
abscurrent="$(fullpath "${1}")"
if test "${abscurrent}" = / ||
basename "${abscurrent}" | grep '^\.' > /dev/null; then
# Stop traversing upwards at / and at hidden directories.
if test -d "${orig}"; then
echo "${orig}"
else
dirname -- "${orig}"
fi
elif ! test -e "${current}"; then
echo "${current}: No such file or directory">&2
return 1
elif test -f "${current}"; then
urchin_root "$(dirname -- "${current}")" "${orig}"
elif test -f "${current}"/.urchin; then
remove_trailing_slash "${current}"
else
urchin_root "${current}"/.. "${orig}"
fi
}
# Urchin version number
VERSION=0.1.0-rc1
indent() {
level="${1}"
if test "${level}" -gt 0; then
printf "%$((2 * ${level}))s"
fi
}
recurse() {
requested_path="${1}"
potential_test="$(fullpath "${2}")"
cycle_shell="${3}"
TEST_SHELL="${4}"
for ignore in setup_dir teardown_dir setup teardown; do
if test "$(basename "${potential_test}")" = "${ignore}"; then
return
fi
done
echo "${requested_path}" | grep "^${potential_test}" > /dev/null ||
echo "${potential_test}" | grep "^${requested_path}" > /dev/null ||
return 0
validate_strings "${potential_test}" 'Test file names'
if [ -x "${potential_test}" ]; then
if [ -d "${potential_test}" ]; then
(
cd -- "${potential_test}"
if test -f setup_dir; then . ./setup_dir; fi
for test in *; do
if test "${test}" = '*' && ! test -e "${test}"; then
# The directory is empty.
break
fi
(
if test -f setup; then . ./setup; fi
if recurse "${requested_path}" "${test}" "${cycle_shell}" \
"${TEST_SHELL}"; then
exit_code=0
else
exit_code="${?}"
fi
if test -f teardown; then . ./teardown; fi
exit "${exit_code}"
) &
if "${run_in_series}"; then
if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
if test -f teardown_dir; then . ./teardown_dir; fi
return 1
fi
fi
done
wait
if test -f teardown_dir; then . ./teardown_dir; fi
)
elif [ -f "${potential_test}" ]; then
cd -- "$(dirname -- "${potential_test}")"
# Determine the environment variable to define for test scripts
# that reflects the specified or implied shell to use for shell-code tests.
while read the_test_shell; do
(
if test -f setup; then . ./setup; fi
# Run the test
start=$(date +%s)
set +e
{
if "${cycle_shell}"; then
if has_shebang_line "${potential_test}"; then
TEST_SHELL="${the_test_shell}" $TIMEOUT "${potential_test}"
else
TEST_SHELL="${the_test_shell}" $TIMEOUT \
"${the_test_shell}" "${potential_test}"
fi
else
# Shell cycling is disabled with -n; use the present value of
# TEST_SHELL or default to /bin/sh
if [ -n "${TEST_SHELL}" ]; then
$TIMEOUT "${potential_test}"
else
TEST_SHELL=/bin/sh $TIMEOUT "${potential_test}"
fi
fi
} > "$(stdout_file "${potential_test}" "${the_test_shell}")" 2>&1
exit_code="${?}"
set -e
finish=$(date +%s)
if test -f teardown; then . ./setup; fi
if [ "${exit_code}" -eq 0 ]; then
result=ok
elif [ "${exit_code}" -eq 3 ]; then
result=skip
else
result=not_ok
fi
elapsed=$(($finish - $start))
printf "${potential_test}\t${the_test_shell}\t${result}\t${elapsed}\n" \
>> "${urchin_tmp}"/log
exit "${exit_code}"
) &
if "${run_in_series}"; then
if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
if test -f teardown_dir; then . ./teardown_dir; fi
return 1
fi
fi
done < "${shell_list}"
wait
else
echo "${potential_test}: Neither file nor directory!?" > /dev/stderr
fi
else
# Shell is ''
printf "${potential_test}\t\tskip\t0\n" >> "${urchin_tmp}"/log
fi
}
report_outcome() {
root="${1}"
tap_format="${2}"
log_file="${3}"
start="${4}"
finish="${5}"
escaped_root="$(fullpath "${root}" | sed 's/\//\\\//g')"
elapsed=$(($finish - $start))
if "${tap_format}"; then
printf \#\
fi
echo Running tests at $(date +%Y-%m-%dT%H:%M:%S)
for number in n oks skips not_oks; do
eval "${number}=0"
done
# Use a temporary file rather than a pipe because a pipe starts a sub-shell
# and thus makes the above variables local.
sorted_log_file=$(mktemp)
# Sort in alphabetical order.
# GNU sort requires -m, and BSD sort doesn't.
sort "${log_file}" > "${sorted_log_file}"
while read line; do
abspath=$(echo "${line}" | cut -f1)
path=$(echo "${abspath}" | sed "s/$escaped_root\/\?//")
the_shell=$(echo "${line}" | cut -f2)
result=$(echo "${line}" | cut -f3)
file_elapsed=$(echo "$line" | cut -f4)
prevdir="${currentdir}"
currentdir="$(dirname -- "${path}")"
# Number of files that have run, including this one
n=$(($n + 1))
# Number of files that have been ok, not ok, and skipped
eval "old_count=${result}s"
eval "${result}s=$(($old_count+1))"
if "${tap_format}"; then
if [ "${result}" = not_ok ]; then
not='not '
else
not=''
fi
if [ "${result}" = skip ]; then
skip='# SKIP'
else
skip=''
fi
if test -z "${the_shell}"; then
the_shell='File is not executable.'
fi
echo "${not}ok $n - ${path} (${the_shell}) ${skip}"
if { test "${result}" = not_ok && "${print_not_ok_stdout}"; } ||
{ test "${result}" = ok && "${print_ok_stdout}"; }; then
echo '# ------------ Begin output ------------'
sed 's/^/# /' "$(stdout_file "${abspath}" "${the_shell}")"
echo '# ------------ End output ------------'
fi
echo "# Previous test took ${file_elapsed} seconds."
else
if test "${prevdir}" != "${currentdir}"; then
echo
fi
if test "${prevpath}" != "${path}"; then
printf "$(dirname -- "${path}")/\n> $(basename -- "${path}")\n"
fi
case "${result}" in
ok)
# On success, print a green '✓'
if "${print_in_color}"; then
printf '\033[32m✓ \033[0m'
else
printf '✓ '
fi
echo "${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
;;
not_ok)
# On not_ok, print a red '✗'
if "${print_in_color}"; then
printf '\033[31m✗ \033[0m'
else
printf '✗ \033[0m'
fi
echo "${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
;;
skip)
if test -z "${the_shell}"; then
echo ' (File is not executable.)'
else
echo " ${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
fi
;;
esac
if { test "${result}" = not_ok && "${print_not_ok_stdout}"; } ||
{ test "${result}" = ok && "${print_ok_stdout}"; }; then
sed 's/^/ # /' "$(stdout_file "${abspath}" "${the_shell}")"
fi
fi
prevpath="${path}"
done < "${sorted_log_file}"
rm "${sorted_log_file}"
if "${tap_format}"; then
echo "# Full test suite took ${elapsed} $(plural second ${elapsed})."
echo 1.."${n}"
else
echo
echo "Done, took ${elapsed} $(plural second ${elapsed})."
printf '%s\n' "${oks} $(plural test "${oks}") passed."
printf '%s\n' "${skips} $(plural test "${skips}") skipped."
# If any tests are not ok, print the message in red.
if [ "${not_oks}" -gt 0 ] && "${print_in_color}"; then
printf '\033[31m'
fi
printf '%s\n' "${not_oks} $(plural test "${not_oks}") failed."
if "${print_in_color}"; then
printf '\033[m\n'
fi
fi
test "${not_oks}" -eq '0'
}
has_shebang_line() {
head -n 1 "${1}" | grep -qE '^#!'
}
USAGE="usage: $0 [<options>] <test directory>"
urchin_help() {
cat <<EOF
${USAGE}
By default, Urchin checks for the following shells and runs every
particular test file once per shell.
$(echo "${DEFAULT_SHELLS}" | sed 's/ /\n /g')
On each run,
1. The TEST_SHELL environment variable is set to the particular shell.
2. If the test file lacks a shebang line, the test script is also
executed in that shell.
The following flags affect how this multiple-shell testing is handled.
-s, --shell <shell> Tell Urchin to use a different list of shells.
(You can pass this flag multiple times.)
-n, --disable-cycling Disable the cycling of shells; Urchin will
execute test files ordinarily, implicitly using
sh for files that lack shebang lines. It will
set the TEST_SHELL variable to "/bin/sh" if and
only if TEST_SHELL is empty or undefined.
The following flags affect how Urchin processes tests.
-b, --run-in-series Run tests in series. The default is to run tests
in parallel where possible.
-e, --exit-on-fail Stop running if any single test fails.
This is useful if you are running something
configuration files with Urchin.
-T, --timeout <seconds> Kill a test if it runs for longer than the
specified duration. The default is no timeout.
-f, --force Force running even if the test directory's name
does not contain the word "test".
These options affect how results are formatted.
-c, --color Print results in color.
-t, --tap Format output in Test Anything Protocol (TAP)
-v, --verbose Print stdout from failing tests.
-vv Print stdout from all tests.
-vvv Print debugging messages (XXX not implemented)
-q, --quiet Print nothing to stdout; the only output is the
exit code (XXX not implemented).
The remaining flags provide information about urchin.
-h, --help Display this help.
--version Display the version number.
Go to https://github.com/tlevine/urchin for documentation on writing tests.
EOF
}
plural () {
# Make $1 a plural according to the number $2.
# If $3 is supplied, use that instead of "${1}s".
# Result is written to stdout.
if [ "${2}" = 1 ]
then
printf '%s\n' "${1}"
else
printf '%s\n' "${3-${1}s}"
fi
}
validate_strings() {
test $(echo "${1}" | wc -l) -eq 1 || {
echo '$1 may not contain tab or newline characters.' >&2
echo 'If this is really a problem, tell me, and I may fix it.' >&2
urchin_exit 11
}
}
main() {
cycle_shell=true
shell_list="${urchin_tmp}"/shell_list
run_in_series=false
force=false
exit_on_not_ok=false
tap_format=false
print_in_color=false
print_ok_stdout=false
print_not_ok_stdout=false
while [ "${#}" -gt 0 ]
do
case "${1}" in
-b|--run-in-series) run_in_series=true;;
-e|--exit-on-fail) exit_on_not_ok=true;;
-f|--force) force=true;;
-s|--shell)
shift
shell_for_sh_tests="${1}"
which "${shell_for_sh_tests}" > /dev/null || {
echo "Cannot find specified shell: '${shell_for_sh_tests}'" >&2
urchin_help >&2
urchin_exit 11
}
validate_strings "${shell_for_sh_tests}" 'Shell paths'
if echo "${shell_for_sh_tests}" | grep \ > /dev/null; then
echo "Warning: It is best if shell paths contain no spaces so that
you don't need to quote the TEST_SHELL variable." > /dev/stderr
fi
echo "${shell_for_sh_tests}" >> "${shell_list}"
;;
-n|--disable-cycling) cycle_shell=false;;
-t|--tap) tap_format=true;;
-T|--timeout) shift; urchin_timeout="${1}" ;;
-c|--color) print_in_color=true;;
-v|--verbose) print_not_ok_stdout=true;;
-vv) print_not_ok_stdout=true
print_ok_stdout=true;;
-h|--help) urchin_help
urchin_exit 0;;
--version) echo "${VERSION}"
urchin_exit;;
-*) urchin_help >&2
urchin_exit 11;;
*) break;;
esac
shift
done
# -------------------- VALIDATE INPUT -------------------- #
if ! "${cycle_shell}" && test -f "${shell_list}"; then
echo "The -n/--disable-cycling and -s/--shell options clash with each other." >&2
urchin_exit 11
fi
# If -s was not passed, use the available default shells.
if ! test -f "${shell_list}"; then
if $cycle_shell; then
for shell in $DEFAULT_SHELLS; do
if which $shell > /dev/null; then
echo $shell >> "$shell_list"
fi
done
else
echo > "$shell_list"
fi
fi
# Verify argument for main stuff
if [ "${#}" != '1' ] || [ ! -e "${1}" ]; then
if [ -n "${1}" ] && [ ! -e "${1}" ]; then
echo "No such file or directory: '${1}'" >&2
fi
echo "${USAGE}" >&2
urchin_exit 11
fi
# Warn about strange sort commands
weird_string='\n- c\n-- b\n--- c\n---- a\n'
if test $(printf "${weird_string}" | sort | tr -d '[ \n-]') != cbca; then
echo 'Your version of sort sorts in dictionary order (-d) by default.
Depending on how you name your tests, your Urchin output may look strange.
If this is a problem, install BusyBox or BSD coreutils.' >&2
fi
if test -n "${urchin_timeout}"; then
# Choose the timeout command
if timeout -t 0 true; then
TIMEOUT="timeout -t ${urchin_timeout}"
elif timeout 0 true; then
TIMEOUT="timeout ${urchin_timeout}"
else
echo I couldn\'t figure out to use your version of timeout >&2
urchin_exit 1
fi
fi
if "${exit_on_not_ok}" && ! "${run_in_series}"; then
echo 'You must also pass -b/--series in order to use -e/--exit-on-fail.' >&2
urchin_exit 11
fi
# Molly guard.
root="$(urchin_root "${1}")"
if ! {
basename "$(fullpath "${root}")" |
grep -i 'test' > /dev/null || "${force}"
}; then
echo 'The root directory of the tests that you are running urchin on
does not contain the word "test", so I am not running,
in case that was an accident. Use the -f flag if you really
want to run urchin on that directory.' >&2
urchin_exit 1
fi
# -------------------- REALLY RUN -------------------- #
start=$(date +%s)
# 1 test file or folder to run
# 2 urchin root
# 3 Should we cycle shells?
# 4 TEST_SHELL
recurse "$(fullpath "${1}")" "${root}" "${cycle_shell}" "${TEST_SHELL}" || :
finish=$(date +%s)
test $(cat "${urchin_tmp}"/log | wc -l) -gt 0 || {
echo 'No tests found' > /dev/stderr
urchin_exit 1
}
report_outcome "${root}" "${tap_format}" "${urchin_tmp}"/log "${start}" \
"${finish}"
urchin_exit "${?}"
}
main "$@"