#!/bin/sh # ---------------------------------------------------------------------- # Copyright (c) 2013, 2014, 2015, 2016 Thomas Levine # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- # Copyright (c) 2014, Michael Klement # Copyright (c) 2012, ScraperWiki Limited # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. # # Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # ---------------------------------------------------------------------- set -e # Delimiters LF="$(printf '\n')" HT="$(printf '\t')" # Urchin version number VERSION=0.0.0-master # Kill subprocesses on interrupt. trap "kill -$$; exit" HUP INT TERM DEFAULT_SHELLS=' sh bash dash ksh posh pdksh mksh yash zsh ' if [ -n "${ZSH_VERSION}" ]; then # avoid "no matches found: *" error when directories are empty setopt NULL_GLOB emulate sh fi # -------------------- Usage -------------------- USAGE="usage: $0 [options]... [test file or directory]..." urchin_help() { cat < Tell Urchin to use a different list of shells. (You can pass this flag multiple times.) The following flags affect how Urchin processes tests. -b, --run-in-series Run tests in series. The default is to run tests in parallel where possible. -e, --exit-on-fail Stop running if any single test fails. This can be useful if you are running something other than test files with Urchin. -T, --timeout Kill a test if it runs for longer than the specified duration. The default is no timeout. -f, --force Force running even if the test directory's name does not contain the word "test". These options affect how results are formatted. Options -q, and -v have no effect when combined with formats other than "urchin". -vv, -vvv, and -vvvv do have effect when combined with formats "urchin" or "tap". -p, --pretty Print results in color and with fancy symbols. -F, --format XXX And these options affect how much is printed. -q, --quiet Print nothing to stdout; the only output is the exit code. (default verbosity) Print names of failed tests and counts of passed, failed, and skipped tests. -v Print stdout from failing tests. -vv Print names of passed tests. -vvv, --verbose Print stdout from all tests. -vvvv, --debug Run with set -x. The remaining flags provide information about urchin. -h, --help Display this help. --version Display the version number. Urchin recognizes certain environment variables. TEST_SHELL This is sometimes over-ridden; see -s. RUN_IN_SERIES Set this to have the same effect as -b/--run-in-series. This is helpful if you are calling urchin inside an urchin test suite. Exit codes have the following meanings 0 All tests were ok 1 At least one test was not ok. 2 No tests were found. 10 Dependencies are missing (locally, not on remotes). 11 Flags were not valid. 12 File names contain unsupported delimiters (HT or LF). 13 An test shell specified with -s/--shell is not available. Go to https://thomaslevine.com/!/urchin/ for documentation on writing tests. EOF } # -------------------- Portable wrappers -------------------- mktemp_dir() { # Support HP-UX mktemp that has wrong exit codes and # can't make directories. tmp=$(mktemp) if test -f "${tmp}"; then rm "${tmp}" fi mkdir "${tmp}" echo "${tmp}" } md5 () { case "${urchin_md5}" in md5sum) echo "${1}" | md5sum | sed 's/ .*//' ;; md5) echo "${1}" | md5 | sed 's/.* //' ;; esac } # -------------------- Utilities -------------------- if command -v md5 1> /dev/null 2> /dev/null; then urchin_md5=md5 elif command -v md5sum 1> /dev/null 2> /dev/null; then urchin_md5=md5sum else echo Could not find MD5 hash command >&2 exit 10 fi epoch_date() { date +%s } epoch_pax() { # Based on http://stackoverflow.com/a/7262588/407226 tmp="$(mktemp)" echo "ibase=8;$({ pax -wx cpio "${tmp}"; echo; } | cut -c 48-59)" | bc rm "${tmp}" } if epoch_date 2>&1 > /dev/null; then epoch=epoch_date elif epoch_pax 2>&1 > /dev/null; then epoch=epoch_pax else echo I could not find a seconds counter. >&2 exit 10 fi plural () { # Make $1 a plural according to the number $2. # If $3 is supplied, use that instead of "${1}s". # Result is written to stdout. if [ "${2}" = 1 ]; then echo "${1}" else echo "${3-${1}s}" fi } has_shebang_line() { head -n 1 "${1}" | grep -v '^#!/bin/sh$' | grep -q '^#!' } indent() { level="${1}" if test "${level}" -gt 0; then printf "%$((2 * ${level}))s" fi } # Expand relative paths fullpath() { if test -e "${1}"; then readlink -f -- "${1}" | sed 's/\/*$//' else echo "Could not find file or directory: ${1}" >&2 return 1 fi } # If $1 is an ascestor of $2, echo the path of $2 relative $1. # If either of $1 or $2 does not exist, return with code 1 # If $1 is not an ancestor of $2, return with code 2 # Otherwise, return with code 2. localpath() { test -e "${1}" && test -e "${2}" || return 1 # A file is missing. parent="$(fullpath "${1}")" child="$(fullpath "${2}")" if echo "${child}" | grep "^${parent}/" > /dev/null; then # Child is really a child. echo "${child#"${parent}/"}" return 0 else # Child is not really a child. return 2 fi } contains() { case "$#" in 1) grep "${1}" > /dev/null ;; 2) echo "${1}" | contains "${2}" ;; 3) contains "${1}" "${2}" && contains "${1}" "${3}" ;; *) container="${1}" && shift contains "${container}" "${1}" && shift && contains "${container}" "${@}" ;; esac } is_set() { set | grep "^${1}=" > /dev/null } # File where a test's stdout and stderr is saved stdout_file() { urchin_tmp="${1}" the_test="${2}" the_shell="${3}" x="${urchin_tmp}/stdout$(localpath "$the_test")" mkdir -p "${x}" echo "${x}/$(md5 "${the_shell}")" } # Print a line of a log file log() { for arg in "$@"; do printf "${arg}\t" done printf '\n' } # Root directory of the present test suite # USAGE: test_suite_root test_suite_root() { # Call recursively but remember the original argument. orig="${2:-$1}" current="${1%/}" abscurrent="$(fullpath "${current}")" if test "${abscurrent}" = / || basename "${abscurrent}" | contains '^\.' ; then # Stop traversing upwards at / and at hidden directories. if test -d "${orig}"; then echo "${orig}" else dirname -- "${orig}" fi elif ! test -e "${current}"; then echo "${current}: No such file or directory">&2 return 1 elif test -f "${current}"; then test_suite_root "$(dirname -- "${current}")" "${orig}" elif test -f "${current}"/.urchin_root; then echo "${current}" else test_suite_root "${current}"/.. "${orig}" fi } # -------------------- Printing output -------------------- # Format functions may read a log file from stdin. meta_verbosity() { echo "if test \${${1}} -ge ${2}; then ${3}=true; fi" } format_tap() { v="${1}" urchin_tmp="${2}" elapsed="${3}" $(meta_verbosity v 2 print_not_ok_stdout) $(meta_verbosity v 3 print_ok_stdout) print_stdout() { echo '# ------------ Begin output ------------' sed 's/^/# /' "$(stdout_file "${urchin_tmp}" "${path}" "${the_shell}")" echo '# ------------ End output ------------' } while IFS="${HT}" read -r remote the_shell path result file_elapsed; do # Number of files that have run, including this one n=$(( ${n:-0} + 1)) case "${result}" in ok) echo "ok $n - ${path} (${the_shell}${on})" if "${print_ok_stdout}"; then print_stdout; fi ;; not_ok) echo "not_ok $n - ${path} (${the_shell}${on})" if "${print_not_ok_stdout}"; then print_stdout; fi ;; skip) "ok $n - ${path} (${the_shell}${on}) # SKIP" ;; esac echo "# Previous test took ${file_elapsed} seconds." done echo "# Full test suite took ${elapsed} $(plural second ${elapsed})." echo 1.."${n}" } format_urchin() { v="${1}" urchin_tmp="${2}" verbosity="${3}" print_in_color="${4}" $(meta_verbosity v 1 print_margins) $(meta_verbosity v 1 print_not_ok) $(meta_verbosity v 2 print_not_ok_stdout) $(meta_verbosity v 2 print_ok) $(meta_verbosity v 3 print_ok_stdout) if $print_in_color; then success_mark=$(printf "\033[32m✓ \033[0m") fail_mark=$(printf "\033[31m✗ \033[0m") else success_mark=.\ fail_mark=F\ fi header() { if test "${prevdir}" != "${currentdir}"; then echo fi if test "${prevpath}" != "${path}"; then printf "$(dirname -- "${path}")/\n> $(basename -- "${path}")\n" fi } print_stdout() { sed 's/^/ | /' "$(stdout_file "${urchin_tmp}" "${path}" "${the_shell}")" } while IFS="${HT}" read -r remote the_shell path result file_elapsed; do abspath=${urchin_tmp}/${path} currentdir="$(dirname -- "${path}")" prevdir="${currentdir}" # Format the message if test -z "${remote}"; then on=" on ${remote}" else on= fi if test result = skip; then parantheses="(skipped)" else parantheses="(${file_elapsed} $(plural second "${file_elapsed}"))" fi message="${the_shell}${on} (${file_elapsed} ${unit})" # Keep track of how many files have been ok, not ok, and skipped. eval "${result}s=$((${result}s+1))" # Print the result. case "${result}" in ok) if "${print_ok}"; then header && echo "${success_mark} ${message}" fi ;; not_ok) if "${print_not_ok}"; then header && echo "${fail_mark} ${message}" if "${print_not_ok_stdout}"; then print_stdout; fi fi ;; skip) if "${print_ok}"; then header && echo "${skip_mark} ${message}" if "${print_ok_stdout}"; then print_stdout; fi fi ;; esac prevpath="${path}" done if "${print_margins}"; then echo echo "Done, took ${elapsed} $(plural second ${elapsed})." echo "${oks} $(plural test "${oks}") passed." echo "${skips} $(plural test "${skips}") skipped." echo "${not_oks} $(plural test "${not_oks}") failed." fi } # -------------------- Main stuff -------------------- # Return codes have the following meaning # 0) All tests succeded # 1) At least one test failed. # *) Something else went wrong; Urchin should exit with the same code. recurse() { echo "$3 $4 $5" abs_root="${1}" abs_requested="${2}" abs_current="${3}" run_in_series_root="${4}" exit_on_not_ok="${5}" rel_requested="$(localpath ${abs_root} ${abs_requested})" rel_current="$(localpath ${abs_root} ${abs_current})" for ignore in setup_dir teardown_dir setup teardown; do if test "$(basename "${abs_current}")" = "${ignore}"; then return 0 fi done if contains "${abs_current}" "${HT}"; then echo "${shell_list}" | while read -r sh; do no_tab="$(echo "${rel_current}" | tr '\t' ' ')" log "${remote}" "${sh}" "${no_tab}" tab '' >> "${urchin_tmp}"/log done elif [ -x "${abs_current}" ]; then meta_finalize=' if test -f .urchin_dir && grep series ./.urchin_dir > /dev/null || "${run_in_series_root}"; then set +e wait ${!} return_code=$? set -e if "${exit_on_not_ok}" && test "${return_code}" -ne 0; then eval "$(meta_dot_if_exists teardown_dir)" return $return_code fi fi ' meta_dot_if_exists() { echo " if test -f ${1}; then . ./${1} fi " } if [ -d "${abs_current}" ]; then ( cd -- "${abs_current}" eval "$(meta_dot_if_exists setup_dir)" for test in *; do if test "${test}" = '*' && ! test -e "${test}"; then # The directory is empty. break fi recurse "${abs_root}" "${abs_requested}" "${abs_current}/${test}" \ "${run_in_series_root}" "${exit_on_not_ok}" #& # eval "${meta_finalize}" done wait eval "$(meta_dot_if_exists teardown_dir)" ) elif [ -f "${abs_current}" ]; then cd -- "$(dirname -- "${abs_current}")" echo "${shell_list}" | while read -r sh; do ( eval "$(meta_dot_if_exists setup)" out_file="$(stdout_file "${urchin_tmp}" "${rel_current}" "${sh}")" # Run with a shell? if has_shebang_line "${abs_current}"; then set -- "${abs_current}" else set -- "${sh}" "${abs_current}" fi # Run the test cmd='TEST_SHELL="${sh}" $TIMEOUT "$@" > "${out_file}" 2>&1' start=$("${epoch}") exit_code="$(catch "${cmd}")" finish=$("${epoch}") elapsed=$(($finish - $start)) eval "$(meta_dot_if_exists teardown)" case "${exit_code}" in 0) result=ok ;; 3) result=skip ;; *) result=not_ok ;; esac log "${remote}" "${rel_curent}" "${result}" "${elapsed}" \ >> "${urchin_tmp}"/log exit "${exit_code}" ) #& # eval "${meta_finalize}" done #wait fi elif test -n "${rel_current}"; then # Skip because the file is not executable. echo "${shell_list}" | while read -r sh; do printf "\t${sh}\t${rel_current}\tskip\t0\n" >> "${urchin_tmp}"/log done fi echo ${rel_current} 3 >> /tmp/bbb } main() { # Defaults format=urchin if "${RUN_IN_SERIES}" 2> /dev/null; then run_in_series=true else run_in_series=false fi exit_on_not_ok=false # Shift if possible; error otherwise. flag_arg() { flag="${1}" if shift; then echo "${1}" else echo Missing argument for "${flag}" >&2 exit 11 fi } # Receive input while [ "${#}" -gt 0 ]; do case "${1}" in -b|--run-in-series) run_in_series=true;; -e|--exit-on-fail) exit_on_not_ok=true;; -f|--force) force=true;; -F|--format) format="$(flag_arg)" ;; -p|--pretty) print_in_color=true;; -q|--quiet) verbosity=0 ;; -v) verbosity=2 ;; -vv) verbosity=3 ;; -vvv|--verbose) verbosity=4 ;; -vvvv|--debug) verbosity=5 ;; --version) echo "${VERSION}" && exit;; -h|--help) urchin_help && exit 0;; -s|--shell) sh="$(flag_arg)" if ! command -v "${sh}" > /dev/null; then echo "Cannot find specified shell: '${sh}'" >&2 urchin_help >&2 exit 13 elif contains "${potential_test}" "${HT}" "${LF}"; then echo 'Shell paths may contain all characters other than' >&2 echo 'horizontal tab (\t) and line feed (\n).' >&2 exit 11 elif contains "${sh}" "[${IFS}]"; then echo "Warning: It is best if field-separator characters (usually spaces) are absent from shell paths so that you don't need to quote the TEST_SHELL variable." >&2 fi shell_list="${sh}${LF}${shell_list}" ;; -T|--timeout) urchin_timeout="$(flag_arg)" if ! contains "${urchin_timeout}" \ '[0-9][0-9.]*\(s\|m\|h\|d\|\)' ; then echo Bad timeout argument: "${urchin_timeout}" >&2 exit 11 fi ;; -*) urchin_help >&2 && exit 11;; *) if contains "${1}" "${HT}" "${LF}"; then echo 'Test file names may contain all characters other than' >&2 echo 'horizontal tab (\t) and line feed (\n).' >&2 exit 11 elif [ ! -e "${1}" ]; then echo "No such file or directory: '${1}'" >&2 echo "${USAGE}" >&2 exit 11 elif ! { # Molly guard root="$(test_suite_root "${1}")" basename "$(fullpath "${root}")" | grep -i 'test' > /dev/null }; then molly=true fi test_seeds="${1}${LF}${test_seeds}" ;; esac shift done if ! is_set test_seeds; then echo Missing test location >&2 exit 2 fi if is_set molly && ! is_set force; then echo 'The root directory of the tests that you are running urchin on doesnot contain the word "test", so I am not running, in case that was an accident. Use the -f flag if you really want to run urchin on that directory.' >&2 exit 12 fi # If -s was not passed, use the available default shells. if ! is_set "${shell_list}"; then if $cycle_shell; then for shell in $DEFAULT_SHELLS; do if command -v "${shell}" 1> /dev/null 2> /dev/null; then shell_list="${shell}${HT}${shell_list}" fi done fi fi if is_set urchin_timeout; then # Choose the timeout command if timeout -t 0 true 2> /dev/null; then TIMEOUT="timeout -t ${urchin_timeout}" elif timeout 0 true 2> /dev/null; then TIMEOUT="timeout ${urchin_timeout}" else echo I couldn\'t figure out how to use your version of timeout >&2 exit 10 fi fi if is_set exit_on_not_ok && ! is_set run_in_series; then echo 'You must also pass -b/--series in order to use -e/--exit-on-fail.' >&2 exit 11 fi # -------------------- REALLY RUN -------------------- # # Temporary files urchin_tmp=$(mktemp_dir) # Write header information echo Running tests at $(date +%Y-%m-%dT%H:%M:%S) >> "${urchin_tmp}"/head printf 'Cycling with the following shells: ' >> "${urchin_tmp}"/head echo "${shell_list}" | tr "${HT}" \ >> "${urchin_tmp}"/head echo >> "${urchin_tmp}"/head start=$("${epoch}") while read -r seed; do root="$(fullpath "$(test_suite_root "${seed}")")" abs="$(fullpath "${seed}")" set +e recurse "${root}" "${abs}" "${root}" \ "${run_in_series}" "${exit_on_not_ok}" return_code=$? set -e echo $return_code if test "${return_code}" -eq 0; then break; fi done< "${sorted_log_file}" # rm "${sorted_log_file}" # test "${not_oks}" -eq '0' else echo 'No tests found' >&2 return_code=2 fi fi rm -Rf "${urchin_tmp}" exit "${return_code}" } echo "kill -$$" is_set NO_MAIN || main "$@"