urchin/urchin

#!/bin/sh

# ----------------------------------------------------------------------
# Copyright (c) 2013, 2014, 2015, 2016 Thomas Levine
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as 
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ----------------------------------------------------------------------


# ----------------------------------------------------------------------
# Copyright (c) 2014, Michael Klement
# Copyright (c) 2012, ScraperWiki Limited
# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 
# Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------

set -e

# Delimiters
LF="$(printf '\n')"
HT="$(printf '\t')"
DELIMITERS="[${LF}${HT}]"

# Urchin version number
VERSION=0.0.0-master

# Kill subprocesses on interrupt.
trap "kill -$$; exit" HUP INT TERM

DEFAULT_SHELLS='
sh
bash
dash
ksh
posh
pdksh
mksh
yash
zsh
'
if [ -n "${ZSH_VERSION}" ]; then
  # avoid "no matches found: *" error when directories are empty
  setopt NULL_GLOB
  emulate sh
fi

# -------------------- Usage -------------------- 
USAGE="usage: $0 [options]... [test file or directory]..."

urchin_help() {
  cat <<EOF

${USAGE}

By default, Urchin checks for the following shells and runs every
particular test file once per shell.

  $(echo "${DEFAULT_SHELLS}" | sed 's/ /\n  /g')

On each run,

1. The TEST_SHELL environment variable is set to the particular shell.
2. If the test file lacks a shebang line or has a shebang line of
   "#!/bin/sh", the test script is also executed in that shell.

The following flags affect how this multiple-shell testing is handled.

-s, --shell <shell>      Tell Urchin to use a different list of shells.
                         (You can pass this flag multiple times.)

-n, --disable-cycling    Disable the cycling of shells; Urchin will
                         execute test files ordinarily, implicitly using
                         sh for files that lack shebang lines. It will
                         set the TEST_SHELL variable to "/bin/sh" if and
                         only if TEST_SHELL is empty or undefined.

The following flags affect how Urchin processes tests.

-b, --run-in-series      Run tests in series. The default is to run tests
                         in parallel where possible.
-e, --exit-on-fail       Stop running if any single test fails.
                         This can be useful if you are running something
                         other than test files with Urchin.
-T, --timeout <seconds>  Kill a test if it runs for longer than the
                         specified duration. The default is no timeout.
-f, --force              Force running even if the test directory's name
                         does not contain the word "test".

These options affect how results are formatted. Options -q, and -v
have no effect when combined with formats other than "urchin".
-vv, -vvv, and -vvvv do have effect when combined with formats "urchin"
or "tap".

-p, --pretty             Print results in color and with fancy symbols.
-F, --format <name>      XXX

And these options affect how much is printed.

-q, --quiet              Print nothing to stdout;
                         the only output is the exit code.
(default verbosity)      Print names of failed tests and counts
                         of passed, failed, and skipped tests.
-v                       Print stdout from failing tests.
-vv                      Print names of passed tests.
-vvv, --verbose          Print stdout from all tests.
-vvvv, --debug           Run with set -x.

The remaining flags provide information about urchin.

-h, --help               Display this help.
--version                Display the version number.

Urchin recognizes certain environment variables.

TEST_SHELL           This is sometimes over-ridden; see -s.
RUN_IN_SERIES        Set this to have the same effect as
                     -b/--run-in-series. This is helpful if you are
                     calling urchin inside an urchin test suite.

Go to https://thomaslevine.com/!/urchin/ for documentation on writing tests.

EOF
}

# -------------------- Dependency checks -------------------- 
if command -v md5 1> /dev/null 2> /dev/null; then
  urchin_md5=md5
elif command -v md5sum 1> /dev/null 2> /dev/null; then
  urchin_md5=md5sum
else
  echo Could not find MD5 hash command >&2
  exit 1
fi

if epoch_date 2>&1 > /dev/null; then
  epoch=epoch_date
elif epoch_pax 2>&1 > /dev/null; then
  epoch=epoch_pax
else
  echo I could not find a seconds counter. >&2
  exit 1
fi


# -------------------- Portable wrappers -------------------- 
mktemp_dir() {
  # Support HP-UX mktemp that has wrong exit codes and
  # can't make directories.
  tmp=$(mktemp /tmp/urchin.XXXXXXXX)
  if test -f "${tmp}"; then
    rm "${tmp}"
  fi
  mkdir "${tmp}"
  echo "${tmp}"
}
mktemp_file() {
  tmp=$(mktemp /tmp/urchin.XXXXXXXX)
  if ! test -f "${tmp}"; then
    > "${tmp}"
  fi
  echo "${tmp}"
}

# -------------------- Temporary directory -------------------- 
urchin_tmp=$(mktemp_dir)
> "${urchin_tmp}/log"

urchin_exit() {
  rm -Rf "${urchin_tmp}"
  exit "$@"
}

# -------------------- Utilities -------------------- 
epoch_date() {
  date +%s
}
epoch_pax() {
  # Based on http://stackoverflow.com/a/7262588/407226
  tmp="$(mktemp_file)"
  echo "ibase=8;$({ pax -wx cpio "${tmp}"; echo; } | cut -c 48-59)" | bc
  rm "${tmp}"
}

plural () {
  # Make $1 a plural according to the number $2.
  # If $3 is supplied, use that instead of "${1}s".
  # Result is written to stdout.
  if [ "${2}" = 1 ]; then
    echo "${1}"
  else
    echo "${3-${1}s}"
  fi
}

has_shebang_line() {
  head -n 1 "${1}" | grep -v '^#!/bin/sh$' | grep -q '^#!'
}

indent() {
  level="${1}"
  if test "${level}" -gt 0; then
    printf "%$((2 * ${level}))s"
  fi
}

stdout_file() {
  the_test="${1}"
  the_shell="${2}"

  x="${urchin_tmp}/stdout$(fullpath "$the_test")"
  mkdir -p "${x}"
  case "${urchin_md5}" in
    md5sum) y=$(echo "${the_shell}" | md5sum | cut -d\  -f1) ;;
    md5) y=$(echo "${the_shell}" | md5 | sed 's/.* //') ;;
    *) echo md5 command is not configured >&2; urchin_exit 1;;
  esac
  echo "${x}/${y}"
}

# Expand relative paths
fullpath() {
  readlink -f -- "${1}"
}

contains() {
  case "$#" in
    1) grep "${1}" > /dev/null ;;
    2) echo "${1}" | contains "${2}" ;;
    3) contains "${1}" "${2}" && contains "${1}" "${3}" ;;
    *) container="${1}" && shift
       contains "${container}" "${1}" && shift &&
       contains "${container}" "${@}" ;;
   esac
}

is_set() {
  set | grep "^${1}=" > /dev/null
}

remove_trailing_slash() {
  echo "$1" | sed s/\\/$//
}

# Find the root directory of the present test suite
urchin_root() {
  # Call recursively but remember the original argument.
  orig="${2:-$1}"
  current="${1%/}"

  abscurrent="$(fullpath "${current}")"
  if test "${abscurrent}" = / ||
    basename "${abscurrent}" | contains '^\.' ; then
    # Stop traversing upwards at / and at hidden directories.
    if test -d "${orig}"; then
      echo "${orig}"
    else
      dirname -- "${orig}"
    fi
  elif ! test -e "${current}"; then
    echo "${current}: No such file or directory">&2
    return 1
  elif test -f "${current}"; then
    urchin_root "$(dirname -- "${current}")" "${orig}"
  elif test -f "${current}"/.urchin_root; then
    echo "${current}"
  else
    urchin_root "${current}"/.. "${orig}"
  fi
}

# -------------------- Metafunctions -------------------- 
meta_verbosity() {
  echo "if test \${${1}} -gt ${2}; then ${3}=true; fi"
}

# -------------------- Printing output -------------------- 
# Format functions may read a log file from stdin.

format_tap() {
  v="${1}"
  tmp_dir="${2}"

  $(verbosity v 2 print_not_ok_stdout)
  $(verbosity v 3 print_ok_stdout)

  while IFS="${HT}" read remote the_shell path result file_elapsed; do
    # Number of files that have run, including this one
    n=$(( ${n:-0} + 1))

    if test -z "${the_shell}"; then
      the_shell='File is not executable.'
    fi
    case "${result}" in
      ok) "ok $n - ${path} (${the_shell}${remote})" ;;
      not_ok) "not_ok $n - ${path} (${the_shell}${remote})" ;;
      skip) "ok $n - ${path} (${the_shell}${remote}) # SKIP" ;;
    esac

    if { test "${result}" = not_ok && "${print_not_ok_stdout}"; } ||
       { test "${result}" = ok && "${print_ok_stdout}"; }; then
      echo '# ------------ Begin output ------------'
      sed 's/^/# /' "$(stdout_file "${path}" "${the_shell}")"
      echo '# ------------ End output ------------'
    fi
    echo "# Previous test took ${file_elapsed} seconds."

    # Number of files that have been ok, not ok, and skipped
    eval "old_count=${result}s"
    eval "${result}s=$(($old_count+1))"

  done
}

format_urchin() {
  v="${1}"
  tmp_dir="${2}"
  verbosity="${3}"
  print_in_color="${4}"

  $(verbosity v 1 print_margins)
  $(verbosity v 1 print_not_ok)
  $(verbosity v 2 print_not_ok_stdout)
  $(verbosity v 2 print_ok)
  $(verbosity v 3 print_ok_stdout)

  if $print_in_color; then
    success_mark=$(printf "\033[32m✓ \033[0m")
    fail_mark=$(printf "\033[31m✗ \033[0m")
  else
    success_mark=.\ 
    fail_mark=F\ 
  fi

  header() {
    if test "${prevdir}" != "${currentdir}"; then
      echo
    fi
    if test "${prevpath}" != "${path}"; then
      printf "$(dirname -- "${path}")/\n> $(basename -- "${path}")\n"
    fi
  }

  while IFS="${HT}" read remote the_shell path result file_elapsed; do
    abspath=${urchin_tmp}/${path}

    currentdir="$(dirname -- "${path}")"
    prevdir="${currentdir}"

    case "${result}" in
      ok)
        if "${print_ok}"; then
          header
          printf "${success_mark} "
          echo "${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
        fi
      ;;
      not_ok)
        if "${print_not_ok}"; then
          header
          printf "${fail_mark} "
          echo "${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
        fi
      ;;
      skip)
        if "${print_ok}"; then
          header
          if test -z "${the_shell}"; then
            echo '  (Specified shell is not executable.)'
          else
            echo "  ${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
          fi
        fi
      ;;
    esac
    if { test "${result}" = not_ok && "${print_not_ok_stdout}"; } ||
       { test "${result}" = ok && "${print_ok_stdout}"; }; then
      sed 's/^/  | /' "$(stdout_file "${path}" "${the_shell}")"
    fi
    prevpath="${path}"
  done
}

# -------------------- Main stuff -------------------- 
recurse() {
  requested_path="${1}"
  potential_test="$(fullpath "${2}")"
  cycle_shell="${3}"
  TEST_SHELL="${4}"

  root="$(urchin_root "${1}")"

  for ignore in setup_dir teardown_dir setup teardown; do
    if test "$(basename "${potential_test}")" = "${ignore}"; then
      return
    fi
  done

  # Return if we should not run this file
  if contains "${potential_test}" "^${requested_path}" ||
     contains "${requested_path}" "^${potential_test}" ; then
    if test "$(dirname "${potential_test}")" = \
            "$(dirname "${requested_path}")" &&
       test "${potential_test}" != "${requested_path}"; then
      return 0
    fi
  else
    return 0
  fi

  if contains "${potential_test}" "${HT}"; then
    echo 'Test file names may not contain tabs (HT).' >&2
    urchin_exit 11
  fi

  if [ -x "${potential_test}" ]; then
    if [ -d "${potential_test}" ]; then
    (
      cd -- "${potential_test}"
      if test -f .urchin_dir && grep series .urchin_dir > /dev/null; then
        run_in_series_dir=true
      else
        run_in_series_dir=false
      fi

      if test -f setup_dir; then
        . ./setup_dir
      fi

      for test in *; do
        if test "${test}" = '*' && ! test -e "${test}"; then
          # The directory is empty.
          break
        fi

        recurse "${requested_path}" "${test}" "${cycle_shell}" \
          "${TEST_SHELL}" &

        if "${run_in_series}" || "${run_in_series_dir}"; then
          if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
          if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
            if test -f teardown_dir; then
              . ./teardown_dir
            fi
            return 1
          fi
        fi
      done
      wait
      if test -f teardown_dir; then
        . ./teardown_dir
      fi
    )
    elif [ -f "${potential_test}" ]; then
      cd -- "$(dirname -- "${potential_test}")"

      # Determine the environment variable to define for test scripts
      # that reflects the specified or implied shell to use for shell-code tests.
      while read the_test_shell; do
        (
          if test -f setup; then
            . ./setup
          fi

          # Run the test
          start=$("${epoch}")
          set +e
          { 
            if "${cycle_shell}"; then
              if has_shebang_line "${potential_test}"; then
                TEST_SHELL="${the_test_shell}" $TIMEOUT "${potential_test}"
              else
                TEST_SHELL="${the_test_shell}" $TIMEOUT \
                  "${the_test_shell}" "${potential_test}"
              fi
            else
              # Shell cycling is disabled with -n; use the present value of
              # TEST_SHELL or default to /bin/sh
              if [ -n "${TEST_SHELL}" ]; then
                $TIMEOUT "${potential_test}"
              else
                TEST_SHELL=/bin/sh $TIMEOUT "${potential_test}"
              fi
            fi
          } > "$(stdout_file "${potential_test}" "${the_test_shell}")" 2>&1
          exit_code="${?}"
          set -e
          finish=$("${epoch}")

          if test -f teardown; then
            . ./teardown
          fi

          if [ "${exit_code}" -eq 0 ]; then
            result=ok
          elif [ "${exit_code}" -eq 3 ]; then
            result=skip
          else
            result=not_ok
          fi

          elapsed=$(($finish - $start))
          rel="${potential_test##"${root}/"}"
          printf "\t${the_test_shell}\t${rel}\t${result}\t${elapsed}\n" \
            >> "${urchin_tmp}"/log
          exit "${exit_code}"
        ) &

        if "${run_in_series}"; then
          if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
          if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
            if test -f teardown_dir; then
              . ./teardown_dir
            fi
            return 1
          fi
        fi
      done < "${shell_list}"
      wait
    else
      echo "${potential_test}: Neither file nor directory!?" >&2
    fi
  else
    # Shell is ''
    printf "\t\t${potential_test}\tskip\t0\n" >> "${urchin_tmp}"/log
  fi
}

# This should not require the root variable; fix the paths before they
# get to the log file.
report_outcome() {
  root="$(fullpath "${1}")"
  format="${2}"
  log_file="${3}"
  start="${4}"
  finish="${5}"

  elapsed=$(($finish - $start))
  for number in n oks skips not_oks; do
    eval "${number}=0"
  done

  # Use a temporary file rather than a pipe because a pipe starts a sub-shell
  # and thus makes the above variables local.
  sorted_log_file=$(mktemp_file)
  cat "${log_file}" | LC_COLLATE=C sort > "${sorted_log_file}"

  rm "${sorted_log_file}"

  if test "${format}" = tap; then
    echo "# Full test suite took ${elapsed} $(plural second ${elapsed})."
    echo 1.."${n}"
  elif test "${format}" = urchin; then
    if "${print_margins}"; then
      echo
      echo "Done, took ${elapsed} $(plural second ${elapsed})."
      echo "${oks} $(plural test "${oks}") passed."
      echo "${skips} $(plural test "${skips}") skipped."
      echo "${not_oks} $(plural test "${not_oks}") failed."
    fi
  fi
  test "${not_oks}" -eq '0'
}


main() {
  cycle_shell=true
  shell_list="${urchin_tmp}"/shell_list
  test_arg_list="${urchin_tmp}"/test_list
  > "${test_arg_list}"
  run_in_series=false
  force=false
  exit_on_not_ok=false
  format=urchin
  print_in_color=false

  print_margins=true
  print_ok=false
  print_not_ok=true
  print_ok_stdout=false
  print_not_ok_stdout=false
  while [ "${#}" -gt 0 ]
  do
      case "${1}" in
          -b|--run-in-series) run_in_series=true;;
          -e|--exit-on-fail) exit_on_not_ok=true;;
          -f|--force) force=true;;
          -s|--shell)
            shift
            shell_for_sh_tests="${1}"
           
            command -v "${shell_for_sh_tests}" > /dev/null || {
              echo "Cannot find specified shell: '${shell_for_sh_tests}'" >&2
              urchin_help >&2
              urchin_exit 11
            }

            if contains "${potential_test}" "${HT}"; then
              echo 'Shell paths may not contain tab characters (HT).' >&2
              urchin_exit 11
            fi

            if contains "${shell_for_sh_tests}" "[${IFS}]"; then
              echo "Warning: It is best if field-separator characters
(usually spaces) are absent from shell paths so that
you don't need to quote the TEST_SHELL variable." >&2
            fi

            echo "${shell_for_sh_tests}" >> "${shell_list}"
            ;;
          -n|--disable-cycling) cycle_shell=false;; 
          -F|--format) shift ; format="${1}";;
          -T|--timeout)
            shift
            urchin_timeout="${1}" 
            if ! contains "${urchin_timeout}" '[0-9][0-9.]*\(s\|m\|h\|d\|\)' ; then
              echo Bad timeout argument: "${urchin_timeout}" >&2
              urchin_exit 1
            fi ;;
          -p|--pretty) print_in_color=true;;

          -q|--quiet)    print_not_ok=false
                         print_margins=false;; 
          -v)            print_not_ok_stdout=true;;
          -vv)           print_not_ok_stdout=true
                         print_ok=true;;
          -vvv|--verbose)print_not_ok_stdout=true
                         print_ok=true;
                         print_ok_stdout=true;;
          -vvvv|--debug) print_not_ok_stdout=true
                         print_ok=true;
                         print_ok_stdout=true
                         set -x;;

          -h|--help) urchin_help
                     urchin_exit 0;;
          --version) echo "${VERSION}"
              urchin_exit;;
          -*) urchin_help >&2
              urchin_exit 11;;
          *)  if contains "${1}" "${HT}"; then
                echo 'Test file names may not contain tab characters (HT).' >&2
                urchin_exit 11
              elif [ ! -e "${1}" ]; then
                echo "No such file or directory: '${1}'" >&2
                echo "${USAGE}" >&2
                urchin_exit 11
              elif ! {
                # Molly guard
                root="$(urchin_root "${1}")"
                basename "$(fullpath "${root}")" |
                grep -i 'test' > /dev/null || "${force}"
              }; then
                echo 'The root directory of the tests that you are running urchin on
does not contain the word "test", so I am not running,
in case that was an accident. Use the -f flag if you really
want to run urchin on that directory.' >&2
                urchin_exit 12
              fi
              echo "${1}" >> "${test_arg_list}" ;;
      esac
      shift
  done
  if "${RUN_IN_SERIES}" 2> /dev/null; then
    run_in_series=true
  fi

  # -------------------- VALIDATE INPUT -------------------- #
  if ! "${cycle_shell}" && ! is_set shell_list; then
    echo "The -n/--disable-cycling and -s/--shell options clash with each other." >&2
    urchin_exit 11
  fi

  # If -s was not passed, use the available default shells.
  if test -z "${shell_list}"; then
    if $cycle_shell; then
      for shell in $DEFAULT_SHELLS; do
        if command -v "${shell}" 1> /dev/null 2> /dev/null; then
          shell_list="${shell}${HT}${shell_list}"
        fi
      done
    fi
  fi

  if $print_margins; then
    if test "${format}" = tap; then
      printf \#\ >> "${urchin_tmp}"/head
    fi
    if test "${format}" = urchin || test "${format}" = tap; then
      echo Running tests at $(date +%Y-%m-%dT%H:%M:%S) >> "${urchin_tmp}"/head
    fi

    if test "${format}" = tap; then
      printf '# ' >> "${urchin_tmp}"/head
    fi
    if test "${format}" = urchin || test "${format}" = tap; then
      printf 'Cycling with the following shells: ' >> "${urchin_tmp}"/head
    fi
    cat "${shell_list}" | tr '\n' \  >> "${urchin_tmp}"/head
    echo >> "${urchin_tmp}"/head
  fi

  if is_set urchin_timeout; then
    # Choose the timeout command
    if timeout -t 0 true 2> /dev/null; then
      TIMEOUT="timeout -t ${urchin_timeout}"
    elif timeout 0 true 2> /dev/null; then
      TIMEOUT="timeout ${urchin_timeout}"
    else
      echo I couldn\'t figure out how to use your version of timeout >&2
      urchin_exit 1
    fi
  fi

  if "${exit_on_not_ok}" && ! "${run_in_series}"; then
    echo 'You must also pass -b/--series in order to use -e/--exit-on-fail.' >&2
    urchin_exit 11
  fi

  # --------------------   REALLY RUN   -------------------- #
  start=$("${epoch}")

  # 1 test file or folder to run
  # 2 urchin root
  # 3 Should we cycle shells?
  # 4 TEST_SHELL
  while read seed; do
    recurse "$(fullpath "${seed}")" "${root}" "${cycle_shell}" \
      "${TEST_SHELL}" || break
  done < "${test_arg_list}"
  finish=$("${epoch}")

  if test $(cat "${urchin_tmp}"/log | wc -l) -eq 0; then
    echo 'No tests found' >&2
    urchin_exit 1
  fi
    
  cat "${urchin_tmp}"/head
  report_outcome "${root}" "${format}" "${urchin_tmp}"/log "${start}" \
    "${finish}"
# cat "${urchin_tmp}"/foot
  urchin_exit "${?}"
}

is_set TESTING_URCHIN_INTERNALS || main "$@"