urchin/urchin

#!/bin/sh

# ----------------------------------------------------------------------
# Copyright (c) 2013, 2014, 2015, 2016 Thomas Levine
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as 
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ----------------------------------------------------------------------


# ----------------------------------------------------------------------
# Copyright (c) 2014, Michael Klement
# Copyright (c) 2012, ScraperWiki Limited
# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 
# Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------

set -e

# Delimiters
LF="$(printf '\n')"
HT="$(printf '\t')"

# Urchin version number
VERSION=0.0.0-master

# Kill subprocesses on interrupt.
trap "kill -$$; exit" HUP INT TERM

DEFAULT_SHELLS='
sh
bash
dash
ksh
posh
pdksh
mksh
yash
zsh
'
if [ -n "${ZSH_VERSION}" ]; then
  # avoid "no matches found: *" error when directories are empty
  setopt NULL_GLOB
  emulate sh
fi

# -------------------- Usage -------------------- 
USAGE="usage: $0 [options]... [test file or directory]..."

urchin_help() {
  cat <<EOF

${USAGE}

By default, Urchin checks for the following shells and runs every
particular test file once per shell.

  $(echo "${DEFAULT_SHELLS}" | sed 's/ /\n  /g')

On each run,

1. The TEST_SHELL environment variable is set to the particular shell.
2. If the test file lacks a shebang line or has a shebang line of
   "#!/bin/sh", the test script is also executed in that shell.

The following flags affect how this multiple-shell testing is handled.

-s, --shell <shell>      Tell Urchin to use a different list of shells.
                         (You can pass this flag multiple times.)

The following flags affect how Urchin processes tests.

-b, --run-in-series      Run tests in series. The default is to run tests
                         in parallel where possible.
-e, --exit-on-fail       Stop running if any single test fails.
                         This can be useful if you are running something
                         other than test files with Urchin.
-T, --timeout <seconds>  Kill a test if it runs for longer than the
                         specified duration. The default is no timeout.
-f, --force              Force running even if the test directory's name
                         does not contain the word "test".

These options affect how results are formatted. Options -q, and -v
have no effect when combined with formats other than "urchin".
-vv, -vvv, and -vvvv do have effect when combined with formats "urchin"
or "tap".

-p, --pretty             Print results in color and with fancy symbols.
-F, --format <name>      XXX

And these options affect how much is printed.

-q, --quiet              Print nothing to stdout;
                         the only output is the exit code.
(default verbosity)      Print names of failed tests and counts
                         of passed, failed, and skipped tests.
-v                       Print stdout from failing tests.
-vv                      Print names of passed tests.
-vvv, --verbose          Print stdout from all tests.
-vvvv, --debug           Run with set -x.

The remaining flags provide information about urchin.

-h, --help               Display this help.
--version                Display the version number.

Urchin recognizes certain environment variables.

TEST_SHELL           This is sometimes over-ridden; see -s.
RUN_IN_SERIES        Set this to have the same effect as
                     -b/--run-in-series. This is helpful if you are
                     calling urchin inside an urchin test suite.

Exit codes have the following meanings

0                    All tests were ok
1                    At least one test was not ok.
2                    No tests were found.
10                   Dependencies are missing (locally, not on remotes).
11                   Flags were not valid.
12                   File names contain unsupported delimiters (HT or LF).
13                   An test shell specified with -s/--shell is not available.

Go to https://thomaslevine.com/!/urchin/ for documentation on writing tests.

EOF
}

# -------------------- Dependency checks -------------------- 
if command -v md5 1> /dev/null 2> /dev/null; then
  urchin_md5=md5
elif command -v md5sum 1> /dev/null 2> /dev/null; then
  urchin_md5=md5sum
else
  echo Could not find MD5 hash command >&2
  exit 10
fi

if epoch_date 2>&1 > /dev/null; then
  epoch=epoch_date
elif epoch_pax 2>&1 > /dev/null; then
  epoch=epoch_pax
else
  echo I could not find a seconds counter. >&2
  exit 10
fi


# -------------------- Portable wrappers -------------------- 
mktemp_dir() {
  # Support HP-UX mktemp that has wrong exit codes and
  # can't make directories.
  tmp=$(mktemp /tmp/urchin.XXXXXXXX)
  if test -f "${tmp}"; then
    rm "${tmp}"
  fi
  mkdir "${tmp}"
  echo "${tmp}"
}
mktemp_file() {
  tmp=$(mktemp /tmp/urchin.XXXXXXXX)
  if ! test -f "${tmp}"; then
    > "${tmp}"
  fi
  echo "${tmp}"
}

md5 () {
  case "${urchin_md5}" in
    md5sum) echo "${1}" | md5sum | sed 's/ .*//' ;;
    md5) echo "${1}" | md5 | sed 's/.* //' ;;
  esac
}

# -------------------- Utilities -------------------- 
epoch_date() {
  date +%s
}
epoch_pax() {
  # Based on http://stackoverflow.com/a/7262588/407226
  tmp="$(mktemp_file)"
  echo "ibase=8;$({ pax -wx cpio "${tmp}"; echo; } | cut -c 48-59)" | bc
  rm "${tmp}"
}

plural () {
  # Make $1 a plural according to the number $2.
  # If $3 is supplied, use that instead of "${1}s".
  # Result is written to stdout.
  if [ "${2}" = 1 ]; then
    echo "${1}"
  else
    echo "${3-${1}s}"
  fi
}

has_shebang_line() {
  head -n 1 "${1}" | grep -v '^#!/bin/sh$' | grep -q '^#!'
}

indent() {
  level="${1}"
  if test "${level}" -gt 0; then
    printf "%$((2 * ${level}))s"
  fi
}

# Expand relative paths
fullpath() {
  readlink -f -- "${1}"
}

contains() {
  case "$#" in
    1) grep "${1}" > /dev/null ;;
    2) echo "${1}" | contains "${2}" ;;
    3) contains "${1}" "${2}" && contains "${1}" "${3}" ;;
    *) container="${1}" && shift
       contains "${container}" "${1}" && shift &&
       contains "${container}" "${@}" ;;
   esac
}

is_set() {
  set | grep "^${1}=" > /dev/null
}

remove_trailing_slash() {
  echo "$1" | sed s/\\/$//
}

# File where a test's stdout and stderr is saved
stdout_file() {
  tmp_dir="${1}"
  the_test="${2}"
  the_shell="${3}"

  x="${tmp_dir}/stdout$(fullpath "$the_test")"
  mkdir -p "${x}"
  echo "${x}/$(md5 "${the_shell}")"
}

# Root directory of the present test suite
# USAGE: test_suite_root <directory>
test_suite_root() {
  # Call recursively but remember the original argument.
  orig="${2:-$1}"
  current="${1%/}"

  abscurrent="$(fullpath "${current}")"
  if test "${abscurrent}" = / ||
    basename "${abscurrent}" | contains '^\.' ; then
    # Stop traversing upwards at / and at hidden directories.
    if test -d "${orig}"; then
      echo "${orig}"
    else
      dirname -- "${orig}"
    fi
  elif ! test -e "${current}"; then
    echo "${current}: No such file or directory">&2
    return 1
  elif test -f "${current}"; then
    test_suite_root "$(dirname -- "${current}")" "${orig}"
  elif test -f "${current}"/.urchin_root; then
    echo "${current}"
  else
    test_suite_root "${current}"/.. "${orig}"
  fi
}

# -------------------- Metafunctions -------------------- 
meta_verbosity() {
  echo "if test \${${1}} -ge ${2}; then ${3}=true; fi"
}

# -------------------- Printing output -------------------- 
# Format functions may read a log file from stdin.

format_tap() {
  v="${1}"
  tmp_dir="${2}"
  elapsed="${3}"

  $(verbosity v 2 print_not_ok_stdout)
  $(verbosity v 3 print_ok_stdout)

  print_stdout() {
    echo '# ------------ Begin output ------------'
    sed 's/^/# /' "$(stdout_file "${tmp_dir}" "${path}" "${the_shell}")"
    echo '# ------------ End output ------------'
  }

  while IFS="${HT}" read remote the_shell path result file_elapsed; do
    # Number of files that have run, including this one
    n=$(( ${n:-0} + 1))

    case "${result}" in
      ok) echo "ok $n - ${path} (${the_shell}${on})"
          if "${print_ok_stdout}"; then print_stdout; fi ;;
      not_ok) echo "not_ok $n - ${path} (${the_shell}${on})"
              if "${print_not_ok_stdout}"; then print_stdout; fi ;;
      skip) "ok $n - ${path} (${the_shell}${on}) # SKIP" ;;
    esac
    echo "# Previous test took ${file_elapsed} seconds."

  done

  echo "# Full test suite took ${elapsed} $(plural second ${elapsed})."
  echo 1.."${n}"
}

format_urchin() {
  v="${1}"
  tmp_dir="${2}"
  verbosity="${3}"
  print_in_color="${4}"

  $(verbosity v 1 print_margins)
  $(verbosity v 1 print_not_ok)
  $(verbosity v 2 print_not_ok_stdout)
  $(verbosity v 2 print_ok)
  $(verbosity v 3 print_ok_stdout)

  if $print_in_color; then
    success_mark=$(printf "\033[32m✓ \033[0m")
    fail_mark=$(printf "\033[31m✗ \033[0m")
  else
    success_mark=.\ 
    fail_mark=F\ 
  fi

  header() {
    if test "${prevdir}" != "${currentdir}"; then
      echo
    fi
    if test "${prevpath}" != "${path}"; then
      printf "$(dirname -- "${path}")/\n> $(basename -- "${path}")\n"
    fi
  }

  print_stdout() {
    sed 's/^/  | /' "$(stdout_file "${tmp_dir}" "${path}" "${the_shell}")"
  }

  while IFS="${HT}" read remote the_shell path result file_elapsed; do
    abspath=${tmp_dir}/${path}

    currentdir="$(dirname -- "${path}")"
    prevdir="${currentdir}"

    # Format the message
    if test -z "${remote}"; then
      on=" on ${remote}"
    else
      on=
    fi
    if test result = skip; then
      parantheses="(skipped)"
    else
      parantheses="(${file_elapsed} $(plural second "${file_elapsed}"))"
    fi
    message="${the_shell}${on} (${file_elapsed} ${unit})"

    # Keep track of how many files have been ok, not ok, and skipped.
    eval "${result}s=$((${result}s+1))"

    # Print the result.
    case "${result}" in
      ok) if "${print_ok}"; then
            header && echo "${success_mark} ${message}"
          fi ;;
      not_ok) if "${print_not_ok}"; then
                header && echo "${fail_mark} ${message}"
                if "${print_not_ok_stdout}"; then print_stdout; fi
              fi ;;
      skip) if "${print_ok}"; then
              header && echo "${skip_mark} ${message}"
              if "${print_ok_stdout}"; then print_stdout; fi
            fi ;;
    esac
    prevpath="${path}"
  done

  if "${print_margins}"; then
    echo
    echo "Done, took ${elapsed} $(plural second ${elapsed})."
    echo "${oks} $(plural test "${oks}") passed."
    echo "${skips} $(plural test "${skips}") skipped."
    echo "${not_oks} $(plural test "${not_oks}") failed."
  fi
}

# -------------------- Main stuff -------------------- 

# Return codes have the following meaning
# 0) All tests succeded
# 1) At least one test failed.
# *) Something else went wrong; Urchin should exit with the same code.
recurse() {
  requested_path="${1}"
  potential_test="$(fullpath "${2}")"
  cycle_shell="${3}"
  TEST_SHELL="${4}"

  root="$(test_suite_root "${1}")"

  for ignore in setup_dir teardown_dir setup teardown; do
    if test "$(basename "${potential_test}")" = "${ignore}"; then
      return
    fi
  done

  # Return if we should not run this file
  if contains "${potential_test}" "^${requested_path}" ||
     contains "${requested_path}" "^${potential_test}" ; then
    if test "$(dirname "${potential_test}")" = \
            "$(dirname "${requested_path}")" &&
       test "${potential_test}" != "${requested_path}"; then
      return 0
    fi
  else
    return 0
  fi

  if contains "${potential_test}" "${HT}"; then
    echo 'Test file names may not contain tabs (HT).' >&2
    return 11
  fi

  if [ -x "${potential_test}" ]; then
    if [ -d "${potential_test}" ]; then
    (
      cd -- "${potential_test}"
      if test -f .urchin_dir && grep series .urchin_dir > /dev/null; then
        run_in_series_dir=true
      else
        run_in_series_dir=false
      fi

      if test -f setup_dir; then
        . ./setup_dir
      fi

      for test in *; do
        if test "${test}" = '*' && ! test -e "${test}"; then
          # The directory is empty.
          break
        fi

        recurse "${requested_path}" "${test}" "${TEST_SHELL}" &

        if "${run_in_series}" || "${run_in_series_dir}"; then
          if ! wait "${!}"; then
            return_code="$?"
            if "${exit_on_not_ok}"; then
              if test -f teardown_dir; then
                . ./teardown_dir
              fi
              return "${return_code}"
            fi
          fi
        fi
      done
      wait
      if test -f teardown_dir; then
        . ./teardown_dir
      fi
    )
    elif [ -f "${potential_test}" ]; then
      cd -- "$(dirname -- "${potential_test}")"
      echo "${shell_list}" | while read the_test_shell; do
        (
          if test -f setup; then
            . ./setup
          fi

          # Run with a shell?
          if has_shebang_line "${potential_test}"; then
            set -- "${potential_test}"
          else
            set -- "${the_test_shell}" "${potential_test}"
          fi
          out_file="$(stdout_file "${tmp_dir}" "${potential_test}" "${the_test_shell}")"

          # Run the test
          start=$("${epoch}")
          set +e
          TEST_SHELL="${the_test_shell}" $TIMEOUT "$@"  > "${out_file}" 2>&1
          exit_code="${?}"
          set -e
          finish=$("${epoch}")

          if test -f teardown; then
            . ./teardown
          fi

          case "${exit_code}" in
            0) result=ok ;;
            3) result=skip ;;
            *) result=not_ok ;;
          esac

          elapsed=$(($finish - $start))
          rel="${potential_test##"${root}/"}"
          printf "\t${the_test_shell}\t${rel}\t${result}\t${elapsed}\n" \
            >> "${urchin_tmp}"/log
          exit "${exit_code}"
        ) &

        if "${run_in_series}"; then
          if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
          if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
            if test -f teardown_dir; then
              . ./teardown_dir
            fi
            return 1
          fi
        fi
      done
      wait
    fi
  else
    # Skip because the file is not executable.
    echo "${shell_list}" | while read sh; do
      printf "\t${sh}\t${potential_test}\tskip\t0\n" >> "${urchin_tmp}"/log
    done
  fi
}

report_outcome() {
  format="${2}"
  log_file="${3}"
  start="${4}"
  finish="${5}"

  elapsed=$(($finish - $start))

  # Use a temporary file rather than a pipe because a pipe starts a sub-shell
  # and thus makes the above variables local.
  sorted_log_file=$(mktemp_file)
  cat "${log_file}" | LC_COLLATE=C sort > "${sorted_log_file}"

  rm "${sorted_log_file}"

  test "${not_oks}" -eq '0'
}


main() {
  # Defaults
  format=urchin
  if "${RUN_IN_SERIES}" 2> /dev/null; then
    run_in_series=true
  fi

  # Shift if possible; error otherwise.
  flag_arg() {
    flag="${1}"
    if shift; then
      echo "${1}"
    else
      echo Missing argument for "${flag}" >&2
      exit 11
    fi
  }

  # Receive input
  while [ "${#}" -gt 0 ]; do
    case "${1}" in
        -b|--run-in-series) run_in_series=true;;
        -e|--exit-on-fail) exit_on_not_ok=true;;
        -f|--force) force=true;;

        -F|--format) format="$(flag_arg)" ;;
        -p|--pretty) print_in_color=true;;

        -q|--quiet)     verbosity=0 ;;
        -v)             verbosity=2 ;;
        -vv)            verbosity=3 ;;
        -vvv|--verbose) verbosity=4 ;;
        -vvvv|--debug)  verbosity=5 ;;

        --version) echo "${VERSION}" && exit;;
        -h|--help) urchin_help && exit 0;;

        -s|--shell) sh="$(flag_arg)"
          if ! command -v "${sh}" > /dev/null; then
            echo "Cannot find specified shell: '${sh}'" >&2
            urchin_help >&2
            exit 13
          elif contains "${potential_test}" "${HT}" "${LF}"; then
            echo 'Shell paths may contain all characters other than' >&2
            echo 'horizontal tab (\t) and line feed (\n).' >&2
            exit 11
          elif contains "${sh}" "[${IFS}]"; then
            echo "Warning: It is best if field-separator characters
(usually spaces) are absent from shell paths so that
you don't need to quote the TEST_SHELL variable." >&2
          fi
          shell_list="${sh}${LF}${shell_list}" ;;

        -T|--timeout) urchin_timeout="$(flag_arg)" 
          if ! contains "${urchin_timeout}" \
            '[0-9][0-9.]*\(s\|m\|h\|d\|\)' ; then
            echo Bad timeout argument: "${urchin_timeout}" >&2
            exit 11
          fi ;;

        -*) urchin_help >&2 && exit 11;;
        *) if contains "${1}" "${HT}" "${LF}"; then
             echo 'Test file names may contain all characters other than' >&2
             echo 'horizontal tab (\t) and line feed (\n).' >&2
             exit 11
           elif [ ! -e "${1}" ]; then
             echo "No such file or directory: '${1}'" >&2
             echo "${USAGE}" >&2
             exit 11
           elif ! {
             # Molly guard
             root="$(test_suite_root "${1}")"
             basename "$(fullpath "${root}")" |
             grep -i 'test' > /dev/null || "${force}"
           }; then
             echo 'The root directory of the tests that you are running urchin on
doesnot contain the word "test", so I am not running,
in case that was an accident. Use the -f flag if you really
wantto run urchin on that directory.' >&2
             exit 12
           fi
           test_seeds="${1}${LF}${test_seeds}" ;;
    esac
    shift
  done

  # If -s was not passed, use the available default shells.
  if ! is_set "${shell_list}"; then
    if $cycle_shell; then
      for shell in $DEFAULT_SHELLS; do
        if command -v "${shell}" 1> /dev/null 2> /dev/null; then
          shell_list="${shell}${HT}${shell_list}"
        fi
      done
    fi
  fi

  if is_set urchin_timeout; then
    # Choose the timeout command
    if timeout -t 0 true 2> /dev/null; then
      TIMEOUT="timeout -t ${urchin_timeout}"
    elif timeout 0 true 2> /dev/null; then
      TIMEOUT="timeout ${urchin_timeout}"
    else
      echo I couldn\'t figure out how to use your version of timeout >&2
      exit 10
    fi
  fi

  if "${exit_on_not_ok}" && ! "${run_in_series}"; then
    echo 'You must also pass -b/--series in order to use -e/--exit-on-fail.' >&2
    exit 11
  fi

  # --------------------   REALLY RUN   -------------------- #

  # Temporary files
  urchin_tmp=$(mktemp_dir)

  # Write header information
  echo Running tests at $(date +%Y-%m-%dT%H:%M:%S) >> "${urchin_tmp}"/head
  printf 'Cycling with the following shells: ' >> "${urchin_tmp}"/head
  echo "${shell_list}" | tr "${HT}" \  >> "${urchin_tmp}"/head
  echo >> "${urchin_tmp}"/head

  start=$("${epoch}")
  echo "${test_seeds}" | while read seed; do
    set +e
    recurse "${root}" "$(fullpath "${seed}")""${TEST_SHELL}"
    return_code=$?
    set -e
    test "${return_code}" -eq 0 || break
  done
  finish=$("${epoch}")

  if test "${return_code}" -le 1; then
    if test -f "${urchin_tmp}"/log ; then
      echo Print output.
    # cat "${urchin_tmp}"/head
    # report_outcome "${root}" "${format}" "${urchin_tmp}"/log "${start}" \
    #   "${finish}"
    # cat "${urchin_tmp}"/foot
    else
      echo 'No tests found' >&2
      return_code=2
    fi
  fi

  rm -Rf "${urchin_tmp}"
  exit "${exit_code}"
}

is_set TESTING_URCHIN_INTERNALS || main "$@"