urchin/urchin

#!/bin/sh

# Copyright (c) 2013, 2014, 2015, 2016 Thomas Levine
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as 
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.


# Copyright (c) 2014, Michael Klement
# Copyright (c) 2012, ScraperWiki Limited
# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 
# Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

set -e

DEFAULT_SHELLS='sh bash dash mksh zsh'
if [ -n "$ZSH_VERSION" ]; then
  # avoid "no matches found: *" error when directories are empty
  setopt NULL_GLOB

  emulate sh
fi


# Make sure that CDPATH isn't set, as it causes `cd` to behave unpredictably -
# notably, it can produce output.
unset CDPATH

# All temporary files go here
urchin_tmp=$(mktemp -d)
> $urchin_tmp/log
urchin_exit() {
  rm -Rf "$urchin_tmp"
  exit "$@"
}

stdout_file() {
  the_test="$1"
  the_shell="$2"

  x="$urchin_tmp/stdout$(fullpath "$the_test")"
  mkdir -p "$x"
  echo "$x/$(echo "$the_shell" | md5sum | cut -d\  -f1)"
}

# Expand relative paths
fullpath() {
  readlink -f -- "$1"
}

remove_trailing_slash() {
  echo "$1" | sed s/\\/$//
}

urchin_root() {
  # Call recursively but remember the original argument.
  current="$(remove_trailing_slash "$1")"
  if test -n "$2"; then
    orig="$2"
  else
    orig="$1"
  fi

  abscurrent="$(fullpath "$1")"
  if test "$abscurrent" = / ||
    basename "$abscurrent" | grep '^\.' > /dev/null; then
    # Stop traversing upwards at / and at hidden directories.
    if test -d "$orig"; then
      echo "$orig"
    else
      dirname -- "$orig"
    fi
  elif ! test -e "$current"; then
    echo "$current: No such file or directory">&2
    return 1
  elif test -f "$current"; then
    urchin_root "$(dirname -- "$current")" "$orig"
  elif test -f "$current"/.urchin; then
    remove_trailing_slash "$current"
  else
    urchin_root "$current"/.. "$orig"
  fi
}

# Urchin version number
VERSION=0.1.0-rc1

indent() {
  level="$1"
  if test "$level" -gt 0; then
    printf "%$((2 * ${level}))s"
  fi
}

recurse() {
  requested_path="$1"
  potential_test="$(fullpath "$2")"
  cycle_shell="$3"
  TEST_SHELL="$4"

  for ignore in setup_dir teardown_dir setup teardown; do
    if test "$(basename "$potential_test")" = $ignore; then
      return
    fi
  done

  echo "$requested_path" | grep "^$potential_test" > /dev/null ||
    echo "$potential_test" | grep "^$requested_path" > /dev/null ||
    return 0

  validate_strings "$potential_test" 'Test file names'

  if [ -d "$potential_test" ]; then
    (
      cd -- "$potential_test" > /dev/null
      if test -f setup_dir; then . ./setup_dir; fi

      for test in *; do
        if test "$test" = '*' && ! test -e $test; then
          # The directory is empty.
          break
        fi

        (
          if test -f setup; then . ./setup; fi
          if recurse "$requested_path" "$test" "$cycle_shell" "$TEST_SHELL"; then
            exit_code=0
          else
            exit_code=$?
          fi
          if test -f teardown; then . ./teardown; fi
          exit $exit_code
        ) &

        if $run_in_series; then
          if wait $!; then exit_code=0; else exit_code=$?; fi
          if $exit_on_not_ok && test $exit_code -ne 0; then
            if test -f teardown_dir; then . ./teardown_dir; fi
            return 1
          fi
        fi
      done
      wait
      if test -f teardown_dir; then . ./teardown_dir; fi
    )
  else
    if [ -x "$potential_test" ]; then
      cd -- "$(dirname -- "$potential_test")"

      # Determine the environment variable to define for test scripts
      # that reflects the specified or implied shell to use for shell-code tests.
      while read the_test_shell; do
        (
          if test -f setup; then . ./setup; fi

          # Run the test
          start=$(date +%s)
          set +e
          { 
            if $cycle_shell &&
              has_sh_or_no_shebang_line "$potential_test"; then
              TEST_SHELL="$the_test_shell" $TIMEOUT \
                "$the_test_shell" "$potential_test"
            else
              # Shell cycling is disabled with -n; use the present value of
              # TEST_SHELL or default to /bin/sh
              if [ -n "$TEST_SHELL" ]; then
                $TIMEOUT "$potential_test"
              else
                TEST_SHELL=/bin/sh $TIMEOUT "$potential_test"
              fi
            fi
          } > "$(stdout_file "$potential_test" "$the_test_shell")" 2>&1
          exit_code="$?"
          set -e
          finish=$(date +%s)

          if test -f teardown; then . ./setup; fi

          if [ $exit_code -eq 0 ]; then
            result=ok
          elif [ $exit_code -eq 3 ]; then
            result=skip
          else
            result=not_ok
          fi

          elapsed=$(($finish - $start))
          printf "${potential_test}\t${the_test_shell}\t${result}\t${elapsed}\n" \
            >> $urchin_tmp/log
          exit "$exit_code"
        ) &

        if $run_in_series; then
          if wait $!; then exit_code=0; else exit_code=$?; fi
          if $exit_on_not_ok && test $exit_code -ne 0; then
            if test -f teardown_dir; then . ./teardown_dir; fi
            return 1
          fi
        fi
      done < $shell_list
      wait
    else
      # Shell is ''
      printf "${potential_test}\t\tskip\t0\n" >> $urchin_tmp/log
    fi
  fi
}

report_outcome() {
  root="$1"
  tap_format="$2"
  log_file="$3"
  start="$4"
  finish="$5"

  escaped_root="$(fullpath "$root" | sed 's/\//\\\//g')"
  elapsed=$(($finish - $start))

  if "$tap_format"; then
    printf \#\ 
  fi
  echo Running tests at $(date +%Y-%m-%dT%H:%M:%S)

  for number in n oks skips not_oks; do
    eval "$number=0"
  done

  # Use a temporary file rather than a pipe because a pipe starts a sub-shell
  # and thus makes the above variables local.
  sorted_log_file=$(mktemp)

  # Sort in alphabetical order.
  # GNU sort requires -m, and BSD sort doesn't.
  sort  "$log_file" > $sorted_log_file

  while read line; do
    abspath=$(echo "$line" | cut -f1)
    path=$(echo "$abspath" | sed "s/$escaped_root\/\?//")
    the_shell=$(echo "$line" | cut -f2)
    result=$(echo "$line" | cut -f3)
    file_elapsed=$(echo "$line" | cut -f4)

    prevdir=$currentdir
    currentdir="$(dirname -- "$path")"

    # Number of files that have run, including this one
    n=$(($n + 1))

    # Number of files that have been ok, not ok, and skipped
    eval "old_count=${result}s"
    eval "${result}s=$(($old_count+1))"

    if $tap_format; then
      if [ "$result" = not_ok ]; then
        not='not '
      else
        not=''
      fi
      if [ "$result" = skip ]; then
        skip='# SKIP'
      else
        skip=''
      fi

      if test -z "$the_shell"; then
        the_shell='File is not executable.'
      fi
      echo "${not}ok $n - ${path} ($the_shell) ${skip}"
      if $verbose || [ "$result" = not_ok ]; then
        echo '# ------------ Begin output ------------'
        sed 's/^/# /' "$(stdout_file "$abspath" "$the_shell")"
        echo '# ------------ End output ------------'
      fi
      echo "# Previous test took $file_elapsed seconds."
    else
      if test "$prevdir" != "$currentdir"; then
        echo
      fi
      if test "$prevpath" != "$path"; then
        printf "$(dirname -- "$path")/\n> $(basename -- "$path")\n"
      fi
      case "$result" in
        ok)
          # On success, print a green '✓'
          printf '\033[32m✓ \033[0m'
          echo "${the_shell} ($file_elapsed $(plural second $file_elapsed))"
        ;;
        not_ok)
          # On not_ok, print a red '✗'
          printf '\033[31m✗ \033[0m'
          echo "${the_shell} ($file_elapsed $(plural second $file_elapsed))"
        ;;
        skip)
          if test -z "$the_shell"; then
            echo '  (File is not executable.)'
          else
            echo "  ${the_shell} ($file_elapsed $(plural second $file_elapsed))"
          fi
        ;;
      esac
      if $verbose || test "$result" = not_ok; then
        sed 's/^/  # /' "$(stdout_file "$abspath" "$the_shell")"
      fi
    fi

  prevpath="$path"
  done < $sorted_log_file
  rm $sorted_log_file

  if $tap_format; then
    echo "# Full test suite took $elapsed $(plural second $elapsed)."
    echo 1..$n
  else
    echo
    echo "Done, took $elapsed $(plural second $elapsed)."
    printf '%s\n' "$oks $(plural test "$oks") passed."
    printf '%s\n' "$skips $(plural test "$skips") skipped."

    # If any tests are not ok, print the message in red.
    if [ $not_oks -gt 0 ] ; then
      printf '\033[31m'
    fi
    printf '%s\n' "$not_oks $(plural test "$not_oks") failed."
    printf '\033[m\n'
  fi
  test "$not_oks" -eq '0'
}

has_sh_or_no_shebang_line() {
  # no shebang line at all
  head -n 1 "$1" | grep -vqE '^#!' && return 0

  # shebang line is '#!/bin/sh' or legal variations thereof
  head -n 1 "$1" | grep -qE '^#![[:blank:]]*/bin/sh($|[[:blank:]])' &&
    return 0  

  return 1
}

USAGE="usage: $0 [<options>] <test directory>"

urchin_help() {
  cat <<EOF

$USAGE

By default, Urchin checks for the following shells and runs every
particular test file once per shell.

  $(echo "$DEFAULT_SHELLS" | sed 's/ /\n  /g')

On each run,

1. The TEST_SHELL environment variable is set to the particular shell.
2. If the test file has a shebang line of "#!/bin/sh" or no shebang
   line at all, the test script is also executed in that shell.

The following flags affect how this multiple-shell testing is handled.

-s, --shell <shell>      Tell Urchin to use a different list of shells.
                         (You can pass this flag multiple times.)

-n, --disable-cycling    Disable the cycling of shells; Urchin will execute
                         test files directly, and it will not manipulate the
                         TEST_SHELL environment variable.

The following flags affect how Urchin processes tests.

-b, --run-in-series      Run tests in series. The default is to run tests
                         in parallel where possible.
-e, --exit-on-fail       Stop running if any single test fails.
                         This is useful if you are running something
                         configuration files with Urchin.
-T, --timeout <seconds>  Kill a test if it runs for longer than the
                         specified duration. The default is no timeout.
-f, --force              Force running even if the test directory's name
                         does not contain the word "test".

These options affect how results are formatted.

-t, --tap                Format output in Test Anything Protocol (TAP)
-v, --verbose            Print stdout from all tests, not just failed tests.

The remaining flags provide information about urchin.

-h, --help               Display this help.
--version                Display the version number.

Go to https://github.com/tlevine/urchin for documentation on writing tests.

EOF
}

plural () {
  # Make $1 a plural according to the number $2.
  # If $3 is supplied, use that instead of "${1}s".
  # Result is written to stdout.
  if [ "$2" = 1 ]
  then
    printf '%s\n' "$1"
  else
    printf '%s\n' "${3-${1}s}"
  fi
}

validate_strings() {
  test $(echo "$1" | wc -l) -eq 1 || {
    echo '$1 may not contain tab or newline characters.' >&2
    echo 'If this is really a problem, tell me, and I may fix it.' >&2
    urchin_exit 11
  }
}

cycle_shell=true
shell_list=$urchin_tmp/shell_list
run_in_series=false
force=false
exit_on_not_ok=false
tap_format=false
verbose=false
while [ $# -gt 0 ]
do
    case "$1" in
        -b|--run-in-series) run_in_series=true;;
        -e|--exit-on-fail) exit_on_not_ok=true;;
        -f|--force) force=true;;
        -s|--shell)
          shift
          shell_for_sh_tests=$1
         
          which "$shell_for_sh_tests" > /dev/null || {
            echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2
            urchin_help >&2
            urchin_exit 11
          }

          validate_strings "$shell_for_sh_tests" 'Shell paths'
          if echo "$shell_for_sh_tests" | grep \  > /dev/null; then
            echo "Warning: It is best if shell paths contain no spaces so that
you don't need to quote the TEST_SHELL variable." > /dev/stderr
          fi

          echo "$shell_for_sh_tests" >> "$shell_list"
          ;;
        -n|--disable-cycling) cycle_shell=false;; 
        -t|--tap) tap_format=true;;
        -T|--timeout) shift; urchin_timeout="$1" ;;
        -v|--verbose) verbose=true;;
        -h|--help) urchin_help
                   urchin_exit 0;;
        --version) echo "$VERSION"
            urchin_exit;;
        -*) urchin_help >&2
            urchin_exit 11;;
        *)  break;;
    esac
    shift
done

if ! $cycle_shell && test -f "$shell_list"; then
  echo "The -n/--disable-cycling and -s/--shell options clash with each other." >&2
  urchin_exit 11
fi

# If -s was not passed, use the available default shells.
if ! test -f "$shell_list"; then
  if $cycle_shell; then
    for shell in $DEFAULT_SHELLS; do
      if which $shell > /dev/null; then
        echo $shell >> "$shell_list"
      fi
    done
  else
    echo > "$shell_list"
  fi
fi

# Verify argument for main stuff
if [ "$#" != '1' ] || [ ! -e "$1" ]; then
  if [ -n "$1" ] && [ ! -e "$1" ]; then
    echo "No such file or directory: '$1'" >&2
  fi
  echo "$USAGE" >&2
  urchin_exit 11
fi

# Warn about strange sort commands
weird_string='\n- c\n-- b\n--- c\n---- a\n'
if test $(printf "$weird_string" | sort | tr -d '[ \n-]') != cbca; then
  echo 'Your version of sort sorts in dictionary order (-d) by default.
Depending on how you name your tests, your Urchin output may look strange.
If this is a problem, install BusyBox or BSD coreutils.' >&2
fi

if test -n "$urchin_timeout"; then
  # Choose the timeout command
  if timeout -t 0 true; then
    TIMEOUT="timeout -t $urchin_timeout"
  elif timeout 0 true; then
    TIMEOUT="timeout $urchin_timeout"
  else
    echo I couldn\'t figure out to use your version of timeout >&2
    urchin_exit 1
  fi
fi

if $exit_on_not_ok && ! $run_in_series; then
  echo 'You must also pass -b/--series in order to use -e/--exit-on-fail.' >&2
  urchin_exit 11
fi

# Run or present the Molly guard.
root="$(urchin_root "$1")"
if basename "$(fullpath "$root")" |
  grep -i 'test' > /dev/null || $force; then

  start=$(date +%s)

  # 1 test file or folder to run
  # 2 urchin root
  # 3 Should we cycle shells?
  # 4 TEST_SHELL
  recurse "$(fullpath "$1")" "$root" "$cycle_shell" "$TEST_SHELL" || :
  finish=$(date +%s)

  report_outcome "$root" $tap_format $urchin_tmp/log $start $finish
  urchin_exit $?
else
  echo 'The root directory of the tests that you are running urchin on
does not contain the word "test", so I am not running,
in case that was an accident. Use the -f flag if you really
want to run urchin on that directory.' >&2
  exit_code=1
fi
urchin_exit $exit_code