urchin/urchin

#!/bin/sh

# This file is part of urchin. It is subject to the license terms in the
# COPYING file found in the top-level directory of this distribution or at
# https://raw.githubusercontent.com/tlevine/urchin/master/COPYING

# No part of urchin, including this file, may be copied, modified, propagated,
# or distributed except according to the terms contained in the COPYING file.

set -e

DEFAULT_SHELLS='sh bash dash mksh zsh'

# Make sure that CDPATH isn't set, as it causes `cd` to behave unpredictably -
# notably, it can produce output.
unset CDPATH

# All temporary files go here
tmp=$(mktemp -d)
> $tmp/log
urchin_exit() {
  rm -Rf "$tmp"
  exit "$@"
}

# Source a setup or teardown file
urchin_source() {
  if test -f "$1"; then
    . ./"$1" > /dev/null
  fi
}

stdout_file() {
  x="$tmp/stdout$(fullpath "$1")"
  mkdir -p "$(dirname -- "$x")"
  echo "$x"
}

# Expand relative paths
alias fullpath='readlink -f --'

remove_trailing_slash() {
  echo "$1" | sed s/\\/$//
}

urchin_root() {
  # Call recursively but remember the original argument.
  current="$(remove_trailing_slash "$1")"
  if test -n "$2"; then
    orig="$2"
  else
    orig="$1"
  fi

  if test "$(readlink -f -- "$1")" = /; then
    # Stop traversing upwards at /
    if test -d "$orig"; then
      echo "$orig"
    else
      dirname -- "$orig"
    fi
  elif ! test -e "$current"; then
    echo "$current: No such file or directory">&2
    return 1
  elif test -f "$current"; then
    urchin_root "$(dirname -- "$current")" "$orig"
  elif test -f "$current"/.urchin; then
    remove_trailing_slash "$current"
  else
    urchin_root "$current"/.. "$orig"
  fi
}

# Urchin version number
VERSION=0.1.0-rc1

indent() {
  level="$1"
  if test "$level" -gt 0; then
    printf "%$((2 * ${level}))s"
  fi
}

recurse() {
  set -e
  requested_path="$1"
  potential_test="$(fullpath "$2")"
  cycle_shell="$3"
  TEST_SHELL="$4"

  for ignore in setup_dir teardown_dir setup teardown; do
    if test "$potential_test" = $ignore; then
      return
    fi
  done

  echo "$requested_path" | grep "^$potential_test" > /dev/null ||
    echo "$potential_test" | grep "^$requested_path" > /dev/null ||
    return 0

  validate_strings "$potential_test" 'Test file names'

  if [ -d "$potential_test" ]; then
    (
      cd -- "$potential_test" > /dev/null
      urchin_source setup_dir

      if [ -n "$ZSH_VERSION" ]; then
        # avoid "no matches found: *" error when directories are empty
        setopt NULL_GLOB
      fi

      for test in *; do
        if test "$test" = '*' && ! test -e $test; then
          # The directory is empty.
          break
        fi

        (
          urchin_source setup
          set +e
          recurse "$requested_path" "$test" "$cycle_shell" "$TEST_SHELL"
          exit_code=$?
          set -e
          urchin_source teardown
          exit $exit_code
        ) &

        if $run_in_series; then
          wait $!
          exit_code=$?
          if $exit_on_not_ok && test $exit_code -ne 0; then
            urchin_source teardown_dir
            urchin_exit 1
          fi
        fi
      done
      wait
      urchin_source teardown_dir
    )
  else
    if [ -x "$potential_test" ]; then
      cd -- "$(dirname -- "$potential_test")"

      # Determine the environment variable to define for test scripts
      # that reflects the specified or implied shell to use for shell-code tests.
      if $cycle_shell && has_sh_or_no_shebang_line "$potential_test"; then
        # Set it to the shell specified via -s.
        while read the_test_shell; do
          (
            urchin_source setup

            # Run the test
            start=$(date +%s)
            set +e
            TEST_SHELL="$the_test_shell" "$the_test_shell" \
              "$potential_test" >> \
              "$(stdout_file "$potential_test")" 2>&1
            exit_code="$?"
            set -e
            finish=$(date +%s)

            urchin_source teardown

            if [ $exit_code -eq 0 ]; then
              result=ok
            elif [ $exit_code -eq 3 ]; then
              result=skip
            else
              result=not_ok
            fi

            elapsed=$(($finish - $start))
            printf "${potential_test}\t${the_test_shell}\t${result}\t${elapsed}\n" \
              >> $tmp/log
            exit "$exit_code"
          ) &

          if $run_in_series; then
            wait $!
            exit_code=$?
            if $exit_on_not_ok && test $exit_code -ne 0; then
              urchin_source teardown_dir
              return 1
            fi
          fi
        done < $shell_list
        wait
      else
        echo 'The -n flag is not implemented' > /dev/stderr
        exit 1
        # Shell cycling is disabled with -n; use the present value of
        # TEST_SHELL or default to /bin/sh
        if [ -n "$TEST_SHELL" ]; then
          the_test_shell="$TEST_SHELL"
        else
          the_test_shell='/bin/sh'
        fi
        TEST_SHELL="$the_test_shell" "$the_test_shell" > \
          "$(stdout_file "$potential_test")" 2>&1
      fi
    else
      # Shell is ''
      printf "${potential_test}\t\tskip\t0\n" >> $tmp/log
    fi
  fi
}

report_outcome() {
  root="$1"
  tap_format="$2"
  log_file="$3"
  start="$4"
  finish="$5"

  escaped_root="$(fullpath "$root" | sed 's/\//\\\//g')"
  elapsed=$(($finish - $start))

  if "$tap_format"; then
    printf \#\ 
  fi
  echo Running tests at $(date +%Y-%m-%dT%H:%M:%S)

  for number in n oks skips not_oks; do
    eval "$number=0"
  done

  # Use a temporary file rather than a pipe because a pipe starts a sub-shell
  # and thus makes the above variables local.
  sorted_log_file=$(mktemp)
  sort "$log_file" > $sorted_log_file
  while read line; do
    abspath=$(echo "$line" | cut -f1)
    path=$(echo "$abspath" | sed "s/$escaped_root\/\?//")
    the_shell=$(echo "$line" | cut -f2)
    result=$(echo "$line" | cut -f3)
    file_elapsed=$(echo "$line" | cut -f4)

    prevdir=$currentdir
    currentdir="$(dirname -- "$path")"

    # Number of files that have run, including this one
    n=$(($n + 1))

    # Number of files that have been ok, not ok, and skipped
    eval "old_count=${result}s"
    eval "${result}s=$(($old_count+1))"

    if $tap_format; then
      if [ "$result" == not_ok ]; then
        not='not '
      else
        not=''
      fi
      if [ "$result" == skip ]; then
        skip='# SKIP'
      else
        skip=''
      fi

      if test -z "$the_shell"; then
        the_shell='File is not executable.'
      fi
      echo "${not}ok $n - ${path} ($the_shell) ${skip}"
      if [ "$result" == not_ok ]; then
        echo '# ------------ Begin output ------------'
        sed 's/^/# /' "$(stdout_file "$abspath")"
        echo '# ------------ End output ------------'
      fi
      echo "# Previous test took $file_elapsed seconds."
    else
      if test "$prevdir" != "$currentdir"; then
        echo
      fi
      if test "$prevpath" != "$path"; then
        printf "$(dirname -- "$path")/\n> $(basename -- "$path")\n"
      fi
      case "$result" in
        ok)
          # On success, print a green '✓'
          printf '\033[32m✓ \033[0m'
          echo "${the_shell} ($file_elapsed $(plural second $file_elapsed))"
        ;;
        not_ok)
          # On not_ok, print a red '✗'
          printf '\033[31m✗ \033[0m'
          echo "${the_shell} ($file_elapsed $(plural second $file_elapsed))"

          # Print output captured from not_ok test in red.
          printf '\033[31m'

          sed 's/^/  # /' "$(stdout_file "$abspath")"
          printf '\033[0m'
        ;;
        skip)
          if test -z "$the_shell"; then
            the_shell='(File is not executable.)'
          fi
          echo "  ${the_shell} ($file_elapsed $(plural second $file_elapsed))"
        ;;
      esac
    fi

  prevpath="$path"
  done < $sorted_log_file
  rm $sorted_log_file

  set +e
  if $tap_format; then
    echo "# Full test suite took $elapsed $(plural second $elapsed)."
    echo 1..$n
  else
    echo
    echo "Done, took $elapsed $(plural second $elapsed)."
    printf '%s\n' "$oks $(plural test "$oks") passed."
    printf '%s\n' "$skips $(plural test "$skips") skipped."

    # If any tests are not ok, print the message in red.
    if [ $not_oks -gt 0 ] ; then
      printf '\033[31m'
    fi
    printf '%s\n' "$not_oks $(plural test "$not_oks") failed."
    printf '\033[m'
  fi
  test "$not_oks" -eq '0'
}

has_sh_or_no_shebang_line() {
  # no shebang line at all
  head -n 1 "$1" | grep -vqE '^#!' && return 0

  # shebang line is '#!/bin/sh' or legal variations thereof
  head -n 1 "$1" | grep -qE '^#![[:blank:]]*/bin/sh($|[[:blank:]])' &&
    return 0  

  return 1
}

USAGE="usage: $0 [<options>] <test directory>"

urchin_help() {
  cat <<EOF

$USAGE

By default, Urchin checks for the following shells and runs every
particular test file once per shell.

  $(echo "$DEFAULT_SHELLS" | sed 's/ /\n  /g')

On each run,

1. The TEST_SHELL environment variable is set to the particular shell.
2. If the test file has a shebang line of "#!/bin/sh" or no shebang
   line at all, the test script is also executed in that shell.

The following flags affect how this multiple-shell testing is handled.

-s, --shell <shell>      Tell Urchin to use a different list of shells.
                         (You can pass this flag multiple times.)

-n, --disable-cycling    Disable the cycling of shells; Urchin will execute
                         test files directly, and it will not manipulate the
                         TEST_SHELL environment variable.

The following flags affect how Urchin processes tests.

-b, --run-in-series      Run tests in series. The default is to run tests
                         in parallel where possible.
-e, --exit-on-fail       Stop running if any single test fails.
                         This is useful if you are running something
                         configuration files with Urchin.
-T, --timeout <seconds>  Kill a test if it runs for longer than the
                         specified duration. The default is no timeout.
                         XXX Not yet implemented
-f, --force              Force running even if the test directory's name
                         does not contain the word "test".

These options affect how results are formatted.

-t, --tap                Format output in Test Anything Protocol (TAP)
-v, --verbose            Print stdout from all tests, not just failed tests.
                         XXX Not yet implemented

The remaining flags provide information about urchin.

-h, --help               Display this help.
--version                Display the version number.

Go to https://github.com/tlevine/urchin for documentation on writing tests.

EOF
}

plural () {
  # Make $1 a plural according to the number $2.
  # If $3 is supplied, use that instead of "${1}s".
  # Result is written to stdout.
  if [ "$2" = 1 ]
  then
    printf '%s\n' "$1"
  else
    printf '%s\n' "${3-${1}s}"
  fi
}

urchin_molly_guard() {
  {
    echo
    echo 'The name of the directory on which you are running urchin'
    echo 'does not contain the word "test", so I am not running,'
    echo 'in case that was an accident. Use the -f flag if you really'
    echo 'want to run urchin on that directory.'
    echo
  } >&2
  urchin_exit 1
}

validate_strings() {
  test $(echo "$1" | wc -l) -eq 1 || {
    echo '$1 may not contain tab or newline characters.' >&2
    echo 'If this is really a problem, tell me, and I may fix it.' >&2
    urchin_exit 11
  }
}

cycle_shell=true
shell_list=$tmp/shell_list
run_in_series=false
force=false
exit_on_not_ok=false
tap_format=false
while [ $# -gt 0 ]
do
    case "$1" in
        -b) run_in_series=true;;
        -e) exit_on_not_ok=true;;
        -f) force=true;;
        -s)
          shift
          shell_for_sh_tests=$1
         
          which "$shell_for_sh_tests" > /dev/null || {
            echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2
            urchin_help >&2
            urchin_exit 11
          }

          validate_strings "$shell_for_sh_tests" 'Shell paths'

          echo "$shell_for_sh_tests" >> "$shell_list"
          ;;
        -n) cycle_shell=false;; 
        -t) tap_format=true;;
        -h|--help) urchin_help
          exit 0;;
        --version) echo "$VERSION"
            urchin_exit;;
        -*) urchin_help >&2
            exit 1;;
        *)  break;;
    esac
    shift
done

# If -s was not passed, use the available default shells.
if ! test -f "$shell_list"; then
  for shell in $DEFAULT_SHELLS; do
    if which $shell > /dev/null; then
      echo $shell >> "$shell_list"
    fi
  done
fi

# Verify argument for main stuff
if [ "$#" != '1' ] || [ ! -e "$1" ]; then
  if [ -n "$1" ] && [ ! -e "$1" ]; then
    echo "No such file or directory: '$1'" >&2
  fi
  echo "$USAGE" >&2
  urchin_exit 11
fi

if $exit_on_not_ok && ! $run_in_series; then
  echo 'You must also pass -b/--series in order to use -e/--exit-on-fail.' >&2
  urchin_exit 11
fi

# Run or present the Molly guard.
root="$(urchin_root "$1")"
if basename "$(fullpath "$root")" |
  grep -Fi 'test' > /dev/null || $force; then

  start=$(date +%s)
  set +e

  # 1 test file or folder to run
  # 2 urchin root
  # 3 Should we cycle shells?
  # 4 TEST_SHELL
  recurse "$(fullpath "$1")" "$root" "$cycle_shell" "$TEST_SHELL"
  exit_code=$?

  set -e
  finish=$(date +%s)

  report_outcome "$root" $tap_format $tmp/log $start $finish

  urchin_exit $exit_code
else
  urchin_molly_guard
fi