remote

TODO

@@ -72,6 +72,114 @@ Setup for other environments includes the following.
 * `touch .zshrc`
 * Copy urchin and tests
 
+
+Fixtures
+------------
+I want to change the way that fixtures are done.
+
+Instead of using setup, teardown, &c., use ordinary programs from within
+your tests. For example.
+
+    # tests/.fixtures/tmp-dir
+    tmp=$(mktemp -d)
+    cd $tmp
+    @$
+    code=$?
+    cd /
+    rm -Rf $tmp
+    exit $code
+
+    # tests/blah
+    ../.fixtures/tmp-dir 'blah blah blah'
+
+It's best if I can wrap a bunch of commands in braces or paratheses
+rather than just one command. Is there a nice way to do that?
+
+Once I have this new way, I guess I might as well keep the old way.
+I think the setup, teardown thing can be easier if you only have simple
+fixtures. And since I'm going to keep it, I'm going to add another one.
+
+* setup_dir runs once for the present directory.
+* setup_children runs once for each child.
+* setup_file runs once for each file descendent.
+
+The present `setup` is renamed to `setup_children`, and the new
+`setup_file` runs on each file (not directory) that is a child,
+grandchild, great-grandchild, and so on.
+
+Dependency checking
+----------------------
+You might want to skip tests based on dependencies. Currently you can
+conditionally skip tests one at a time by exiting with code 3. I want to
+be able to skip an entire directory.
+
+So we add a new magic file called `dep`. If it exists, it is run before
+everything else in the directory.
+
+* If it exits with code 0, tests continue as if dep did not exist.
+* If it exits with code 3, all tests in the directory are marked as
+  skipped.
+* If it exits with code 1, all tests in the directory are marked as
+  failed. To make the implementation easier, I'll probably treat the
+  directory as a single test in this case.
+
+A note on magic files
+-------------------------
+It is nice to have access to things like setup and dep (magic files)
+once in a while, but you need to be doing rather substantial testing
+before they make your test suite simpler; the documentation should
+strongly recommend writing your tests without magic files and then
+refactoring and only then considering moving things to magic files.
+
+Remote testing
+----------------
+In order to test Urchin across multiple operating systems, I have
+already added tests in Urchin's test suite that run Urchin tests in
+remote servers. I would like to move this to Urchin itself so that
+Urchin can test other things on remote servers.
+
+Urchin's output presently looks like this.
+
+    Cycling with the following shells: sh bash dash mksh zsh 
+    Running tests at 2016-04-07T12:33:49
+
+    Flags/
+    > --timeout output
+    . bash (0 seconds)
+    . dash (0 seconds)
+    . mksh (0 seconds)
+    . sh (0 seconds)
+    . zsh (0 seconds)
+
+    Done, took 1 second.
+    5 tests passed.
+    0 tests skipped.
+    0 tests failed.
+
+After the change, the output should look like this.
+
+    Cycling with the following shells: sh dash mksh
+    Running tests at 2016-04-07T12:33:49
+
+    Flags/
+    > --timeout output
+    . dash on localhost (0 seconds)
+    . dash on localhost:8080 (0 seconds)
+    . dash on tlevine@hpux.polarhome.com (0 seconds)
+    . mksh on localhost (0 seconds)
+    . mksh on tlevine@hpux.polarhome.com (0 seconds)
+    . sh on localhost (0 seconds)
+    . sh on localhost:8080 (0 seconds)
+    . sh on tlevine@hpux.polarhome.com (0 seconds)
+
+    Done, took 1 second.
+    8 tests passed.
+    0 tests skipped.
+    0 tests failed.
+
+This is just how the output should look; the tests run in whatever order
+makes sense.
+
 Bugs
 -------
 
@@ -114,3 +222,59 @@ date
     tlevine@hpux64$ ./urchin tests/ -n -vv
     date: bad format character - s
 
+So I need a portable seconds-from epoch
+
+I also need to handle when no arguments are passed to urchin.
+
+Exit code is wrong for which on HP-UX
+
+## `$(...)`
+Solaris doesn't support `$(...)`; you need `\`...\`` instead.
+
+    tlevine@solaris$ ./urchin --run-in-series tests/Errors/
+    ./urchin: syntax error at line 84: `tmp=$' unexpected
+
+I use this a lot.
+
+    $ grep -c '\$(' urchin 
+    52
+
+Darn
+
+
+
+
+Update tests to support
+
+* md5
+* rsync
+* mktemp
+* epoch
+* Report cycling by default
+* New format for reporting cycling
+
+
+
+
+Support systems without rsync
+
+
+BSD mktemp
+
+    | NetBSD 6.1.3 
+    | Welcome to NetBSD ...member of polarhome.com realm
+    | 
+    | Usage: mktemp [-dqu] [-p <tmpdir>] {-t prefix | template ...}
+    | mkdir: : No such file or directory
+    | ./urchin: cannot create /log: permission denied
+
+
+NetBSD
+
+    md5: unknown option -- q
+    usage: cksum [-n] [-a algorithm [-ptx] [-s string]] [-o 1|2]
+                 [file ... | -c [-w] [sumfile]]
+
+
+
+

cross-os-tests/.run

@@ -8,6 +8,7 @@ fi
 urchin_dir=.urchin-cross-shell-test
 
 rsync --archive -e "ssh ${flags}" $RSYNC_FLAGS \
-  ../urchin ../tests "${hostname}":"${urchin_dir}"
+  ../urchin ../tests "${hostname}":"${urchin_dir}" ||
+  scp -r ${flags} ../urchin ../tests "${hostname}":"${urchin_dir}"
 ssh "${hostname}" ${flags} \
   "cd ${urchin_dir} && ./urchin --run-in-series tests"

tests/Flags/Test Anything Protocol format

@@ -1,3 +1,3 @@
 $TEST_SHELL ../../urchin -v -s sh -t .testsuite/ |
-  sed -e 1d -e /second/d > $tmp
+  sed -e 1,2\ d -e /second/d > $tmp
 diff $tmp .tap-output-expectation

tests/Flags/Test Anything Protocol format 2

@@ -0,0 +1,5 @@
+lines=$(
+  $TEST_SHELL ../../urchin -v -s sh -t .testsuite/ |
+    tee $tmp | grep -v '^#' | wc -l)
+cat $tmp
+test $lines -eq 4

tests/Flags/Urchin format

@@ -1,3 +1,3 @@
 $TEST_SHELL ../../urchin -vv -s sh .testsuite/ |
-  sed -e 1d -e 's/. seconds\?/1 second/' > $tmp
+  sed -e 1,2\ d -e 's/. seconds\?/1 second/' > $tmp
 diff $tmp .urchin-output-expectation

tests/Flags/Urchin format with --color

@@ -1,3 +1,3 @@
 $TEST_SHELL ../../urchin --pretty -vv --shell sh .testsuite/ |
-  sed -e 1d -e 's/. seconds\?/1 second/' > $tmp
+  sed -e 1,2\ d -e 's/. seconds\?/1 second/' > $tmp
 diff $tmp .urchin-output-expectation-color

urchin

@@ -68,6 +68,35 @@ if [ -n "${ZSH_VERSION}" ]; then
   emulate sh
 fi
 
+epoch_date() {
+  date +%s
+}
+epoch_pax() {
+  # Based on http://stackoverflow.com/a/7262588/407226
+  tmp="$(mktemp_file)"
+  echo "ibase=8;$({ pax -wx cpio "${tmp}"; echo; } | cut -c 48-59)" | bc
+  rm "${tmp}"
+}
+
+mktemp_dir() {
+  # Support HP-UX mktemp that has wrong exit codes and
+  # can't make directories.
+  tmp=$(mktemp /tmp/urchin.XXXXXXXX)
+  if test -f "${tmp}"; then
+    rm "${tmp}"
+  fi
+  mkdir "${tmp}"
+  echo "${tmp}"
+}
+mktemp_file() {
+  tmp=$(mktemp /tmp/urchin.XXXXXXXX)
+  if ! test -f "${tmp}"; then
+    > "${tmp}"
+  fi
+  echo "${tmp}"
+}
+
+
 validate_test_arg() {
   # Must be a file or directory
   if [ ! -e "${1}" ]; then
@@ -91,15 +120,7 @@ validate_test_arg() {
 }
 
 # All temporary files go here
-urchin_tmp=$(mktemp)
-
-# Support HP-UX mktemp that has wrong exit codes and
-# can't make directories.
-if test -f "${urchin_tmp}"; then
-  rm "${urchin_tmp}"
-fi
-mkdir "${urchin_tmp}"
-
+urchin_tmp=$(mktemp_dir)
 > "${urchin_tmp}/log"
 
 urchin_exit() {
@@ -116,18 +137,35 @@ else
   urchin_exit 1
 fi
 
+if epoch_date 2>&1 > /dev/null; then
+  epoch=epoch_date
+elif epoch_pax 2>&1 > /dev/null; then
+  epoch=epoch_pax
+else
+  echo I could not find a seconds counter. >&2
+  urchin_exit 1
+fi
 
 stdout_file() {
   the_test="${1}"
   the_shell="${2}"
+  host="${3}"
+
+  if test -n "${host}"; then
+    # This assumes the tests ran on a remote and have been copied.
+    x="${urchin_tmp}/remote/${host}/stdout${the_test}"
+  else
+    # This can be run during the tests.
+    x="${urchin_tmp}/stdout$(fullpath "$the_test")"
+    mkdir -p "${x}"
+  fi
 
-  x="${urchin_tmp}/stdout$(fullpath "$the_test")"
-  mkdir -p "${x}"
   case "${urchin_md5}" in
     md5sum) y=$(echo "${the_shell}" | md5sum | cut -d\  -f1) ;;
-    md5) y=$(echo "${the_shell}" | md5 -q) ;;
+    md5) y=$(echo "${the_shell}" | md5 | sed 's/.* //') ;;
     *) echo md5 command is not configured >&2; urchin_exit 1;;
   esac
+
   echo "${x}/${y}"
 }
 
@@ -186,10 +224,6 @@ recurse() {
   cycle_shell="${3}"
   TEST_SHELL="${4}"
 
-  if $print_debug; then
-    echo Entered directory "${PWD}"
-  fi
-
   for ignore in setup_dir teardown_dir setup teardown; do
     if test "$(basename "${potential_test}")" = "${ignore}"; then
       return
@@ -220,9 +254,6 @@ recurse() {
       fi
 
       if test -f setup_dir; then
-        if $print_debug; then
-          echo Sourcing "${PWD}/setup_dir"
-        fi
         . ./setup_dir
       fi
 
@@ -239,9 +270,6 @@ recurse() {
           if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
           if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
             if test -f teardown_dir; then
-              if $print_debug; then
-                echo Sourcing "${PWD}/teardown_dir"
-              fi
               . ./teardown_dir
             fi
             return 1
@@ -250,32 +278,22 @@ recurse() {
       done
       wait
       if test -f teardown_dir; then
-        if $print_debug; then
-          echo Sourcing "${PWD}/teardown_dir"
-        fi
         . ./teardown_dir
       fi
     )
     elif [ -f "${potential_test}" ]; then
       cd -- "$(dirname -- "${potential_test}")"
 
-      if $print_debug; then
-        echo Running "${potential_test}"
-      fi
-
       # Determine the environment variable to define for test scripts
       # that reflects the specified or implied shell to use for shell-code tests.
       while read the_test_shell; do
         (
           if test -f setup; then
-            if $print_debug; then
-              echo Sourcing "${PWD}/setup"
-            fi
             . ./setup
           fi
 
           # Run the test
-          start=$(date +%s)
+          start=$("${epoch}")
           set +e
           { 
             if "${cycle_shell}"; then
@@ -297,12 +315,9 @@ recurse() {
           } > "$(stdout_file "${potential_test}" "${the_test_shell}")" 2>&1
           exit_code="${?}"
           set -e
-          finish=$(date +%s)
+          finish=$("${epoch}")
 
           if test -f teardown; then
-            if $print_debug; then
-              echo Sourcing "${PWD}/teardown"
-            fi
             . ./teardown
           fi
 
@@ -324,9 +339,6 @@ recurse() {
           if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
           if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
             if test -f teardown_dir; then
-              if $print_debug; then
-                echo Sourcing "${PWD}/teardown_dir"
-              fi
               . ./teardown_dir
             fi
             return 1
@@ -347,24 +359,33 @@ report_outcome() {
   root="${1}"
   tap_format="${2}"
   log_file="${3}"
-  elapsed="${4}"
+  start="${4}"
+  finish="${5}"
+
+  host="${6}"
+  if test -n "${host}"; then
+    onhost=" on ${host}"
+  fi
 
   escaped_root="$(fullpath "${root}" | sed 's/\//\\\//g')"
+  elapsed=$(($finish - $start))
 
-  if "${tap_format}"; then
-    printf \#\ 
-  fi
   if "${print_margins}" || "${tap_format}"; then
-    echo Running tests at $(date +%Y-%m-%dT%H:%M:%S)
+    if $tap_format; then printf \#\ ; fi
+    echo Ran tests at $(date +%Y-%m-%dT%H:%M:%S) with the following shells:
+    if $tap_format; then printf \#\ ; fi
+    cat "${shell_list}" | tr '\n' \ 
+    echo
   fi
 
+
   for number in n oks skips not_oks; do
     eval "${number}=0"
   done
 
   # Use a temporary file rather than a pipe because a pipe starts a sub-shell
   # and thus makes the above variables local.
-  sorted_log_file=$(mktemp)
+  sorted_log_file=$(mktemp_file)
   cat "${log_file}" | LC_COLLATE=C sort > "${sorted_log_file}"
 
   while read line; do
@@ -397,13 +418,13 @@ report_outcome() {
       fi
 
       if test -z "${the_shell}"; then
-        the_shell='File is not executable.'
+        the_shell='File is not executable'
       fi
-      echo "${not}ok $n - ${path} (${the_shell}) ${skip}"
+      echo "${not}ok $n - ${path} (${the_shell}${onhost}) ${skip}"
       if { test "${result}" = not_ok && "${print_not_ok_stdout}"; } ||
          { test "${result}" = ok && "${print_ok_stdout}"; }; then
         echo '# ------------ Begin output ------------'
-        sed 's/^/# /' "$(stdout_file "${abspath}" "${the_shell}")"
+        sed 's/^/# /' "$(stdout_file "${abspath}" "${the_shell}" "${host}")"
         echo '# ------------ End output ------------'
       fi
       echo "# Previous test took ${file_elapsed} seconds."
@@ -426,7 +447,7 @@ report_outcome() {
             else
               printf "${success_mark} "
             fi
-            echo "${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
+            echo "${the_shell}${onhost} (${file_elapsed} $(plural second $file_elapsed))"
           fi
         ;;
         not_ok)
@@ -437,23 +458,23 @@ report_outcome() {
             else
               printf "${fail_mark} "
             fi
-            echo "${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
+            echo "${the_shell}${onhost} (${file_elapsed} $(plural second $file_elapsed))"
           fi
         ;;
         skip)
           if "${print_ok}"; then
             header
             if test -z "${the_shell}"; then
-              echo '  (File is not executable.)'
+              echo "  (File is not executable${onhost}.)"
             else
-              echo "  ${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
+              echo "  ${the_shell}${onhost} (${file_elapsed} $(plural second $file_elapsed))"
             fi
           fi
         ;;
       esac
       if { test "${result}" = not_ok && "${print_not_ok_stdout}"; } ||
          { test "${result}" = ok && "${print_ok_stdout}"; }; then
-        sed 's/^/  | /' "$(stdout_file "${abspath}" "${the_shell}")"
+        sed 's/^/  | /' "$(stdout_file "${abspath}" "${the_shell}" "${host}")"
       fi
     fi
 
@@ -543,13 +564,20 @@ And these options affect how much is printed.
 -v                       Print stdout from failing tests.
 -vv                      Print names of passed tests.
 -vvv, --verbose          Print stdout from all tests.
--vvvv, --debug           Print debugging messages.
+-vvvv, --debug           Run with set -x.
 
 The remaining flags provide information about urchin.
 
 -h, --help               Display this help.
 --version                Display the version number.
 
+Urchin recognizes certain environment variables.
+
+TEST_SHELL           This is sometimes over-ridden; see -s.
+RUN_IN_SERIES        Set this to true to have the same effect as
+                     -b/--run-in-series. This is helpful if you are
+                     calling urchin inside an urchin test suite.
+
 Go to https://thomaslevine.com/!/urchin/ for documentation on writing tests.
 
 EOF
@@ -576,9 +604,13 @@ validate_strings() {
 }
 
 main() {
+  argv="$@"
+
   cycle_shell=true
   shell_list="${urchin_tmp}"/shell_list
   test_arg_list="${urchin_tmp}"/test_list
+  > "${test_arg_list}"
+  remotes_list="${urchin_tmp}"/remotes
   run_in_series=false
   force=false
   exit_on_not_ok=false
@@ -590,7 +622,6 @@ main() {
   print_not_ok=true
   print_ok_stdout=false
   print_not_ok_stdout=false
-  print_debug=false
   while [ "${#}" -gt 0 ]
   do
       case "${1}" in
@@ -640,7 +671,7 @@ main() {
           -vvvv|--debug) print_not_ok_stdout=true
                          print_ok=true;
                          print_ok_stdout=true
-                         print_debug=true;;
+                         set -x;;
 
           -h|--help) urchin_help
                      urchin_exit 0;;
@@ -654,6 +685,9 @@ main() {
       esac
       shift
   done
+  if "${RUN_IN_SERIES}" 2> /dev/null; then
+    run_in_series=true
+  fi
 
   if $print_in_color; then
     success_mark=✓
@@ -682,11 +716,6 @@ main() {
     fi
   fi
 
-  if $print_debug; then
-    echo Cycling with the following shells:
-    cat "${shell_list}"
-  fi
-
   if test -n "${urchin_timeout}"; then
     # Choose the timeout command
     if timeout -t 0 true 2> /dev/null; then
@@ -705,27 +734,52 @@ main() {
   fi
 
   # --------------------   REALLY RUN   -------------------- #
-  elapsed=$({
-    time {
-      # 1 test file or folder to run
-      # 2 urchin root
-      # 3 Should we cycle shells?
-      # 4 TEST_SHELL
-      while read seed; do
-        recurse "$(fullpath "${seed}")" "${root}" "${cycle_shell}" \
-          "${TEST_SHELL}" || break
-      done < "${test_arg_list}"
-    }
-  }  3>&1 1>&2 2>&3 3>& | grep ^real | cut -f 2)
+  start=$("${epoch}")
+
+  # 1 test file or folder to run
+  # 2 urchin root
+  # 3 Should we cycle shells?
+  # 4 TEST_SHELL
+  while read seed; do
+    recurse "$(fullpath "${seed}")" "${root}" "${cycle_shell}" \
+      "${TEST_SHELL}" || break
+  done < "${test_arg_list}"
+  finish=$("${epoch}")
 
   if test $(cat "${urchin_tmp}"/log | wc -l) -eq 0; then
     echo 'No tests found' >&2
     urchin_exit 1
   fi
     
-  report_outcome "${root}" "${tap_format}" "${urchin_tmp}"/log "${start}" \
-    "${finish}"
-  urchin_exit "${?}"
+  if test -n "${RUNNING_ON_REMOTE}"; then
+    echo "${urchin_tmp}"
+  elif test -f "${remotes_list}"; then
+    while read remote; do
+      remote_main $remote "${argv}"
+    done < "${remotes_list}"
+  else
+    report_outcome "${root}" "${tap_format}" "${urchin_tmp}"/log \
+      "${start}" "${finish}"
+    urchin_exit "${?}"
+  fi
+}
+
+remote_main() {
+  hostname="${1}"; shift
+  flags="${1}"; shift
+  urchin_dir=.urchin-cross-shell-test
+
+  rsync --archive -e "ssh ${flags}"
+    ../urchin ../tests "${hostname}":"${urchin_dir}" ||
+    scp -r ${flags} ../urchin ../tests "${hostname}":"${urchin_dir}"
+
+  remote_tmp="$(ssh "${hostname}" ${flags} "cd ${urchin_dir} && ./urchin $@")"
+
+  remotedir="${hostname}":"${remote_tmp}"
+  localdir="${urchin_tmp}/remote/${host}"
+  rsync --archive -e "ssh ${flags}" "${remotedir}" "${localdir}" ||
+    scp -r ${flags} "${remotedir}"
+  
 }
 
 test -n "${TESTING_URCHIN_INTERNALS}" || main "$@"