115 Commits

Author SHA1 Message Date
Thomas Levine
61315d377e remote 2016-04-08 07:32:01 +00:00
Thomas Levine
7daa85cd32 remote host names 2016-04-08 07:08:05 +00:00
Thomas Levine
90b2f93de6 output format to support remotes 2016-04-08 06:56:22 +00:00
Thomas Levine
fa432b5c09 remote testing proposal 2016-04-07 12:38:23 +00:00
Thomas Levine
236b8f86c2 dep proposal 2016-04-07 12:32:19 +00:00
Thomas Levine
74e9d95f77 fixtures idea 2016-04-07 12:20:31 +00:00
Thomas Levine
0395ebb5b2 comment the line in tap 2016-04-07 05:19:47 +00:00
Thomas Levine
cebf0d3add update format tests 2016-04-07 05:18:56 +00:00
Thomas Levine
123f04270b md5 format 2016-04-07 03:55:43 +00:00
Thomas Levine
82c81822d3 mktemp templates 2016-04-07 03:51:01 +00:00
Thomas Levine
9fe6058fbf Automatic commit with j 2016-04-07 03:46:38 +00:00
Thomas Levine
e58ae17704 supporting no rsync 2016-04-07 03:41:33 +00:00
Thomas Levine
367ae3f2b1 tests to write 2016-04-07 03:25:45 +00:00
Thomas Levine
5ffd065317 solaris 2016-04-07 03:23:19 +00:00
Thomas Levine
c0239915a2 error 2016-04-07 03:20:39 +00:00
Thomas Levine
b339c5f98e wrong flag 2016-04-07 03:17:08 +00:00
Thomas Levine
92d40c9ff1 DISABLE_CYCLING 2016-04-07 03:13:42 +00:00
Thomas Levine
24ecd302cd document environment variables 2016-04-07 03:12:10 +00:00
Thomas Levine
135c24fd72 hpux exit codes 2016-04-07 03:03:07 +00:00
Thomas Levine
c219f0a0e6 shorter output 2016-04-07 02:54:30 +00:00
Thomas Levine
adf8dc3562 epoch 2016-04-07 02:52:40 +00:00
Thomas Levine
969340bfd6 bugs 2016-04-07 02:52:38 +00:00
Thomas Levine
a484300263 oops 2016-04-07 02:48:58 +00:00
Thomas Levine
e8d946cc5c switch debug to set -x 2016-04-07 02:47:55 +00:00
Thomas Levine
81c4cdeac0 oops 2016-04-07 02:44:22 +00:00
Thomas Levine
b79045c6ee refactor mktemp 2016-04-07 02:42:29 +00:00
Thomas Levine
c6061b377b alternative ways of getting a seconds counter 2016-04-07 02:39:28 +00:00
Thomas Levine
539027db59 find the next bug 2016-04-07 01:59:28 +00:00
Thomas Levine
5e2ecab592 mktemp 2016-04-07 01:58:34 +00:00
Thomas Levine
efe3af6f87 new version 2016-04-07 00:25:32 +00:00
Thomas Levine
f244c22055 another bug 2016-04-07 00:24:52 +00:00
Thomas Levine
2cdd3ec049 fix hpux rsync path 2016-04-07 00:24:06 +00:00
Thomas Levine
6744b2cb4a more on bugs 2016-04-07 00:16:09 +00:00
Thomas Levine
6e32dab0b7 bugs 2016-04-07 00:11:41 +00:00
Thomas Levine
bb3876f334 oops 2016-04-07 00:06:44 +00:00
Thomas Levine
0dc6e01fbc that alias approach was invalid 2016-04-07 00:06:02 +00:00
Thomas Levine
ca71678522 md5 2016-04-07 00:02:11 +00:00
Thomas Levine
9b4f14c937 note on MirBSD 2016-04-06 23:58:58 +00:00
Thomas Levine
27592e58ac untested md5 handler 2016-04-06 23:58:26 +00:00
Thomas Levine
42c09b775b openbsd bug 2016-04-06 23:50:46 +00:00
Thomas Levine
2923d3a8b0 more systems 2016-04-06 23:45:44 +00:00
Thomas Levine
12f458a933 wrong port 2016-04-06 23:26:42 +00:00
Thomas Levine
6c217acae8 oops 2016-04-06 23:21:04 +00:00
Thomas Levine
6a7e58c944 all flags as flags 2016-04-06 23:18:38 +00:00
Thomas Levine
32ac65cf3e more data in filename 2016-04-06 23:14:25 +00:00
Thomas Levine
d1c8f78585 todo 2016-04-06 21:43:18 +00:00
Thomas Levine
132e1695db HISTORY 2016-04-04 21:42:02 +00:00
Thomas Levine
d67185ce25 don't use /dev/stderr 2016-04-04 21:37:47 +00:00
Thomas Levine
16988c48a9 simpler check 2016-04-04 21:29:51 +00:00
Thomas Levine
ddb74d43d8 longer shell list 2016-04-04 21:21:24 +00:00
Thomas Levine
04c4f54789 strange name so we don't have conflicts 2016-04-04 21:10:37 +00:00
Thomas Levine
552d7d0dbc record test command in makefile 2016-04-04 21:07:29 +00:00
Thomas Levine
00c99773d0 run cross-os tests in series
because i sometimes run them on weak computers
2016-04-04 21:04:05 +00:00
Thomas Levine
7899657971 simpler cross-os tests 2016-04-04 20:59:21 +00:00
Thomas Levine
2ee965d2ae start on cross-os tests 2016-04-04 20:55:14 +00:00
Thomas Levine
fdb0b1b3ea LC_COLLATE rather than LC_ALL 2016-04-04 19:15:00 +00:00
Thomas Levine
db3dd1c662 typos 2016-04-04 01:22:05 +00:00
Thomas Levine
a9e0f856fb error message text 2016-04-04 01:17:58 +00:00
Thomas Levine
72f0700598 fixing timeout flag 2016-04-04 01:16:51 +00:00
Thomas Levine
a486a6f296 check timeout error message 2016-04-04 01:07:29 +00:00
Thomas Levine
a9ba8e79d3 suppress timeout errors 2016-04-04 01:04:04 +00:00
Thomas Levine
586f46600a assert that there is no timeout output 2016-04-04 01:03:05 +00:00
Thomas Levine
940cd549ab more timeout tests 2016-04-04 00:58:37 +00:00
Thomas Levine
cd7f773d58 fix debug 2016-04-04 00:42:08 +00:00
Thomas Levine
0de2c3264a version in readme 2016-04-04 00:24:54 +00:00
Thomas Levine
fc51c34019 debug messages 2016-04-01 19:52:34 +00:00
Thomas Levine
c61d31fcea docs 2016-04-01 19:43:02 +00:00
Thomas Levine
6275c28ebf update history 2016-04-01 18:25:53 +00:00
Thomas Levine
b0a7a8b07a remove out-of-date windows screenshots 2016-04-01 18:24:59 +00:00
Thomas Levine
f7ffd82893 fix name 2016-04-01 18:21:26 +00:00
Thomas Levine
a5ffad0446 Automatic commit with j 2016-04-01 18:21:01 +00:00
Thomas Levine
b402f466b5 package for nongnu 2016-04-01 18:20:00 +00:00
Thomas Levine
7a34da0906 packaging 2016-04-01 18:19:28 +00:00
Thomas Levine
3ebe974202 bump version 2016-03-31 22:25:49 +00:00
Thomas Levine
a957ca9065 locale 2016-03-31 20:42:22 +00:00
Thomas Levine
3122b2cc63 remove nix dependencies 2016-03-31 20:00:45 +00:00
Thomas Levine
4f7541b1fd change flag name 2016-03-31 19:56:12 +00:00
Thomas Levine
5b2ab72ee1 change success and failure symbols 2016-03-31 19:54:27 +00:00
Thomas Levine
73035208d3 update | in test 2016-03-31 19:48:40 +00:00
Thomas Levine
64530e8920 count setup better 2016-03-31 19:46:13 +00:00
Thomas Levine
e66551f50f switch # to | so output is easier to read 2016-03-31 19:43:04 +00:00
Thomas Levine
d42079ef38 fix shebang tests 2016-03-31 19:40:46 +00:00
Thomas Levine
5c24879dd8 oops 2016-03-31 19:28:28 +00:00
Thomas Levine
9ea757722a rename 2016-03-31 19:26:37 +00:00
Thomas Levine
72ea60bb8d separate has_shebang_line tests 2016-03-31 19:26:11 +00:00
Thomas Levine
edda2a0e90 fix has_shebang_line test 2016-03-31 19:10:31 +00:00
Thomas Levine
3c5593c812 fix has_shebang_line 2016-03-31 19:06:23 +00:00
Thomas Levine
ce9273abfc test has_shebang_line 2016-03-31 19:05:02 +00:00
Thomas Levine
406e5c360c document sorting 2016-03-31 18:55:28 +00:00
Thomas Levine
cac691805c remove sort_python test 2016-03-31 18:46:45 +00:00
Thomas Levine
635bcc83f2 set LC_ALL for sort 2016-03-31 18:46:26 +00:00
Thomas Levine
1abba45a39 hah 2016-03-31 18:42:54 +00:00
Thomas Levine
79ebe8e3f2 sort 2016-03-31 18:36:31 +00:00
Thomas Levine
e4b2a4e7ea sort 2016-03-31 18:20:54 +00:00
Thomas Levine
da10808f2a sorting 2016-03-31 18:03:24 +00:00
Thomas Levine
3e83e2cb30 use grep -q 2016-03-29 16:57:21 +00:00
Thomas Levine
2b4444dbed revert /bin/sh shebang behavior 2016-03-29 16:54:40 +00:00
Thomas Levine
5604ec97d6 remove obselete docs 2016-03-29 16:51:38 +00:00
Thomas Levine
8ecea33c16 docs on order 2016-03-29 16:50:18 +00:00
Thomas Levine
dd12693da1 test to change #!/bin/sh shell 2016-03-29 16:48:56 +00:00
Thomas Levine
9f76434bfc proofread 2016-03-21 20:28:59 +00:00
Thomas Levine
c96c2d2157 add python and ruby to default 2016-03-18 19:30:46 +00:00
Thomas Levine
9b4c502d25 note JSON.sh 2016-03-13 23:46:58 +00:00
Thomas Levine
74651bd169 update USAGE 2016-03-10 11:12:17 +00:00
Thomas Levine
a7c44cab70 windows download links 2016-03-06 18:39:22 +00:00
Thomas Levine
5cbafb2a78 document .urchin_dir 2016-03-06 15:46:06 +00:00
Thomas Levine
406d6f43f8 sort alternatives 2016-03-06 15:16:16 +00:00
Thomas Levine
1a4b8674ea perl sort 2016-03-06 15:06:34 +00:00
Thomas Levine
6a0e9f3e64 set root-level running in series 2016-03-06 14:55:11 +00:00
Thomas Levine
5ab6a53961 typo 2016-03-06 14:54:49 +00:00
Thomas Levine
edec12c2c0 remove unnecessary recurse call padding 2016-03-06 14:54:42 +00:00
Thomas Levine
8b9747a997 run in series by dir 2016-03-06 14:44:39 +00:00
Thomas Levine
5864d81f55 fix verbosity levels in tests 2016-03-06 14:40:47 +00:00
Thomas Levine
5cd182ff8f look for .urchin_root rather than .urchin 2016-03-06 14:38:56 +00:00
Thomas Levine
de8bda1b32 crashing in todo 2016-03-06 14:37:00 +00:00
71 changed files with 714 additions and 193 deletions

31
HISTORY
View File

@@ -1,10 +1,18 @@
HISTORY
=======
Version 0.1.0-rc1
Version 0.1.0
---------------------
This release includes breaking changes.
### Cross-OS testing
I have started testing Urchin across multiple operating systems.
This gives access to more shells, as some shels are easier to install on
certain operating systems.
With this cross-OS test suite, I have extended support to more shells.
A later version of Urchin could include a remote testing feature.
### Test root directory
We introduce a concept of the root directory of a test suite.
Such a concept is important in case you want to run subsets of your
@@ -12,18 +20,18 @@ test suite, as we need to know how far up to apply the setup
and teardown files.
The Urchin root directory is determined by moving higher in the directory
tree in search of a file named `.urchin`.
tree in search of a file named `.urchin_root`.
The closest directory that contains such a file is considered the root.
In the following filesystem, for example, `/a/b/c` would be the root.
mkdir -p /a/b/c/d
touch /a/b/c/d/e
chmod +x /a/b/c/d/e
touch /a/b/c/.urchin
touch /a/b/c/.urchin_root
urchin /a/b/c/d
There are two situations in which we would stop looking without having
found a `.urchin` file.
found a `.urchin_root` file.
1. The system root, `/`, because we can't go any higher
2. A directory that starts with a dot, because an urchin call on a higher
@@ -72,7 +80,7 @@ might look for the dependency and then skip if it does not see the dependency.
It might look like this.
#!/bin/sh
if which inkscape; then
if ! which inkscape; then
exit 3 # status code 3 for skip
fi
inkscape blah blah ...
@@ -97,6 +105,13 @@ turned on.
Parallel processing and shell cycling are both enabled by default.
You may want make only some directories run in series, you can create
".urchin_dir" files in those directories.
If .urchin_dir contains the phrase "series", run that directory in series
rather than in parallel.
This is helpful when directories actually need to run in series
and also when running all your tests in parallel crashes your computer.
### Options
Long options are now available for all command line flags.
For example, the `-s` flag is now available as `--shell` as well.
@@ -164,10 +179,6 @@ set as an environment variable, and the latter was set with the -s flag..
Urchin now uses the -s flag for both of these settings, and it mostly ignores
the exported TEST_SHELL variable.
Urchin also inspects the shebang line differently. Previously, Urchin would
vary the shells with which a test is run if the shebang line either was absent
or was #!/bin/sh. Now it varies the shell only if the shebang line is absent.
If you pass -n/--disable-cycling, Urchin will invoke tests ordinarily and will
only set the TEST_SHELL variable if it does not exist. If the TEST_SHELL
variable is absent, it will be set to /bin/sh.
@@ -229,7 +240,7 @@ other Urchin call), and the test suite is recursively descended. Setup and
teardown files are sourced, and everything but the specified test file is
otherwise ignored.
If you don't explicitly specify the Urchin root with a .urchin file, we
If you don't explicitly specify the Urchin root with a .urchin_root file, we
consider the test suite root directory to be the parent of the file that
you ran Urchin on.

8
Makefile Normal file
View File

@@ -0,0 +1,8 @@
.PHONY: test install
test:
./urchin tests
./urchin -s sh -v ./cross-os-tests
install:
cp ./urchin /usr/bin

82
SORTING
View File

@@ -1,5 +1,67 @@
On the criteria for ordering
==============================
I was confused by the documentation for sort's "-d" flag. This confusion
relates to GNU coreutil's locale-specific sort. [^]
Below I discuss sort order differences between different implementations
of sort and of sh "*" for my particular environments.
Sorting with sort
------------
Consider the following two sort commands.
printf '@ b\n- d\n? a\n~ c\n! e\n' | sort
printf '@ b\n- d\n? a\n~ c\n! e\n' | sort -d
With BusyBox v1.23.2 on NixOS 15.09, the first of these commands returns
ASCIIbetical order,
! e
- d
? a
@ b
~ c
and the second returns dictionary order.
? a
@ b
~ c
- d
! e
With GNU coreutils version 8.24 on NixOS, both commands return
dictionary order. The same is true for GNU coreutils version 8.23 on
Debian Wheezy.
? a
@ b
~ c
- d
! e
IEEE Std 1003.1, 2013 Edition [^^] specifies that the "-d" flag should
enable dictionary order. All of these versions of sort have clear
documentation about the order that should be returned when the "-d" flag
is set, (See --help, man, or info.) and the implementations match the
documentation as far as I can tell.
I have found no explicit documentation from any relevant source as to
what the default sort order should be. On the other hand, they all
suggest that "-d" produces an order different from the default order.
In GNU coreutils 8.24, for example, "-d" is a direction to "consider
only blanks and alphanumeric characters". It lacks any mention that the
"-d" flag has no effect or that it is the default. Furthermore, on my
first reading, I took it to mean that the default is to consider all
characters and that "-d" limits the considered characters to blanks and
alphanumeric characters.
Sorting in *
-------------
I think this is related to the order returned by "*" in sh.
The following sh code creates several files in a directory and then
calls "*", listing them in order.
@@ -8,7 +70,8 @@ calls "*", listing them in order.
done
for file in *; do echo "$file"; done
On one computer, running FreeBSD, the order is apparently ASCIIbetical.
On one computer, running FreeBSD, the order is apparently
ASCIIbetical.
! e
- d
@@ -16,7 +79,7 @@ On one computer, running FreeBSD, the order is apparently ASCIIbetical.
@ b
~ c
On another computer, running NixOS, the following commands print results
On two GNU systems, running NixOS and Debian, respectively, output is
in dictionary order. I'm not exactly sure what dictionary order is, but
it is something like sorting on the alphabetical characters before
sorting on the rest of the line.
@@ -27,16 +90,9 @@ sorting on the rest of the line.
- d
! e
While I don't really know what dictionary order is, I was able to determine
(I don't really know what dictionary order is, I was able to determine
that the above results are in dictionary order because of my investigation of
incompatible implementations of sort. Consider the following two sort
commands.
incompatible implementations of sort.)
printf '@ b\n- d\n? a\n~ c\n! e\n' | sort
printf '@ b\n- d\n? a\n~ c\n! e\n' | sort -d
With BSD sort, the first of these commands print ASCIIbetical order and
the second prints dictionary order. With GNU sort, both print dictionary
order.
How annoying.
[^] https://www.gnu.org/software/coreutils/faq/coreutils-faq.html#Sort-does-not-sort-in-normal-order_0021
[^^] http://pubs.opengroup.org/onlinepubs/9699919799/

238
TODO
View File

@@ -26,11 +26,14 @@ Windows
Try running Urchin in Windows somehow. Interpreters include
* CygWin (https://www.cygwin.com/)
* https://cygwin.com/setup-x86.exe
* MSYS (http://mingw.org/wiki/msys)
* GNU on Windows (https://github.com/bmatzelle/gow/wiki)
* Git for Windows (https://git-scm.com/download/win)
* https://github.com/git-for-windows/git/releases/download/v2.7.2.windows.1/Git-2.7.2-32-bit.exe
* win-bash (http://win-bash.sourceforge.net/)
shall
----------
Add shall to my NYC*BUG talk.
@@ -46,15 +49,232 @@ List some shell linters somewhere.
* checkbashisms
Set parallel and series
----------
I want to be able to make only some directories run in series.
* Rename .urchin to .urchin_root.
* Look for a .urchin_dir file.
* If .urchin_dir contains "series", run that directory in series rather
than in parallel.
Rename to something other than "test"?
----------
Maybe wait until I have a use for this.
More sort alternatives
-----------
awk
https://stackoverflow.com/questions/20250937/sorting-lines-in-a-file-alphabetically-using-awk-and-or-sed
bash
https://stackoverflow.com/questions/7442417/how-to-sort-an-array-in-bash
Alternatives
--------------
JSON.sh test suite
Running in multiple environments
-----------------------------------
Setup for other environments includes the following.
* Installing packages
* `touch .zshrc`
* Copy urchin and tests
Fixtures
------------
I want to change the way that fixtures are done.
Instead of using setup, teardown, &c., use ordinary programs from within
your tests. For example.
# tests/.fixtures/tmp-dir
tmp=$(mktemp -d)
cd $tmp
@$
code=$?
cd /
rm -Rf $tmp
exit $code
# tests/blah
../.fixtures/tmp-dir 'blah blah blah'
It's best if I can wrap a bunch of commands in braces or paratheses
rather than just one command. Is there a nice way to do that?
Once I have this new way, I guess I might as well keep the old way.
I think the setup, teardown thing can be easier if you only have simple
fixtures. And since I'm going to keep it, I'm going to add another one.
* setup_dir runs once for the present directory.
* setup_children runs once for each child.
* setup_file runs once for each file descendent.
The present `setup` is renamed to `setup_children`, and the new
`setup_file` runs on each file (not directory) that is a child,
grandchild, great-grandchild, and so on.
Dependency checking
----------------------
You might want to skip tests based on dependencies. Currently you can
conditionally skip tests one at a time by exiting with code 3. I want to
be able to skip an entire directory.
So we add a new magic file called `dep`. If it exists, it is run before
everything else in the directory.
* If it exits with code 0, tests continue as if dep did not exist.
* If it exits with code 3, all tests in the directory are marked as
skipped.
* If it exits with code 1, all tests in the directory are marked as
failed. To make the implementation easier, I'll probably treat the
directory as a single test in this case.
A note on magic files
-------------------------
It is nice to have access to things like setup and dep (magic files)
once in a while, but you need to be doing rather substantial testing
before they make your test suite simpler; the documentation should
strongly recommend writing your tests without magic files and then
refactoring and only then considering moving things to magic files.
Remote testing
----------------
In order to test Urchin across multiple operating systems, I have
already added tests in Urchin's test suite that run Urchin tests in
remote servers. I would like to move this to Urchin itself so that
Urchin can test other things on remote servers.
Urchin's output presently looks like this.
Cycling with the following shells: sh bash dash mksh zsh
Running tests at 2016-04-07T12:33:49
Flags/
> --timeout output
. bash (0 seconds)
. dash (0 seconds)
. mksh (0 seconds)
. sh (0 seconds)
. zsh (0 seconds)
Done, took 1 second.
5 tests passed.
0 tests skipped.
0 tests failed.
After the change, the output should look like this.
Cycling with the following shells: sh dash mksh
Running tests at 2016-04-07T12:33:49
Flags/
> --timeout output
. dash on localhost (0 seconds)
. dash on localhost:8080 (0 seconds)
. dash on tlevine@hpux.polarhome.com (0 seconds)
. mksh on localhost (0 seconds)
. mksh on tlevine@hpux.polarhome.com (0 seconds)
. sh on localhost (0 seconds)
. sh on localhost:8080 (0 seconds)
. sh on tlevine@hpux.polarhome.com (0 seconds)
Done, took 1 second.
8 tests passed.
0 tests skipped.
0 tests failed.
This is just how the output should look; the tests run in whatever order
makes sense.
Bugs
-------
Both md5sum and md5 should be supported.
Trouble logging in to hpux, irix, miros, netbsd, tru64, qnx, ....
$ rsync -e 'ssh -p 785' urchin tlevine@hpux.polarhome.com:.blah
HP-UX hpux.polarhome.com B.11.11 U 9000/785 (ta)
Welcome to HPUX/PA... member of polarhome.com realm
bash: rsync: command not found
rsync: connection unexpectedly closed (0 bytes received so far) [sender]
rsync error: remote command not found (code 127) at io.c(226)
[sender=3.1.1]
OpenIndiana grep does not support -q
I get `/urchin: syntax error at line 84: \`}' unexpected` on
unixware and solaris.
mktemp
> tlevine@hpux.polarhome.com -p 785
F sh (8 seconds)
|
| HP-UX hpux.polarhome.com B.11.11 U 9000/785 (ta)
| Welcome to HPUX/PA... member of polarhome.com realm
|
|
| HP-UX hpux.polarhome.com B.11.11 U 9000/785 (ta)
| Welcome to HPUX/PA... member of polarhome.com realm
|
| mktemp: option requires an argument -- d
| ./urchin[96]: /tmp/tlevinea21441/log: Cannot create the specified file.
date
tlevine@hpux64$ ./urchin tests/ -n -vv
date: bad format character - s
So I need a portable seconds-from epoch
I also need to handle when no arguments are passed to urchin.
Exit code is wrong for which on HP-UX
## `$(...)`
Solaris doesn't support `$(...)`; you need `\`...\`` instead.
tlevine@solaris$ ./urchin --run-in-series tests/Errors/
./urchin: syntax error at line 84: `tmp=$' unexpected
I use this a lot.
$ grep -c '\$(' urchin
52
Darn
Update tests to support
* md5
* rsync
* mktemp
* epoch
* Report cycling by default
* New format for reporting cycling
Support systems without rsync
BSD mktemp
| NetBSD 6.1.3
| Welcome to NetBSD ...member of polarhome.com realm
|
| Usage: mktemp [-dqu] [-p <tmpdir>] {-t prefix | template ...}
| mkdir: : No such file or directory
| ./urchin: cannot create /log: permission denied
NetBSD
md5: unknown option -- q
usage: cksum [-n] [-a algorithm [-ptx] [-s string]] [-o 1|2]
[file ... | -c [-w] [sumfile]]

14
cross-os-tests/.run Executable file
View File

@@ -0,0 +1,14 @@
params="$(basename "${0}")"
hostname="$(echo "${params}" | cut -d\ -f1)"
if echo "${params}" | grep -q \ ; then
flags="$(echo "${params}" | cut -d\ -f2-)"
fi
urchin_dir=.urchin-cross-shell-test
rsync --archive -e "ssh ${flags}" $RSYNC_FLAGS \
../urchin ../tests "${hostname}":"${urchin_dir}" ||
scp -r ${flags} ../urchin ../tests "${hostname}":"${urchin_dir}"
ssh "${hostname}" ${flags} \
"cd ${urchin_dir} && ./urchin --run-in-series tests"

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,3 @@
#!/bin/sh
# apt-get install bash dash ksh posh pdksh mksh yash zsh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,3 @@
#!/bin/sh
RSYNC_FLAGS='--rsync-path=/usr/local/bin/rsync'
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,3 @@
#!/bin/sh
# SSH public key needs to be in ~/.etc/ssh/authorized_keys
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

View File

@@ -0,0 +1,2 @@
#!/bin/sh
. ./.run

1
packages/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
*.tar.gz

11
packages/nongnu.sh Executable file
View File

@@ -0,0 +1,11 @@
#!/bin/sh
name=urchin-$(../urchin --version)
tmp=$(mktemp -d)
mkdir $tmp/$name
cp ../urchin ../readme.md ../AUTHORS ../COPYING $tmp/$name
cd $tmp
tar czf $name.tar.gz $name
cd - > /dev/null
mv $tmp/$name.tar.gz .
rm -R $tmp

View File

@@ -43,25 +43,15 @@ Urchin depends on the following programs.
* timeout
* sort
All of the above programs are usually included on base BSD installations.
On GNU systems it should be sufficient to install the busybox package.
Urchin uses sort to format its output. GNU sort (as of GNU coreutils version
8.24) lacks the ability to sort in lexicographic order, and this feature is
necessary for the output to look right. If your version of sort lacks this
feature, Urchin will try to use one of the following tools for sorting.
If no acceptable sorting program is available, Urchin will print a warning
and use the incomplete sort that is installed on your system. This is not a
big deal; if your test files all start with alphanumeric letters, the output
should look fine.
Vanilla installations of modern BSD and GNU systems usually include all
of these programs.
## Install
Urchin is contained in a single file, so you can install it by copying it to a
directory in your `PATH`. For example, you can run the following as root.
cd /usr/local/bin
wget https://raw.githubusercontent.com/tlevine/urchin/v0.0.6/urchin
wget https://raw.githubusercontent.com/tlevine/urchin/v0.1.0-rc3/urchin
chmod +x urchin
Urchin can be installed with npm too.
@@ -118,14 +108,36 @@ Files are only run if they are executable, and files beginning with `.` are
ignored. Thus, fixtures and libraries can be included sloppily within the test
directory tree. The test passes if the file exits 0; otherwise, it fails.
Tests files and subdirectories are run in ASCIIbetical order within each
directory; that is,
urchin looks for files within a directory in the following manner.
urchin looks for files within a directory in the following manner,
for file in *; do
do_something_with_test_file $file
done
so files are run in whatever order `*` produces. The order is
configured in your environment, at least in
[GNU systems](https://www.gnu.org/software/coreutils/faq/coreutils-faq.html#Sort-does-not-sort-in-normal-order_0021).
Other systems may ignore the locales configured in the environment and
always produce ASCIIbetical order.
Results are always printed in ASCIIbetical order, regardless of what
order the tests ran in.
Below you can see how the locale can affect the order.
$ printf '!c\n@a\n~b\n' | LC_COLLATE=C sort
!c
@a
~b
$ printf '!c\n@a\n~b\n' | LC_COLLATE=en_US.UTF-8 sort
@a
~b
!c
$ printf '!c\n@a\n~b\n' | sort -d
@a
~b
!c
### Writing cross-shell compatibility tests for testing shell code
While you could write your test scripts to explicitly invoke the functionality
@@ -179,14 +191,9 @@ shell.)
Note that only test scripts that either have no shebang line at all or
have shebang line `#!/bin/sh` are invoked with the specified shell.
This allows non-shell test scripts or test scripts for specific
shells to coexist with those whose invocation should be controlled by `-s`.
To test with multiple shells in sequence, use something like:
for shell in sh bash ksh zsh; do
urchin -s $shell ./tests
done
This allows non-shell test scripts or test scripts for other languages
or for specific shells to coexist with those whose invocation should be
controlled by `-s`.
## References

1
tests/.urchin_dir Normal file
View File

@@ -0,0 +1 @@
series

View File

@@ -1,6 +0,0 @@
#!/bin/sh
# This script should run with /bin/sh
# regardless of whether -s or -n is passed.
ps -o pid,comm,args | grep $$ | grep /bin/sh

View File

@@ -0,0 +1,6 @@
#!/bin/sh
# Assuming that urchin was invoked with `-s bash`,
# this script should be being run with bash.
ps -o pid,comm,args | grep $$ | grep .special-shell

View File

@@ -1 +1 @@
! $TEST_SHELL ../../urchin ../Flags/Urchin\ format|grep -- --color
! $TEST_SHELL ../../urchin ../Flags/Urchin\ format|grep -- --pretty

View File

@@ -0,0 +1,2 @@
../../urchin -T aoeu .testsuite 2>&1 | grep Bad
../../urchin -T .testsuite 2>&1 | grep Bad

View File

@@ -1,3 +1 @@
set -e
$TEST_SHELL ../../urchin --shell sh .slow-tests
! $TEST_SHELL ../../urchin --shell sh --timeout 0.3 .slow-tests

4
tests/Flags/--timeout output Executable file
View File

@@ -0,0 +1,4 @@
$TEST_SHELL ../../urchin --shell sh --timeout 0.3 .slow-tests 2>&1 |
grep -v -- --timeout |
grep timeout
test $? = 1

1
tests/Flags/--timeout success Executable file
View File

@@ -0,0 +1 @@
$TEST_SHELL ../../urchin --shell sh .slow-tests --timeout 1000

View File

@@ -1,7 +1,7 @@
#!/bin/sh
set -e
! $TEST_SHELL ../../urchin --run-in-series --exit-on-fail \
! $TEST_SHELL ../../urchin -vv --run-in-series --exit-on-fail \
./.test_-e,--exit-on-fail > $tmp
grep '1 should run.' $tmp

View File

@@ -1,11 +1,11 @@
./
> a
sh (1 second)
# This is stdout from a.
F sh (1 second)
| This is stdout from a.
./
> b
sh (1 second)
. sh (1 second)
./
> c
(File is not executable.)

View File

@@ -2,7 +2,7 @@
./
> a
✗ sh (1 second)
# This is stdout from a.
| This is stdout from a.
./
> b
✓ sh (1 second)

View File

@@ -1,3 +1,3 @@
$TEST_SHELL ../../urchin -s sh -t .testsuite/ |
sed -e 1d -e /second/d > $tmp
$TEST_SHELL ../../urchin -v -s sh -t .testsuite/ |
sed -e 1,2\ d -e /second/d > $tmp
diff $tmp .tap-output-expectation

View File

@@ -0,0 +1,5 @@
lines=$(
$TEST_SHELL ../../urchin -v -s sh -t .testsuite/ |
tee $tmp | grep -v '^#' | wc -l)
cat $tmp
test $lines -eq 4

View File

@@ -1,3 +1,3 @@
$TEST_SHELL ../../urchin -vv -s sh .testsuite/ |
sed -e 1d -e 's/. seconds\?/1 second/' > $tmp
sed -e 1,2\ d -e 's/. seconds\?/1 second/' > $tmp
diff $tmp .urchin-output-expectation

View File

@@ -1,3 +1,3 @@
$TEST_SHELL ../../urchin --color -vv --shell sh .testsuite/ |
sed -e 1d -e 's/. seconds\?/1 second/' > $tmp
$TEST_SHELL ../../urchin --pretty -vv --shell sh .testsuite/ |
sed -e 1,2\ d -e 's/. seconds\?/1 second/' > $tmp
diff $tmp .urchin-output-expectation-color

View File

@@ -0,0 +1,3 @@
echo "$1" > $tmp
TESTING_URCHIN_INTERNALS=true . ../../../urchin
has_shebang_line $tmp

View File

@@ -0,0 +1 @@
./.run '#!/bin/bash'

View File

@@ -0,0 +1 @@
! ./.run ''

View File

@@ -0,0 +1,5 @@
! ./.run '
'

View File

@@ -0,0 +1 @@
./.run '#!/usr/bin/env true'

View File

@@ -0,0 +1 @@
export tmp=$(mktemp)

View File

@@ -0,0 +1 @@
! ./.run '#!/bin/sh'

View File

@@ -0,0 +1 @@
rm -R "$tmp"

View File

@@ -1,4 +0,0 @@
set -e
TESTING_URCHIN_INTERNALS=true . ../../urchin
unsorted='@ b\n- d\n? a\n~ c\n! e\n'
test $(printf "${unsorted}" | sort_python | cut -d\ -f2|tr -d '\n') = edabc

View File

@@ -0,0 +1,3 @@
#!/bin/sh
[ $(grep -c 'setup has run' $log) -eq '1' ]

View File

@@ -0,0 +1,3 @@
#!/bin/sh
[ $(grep -c 'setup has run' $log) -eq '2' ]

View File

@@ -1,3 +0,0 @@
#!/bin/sh
[ $(grep -c 'setup has run' $log) -gt '2' ]

View File

@@ -1,3 +0,0 @@
#!/bin/sh
[ $(grep -c 'setup has run' $log) -gt '2' ]

339
urchin
View File

@@ -51,13 +51,52 @@ set -e
# Kill subprocesses on interrupt.
trap "kill -$$; exit" HUP INT TERM
DEFAULT_SHELLS='sh bash dash mksh zsh'
DEFAULT_SHELLS='
sh
bash
dash
ksh
posh
pdksh
mksh
yash
zsh
'
if [ -n "${ZSH_VERSION}" ]; then
# avoid "no matches found: *" error when directories are empty
setopt NULL_GLOB
emulate sh
fi
epoch_date() {
date +%s
}
epoch_pax() {
# Based on http://stackoverflow.com/a/7262588/407226
tmp="$(mktemp_file)"
echo "ibase=8;$({ pax -wx cpio "${tmp}"; echo; } | cut -c 48-59)" | bc
rm "${tmp}"
}
mktemp_dir() {
# Support HP-UX mktemp that has wrong exit codes and
# can't make directories.
tmp=$(mktemp /tmp/urchin.XXXXXXXX)
if test -f "${tmp}"; then
rm "${tmp}"
fi
mkdir "${tmp}"
echo "${tmp}"
}
mktemp_file() {
tmp=$(mktemp /tmp/urchin.XXXXXXXX)
if ! test -f "${tmp}"; then
> "${tmp}"
fi
echo "${tmp}"
}
validate_test_arg() {
# Must be a file or directory
if [ ! -e "${1}" ]; then
@@ -70,7 +109,7 @@ validate_test_arg() {
root="$(urchin_root "${1}")"
if ! {
basename "$(fullpath "${root}")" |
grep -i 'test' > /dev/null || "${force}"
grep -qi 'test' || "${force}"
}; then
echo 'The root directory of the tests that you are running urchin on
does not contain the word "test", so I am not running,
@@ -80,28 +119,54 @@ validate_test_arg() {
fi
}
sort_python() {
python -c 'import sys
for line in sorted(sys.stdin.readlines()):
sys.stdout.write(line)
'
}
# All temporary files go here
urchin_tmp=$(mktemp -d)
urchin_tmp=$(mktemp_dir)
> "${urchin_tmp}/log"
urchin_exit() {
rm -Rf "${urchin_tmp}"
exit "$@"
}
if which md5 1> /dev/null 2> /dev/null; then
urchin_md5=md5
elif which md5sum 1> /dev/null 2> /dev/null; then
urchin_md5=md5sum
else
echo Could not find MD5 hash command >&2
urchin_exit 1
fi
if epoch_date 2>&1 > /dev/null; then
epoch=epoch_date
elif epoch_pax 2>&1 > /dev/null; then
epoch=epoch_pax
else
echo I could not find a seconds counter. >&2
urchin_exit 1
fi
stdout_file() {
the_test="${1}"
the_shell="${2}"
host="${3}"
x="${urchin_tmp}/stdout$(fullpath "$the_test")"
mkdir -p "${x}"
echo "${x}/$(echo "${the_shell}" | md5sum | cut -d\ -f1)"
if test -n "${host}"; then
# This assumes the tests ran on a remote and have been copied.
x="${urchin_tmp}/remote/${host}/stdout${the_test}"
else
# This can be run during the tests.
x="${urchin_tmp}/stdout$(fullpath "$the_test")"
mkdir -p "${x}"
fi
case "${urchin_md5}" in
md5sum) y=$(echo "${the_shell}" | md5sum | cut -d\ -f1) ;;
md5) y=$(echo "${the_shell}" | md5 | sed 's/.* //') ;;
*) echo md5 command is not configured >&2; urchin_exit 1;;
esac
echo "${x}/${y}"
}
# Expand relative paths
@@ -124,7 +189,7 @@ urchin_root() {
abscurrent="$(fullpath "${1}")"
if test "${abscurrent}" = / ||
basename "${abscurrent}" | grep '^\.' > /dev/null; then
basename "${abscurrent}" | grep -q '^\.' ; then
# Stop traversing upwards at / and at hidden directories.
if test -d "${orig}"; then
echo "${orig}"
@@ -136,7 +201,7 @@ urchin_root() {
return 1
elif test -f "${current}"; then
urchin_root "$(dirname -- "${current}")" "${orig}"
elif test -f "${current}"/.urchin; then
elif test -f "${current}"/.urchin_root; then
remove_trailing_slash "${current}"
else
urchin_root "${current}"/.. "${orig}"
@@ -144,7 +209,7 @@ urchin_root() {
}
# Urchin version number
VERSION=0.1.0-rc1
VERSION=0.1.1-unstable
indent() {
level="${1}"
@@ -165,8 +230,8 @@ recurse() {
fi
done
if echo "${requested_path}" | grep "^${potential_test}" > /dev/null ||
echo "${potential_test}" | grep "^${requested_path}" > /dev/null; then
if echo "${requested_path}" | grep -q "^${potential_test}" ||
echo "${potential_test}" | grep -q "^${requested_path}" ; then
if test "$(dirname "${potential_test}")" = \
"$(dirname "${requested_path}")" &&
test "${potential_test}" != "${requested_path}"; then
@@ -182,7 +247,15 @@ recurse() {
if [ -d "${potential_test}" ]; then
(
cd -- "${potential_test}"
if test -f setup_dir; then . ./setup_dir; fi
if test -f .urchin_dir && grep -q series .urchin_dir ; then
run_in_series_dir=true
else
run_in_series_dir=false
fi
if test -f setup_dir; then
. ./setup_dir
fi
for test in *; do
if test "${test}" = '*' && ! test -e "${test}"; then
@@ -190,28 +263,23 @@ recurse() {
break
fi
(
if test -f setup; then . ./setup; fi
if recurse "${requested_path}" "${test}" "${cycle_shell}" \
"${TEST_SHELL}"; then
exit_code=0
else
exit_code="${?}"
fi
if test -f teardown; then . ./teardown; fi
exit "${exit_code}"
) &
recurse "${requested_path}" "${test}" "${cycle_shell}" \
"${TEST_SHELL}" &
if "${run_in_series}"; then
if "${run_in_series}" || "${run_in_series_dir}"; then
if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
if test -f teardown_dir; then . ./teardown_dir; fi
if test -f teardown_dir; then
. ./teardown_dir
fi
return 1
fi
fi
done
wait
if test -f teardown_dir; then . ./teardown_dir; fi
if test -f teardown_dir; then
. ./teardown_dir
fi
)
elif [ -f "${potential_test}" ]; then
cd -- "$(dirname -- "${potential_test}")"
@@ -220,10 +288,12 @@ recurse() {
# that reflects the specified or implied shell to use for shell-code tests.
while read the_test_shell; do
(
if test -f setup; then . ./setup; fi
if test -f setup; then
. ./setup
fi
# Run the test
start=$(date +%s)
start=$("${epoch}")
set +e
{
if "${cycle_shell}"; then
@@ -234,7 +304,7 @@ recurse() {
"${the_test_shell}" "${potential_test}"
fi
else
# Shell cycling is disabled with -d; use the present value of
# Shell cycling is disabled with -n; use the present value of
# TEST_SHELL or default to /bin/sh
if [ -n "${TEST_SHELL}" ]; then
$TIMEOUT "${potential_test}"
@@ -245,9 +315,11 @@ recurse() {
} > "$(stdout_file "${potential_test}" "${the_test_shell}")" 2>&1
exit_code="${?}"
set -e
finish=$(date +%s)
finish=$("${epoch}")
if test -f teardown; then . ./setup; fi
if test -f teardown; then
. ./teardown
fi
if [ "${exit_code}" -eq 0 ]; then
result=ok
@@ -266,14 +338,16 @@ recurse() {
if "${run_in_series}"; then
if wait "${!}"; then exit_code=0; else exit_code="${?}"; fi
if "${exit_on_not_ok}" && test "${exit_code}" -ne 0; then
if test -f teardown_dir; then . ./teardown_dir; fi
if test -f teardown_dir; then
. ./teardown_dir
fi
return 1
fi
fi
done < "${shell_list}"
wait
else
echo "${potential_test}: Neither file nor directory!?" > /dev/stderr
echo "${potential_test}: Neither file nor directory!?" >&2
fi
else
# Shell is ''
@@ -288,24 +362,31 @@ report_outcome() {
start="${4}"
finish="${5}"
host="${6}"
if test -n "${host}"; then
onhost=" on ${host}"
fi
escaped_root="$(fullpath "${root}" | sed 's/\//\\\//g')"
elapsed=$(($finish - $start))
if "${tap_format}"; then
printf \#\
fi
if "${print_margins}" || "${tap_format}"; then
echo Running tests at $(date +%Y-%m-%dT%H:%M:%S)
if $tap_format; then printf \#\ ; fi
echo Ran tests at $(date +%Y-%m-%dT%H:%M:%S) with the following shells:
if $tap_format; then printf \#\ ; fi
cat "${shell_list}" | tr '\n' \
echo
fi
for number in n oks skips not_oks; do
eval "${number}=0"
done
# Use a temporary file rather than a pipe because a pipe starts a sub-shell
# and thus makes the above variables local.
sorted_log_file=$(mktemp)
cat "${log_file}" | "${sort}" > "${sorted_log_file}"
sorted_log_file=$(mktemp_file)
cat "${log_file}" | LC_COLLATE=C sort > "${sorted_log_file}"
while read line; do
abspath=$(echo "${line}" | cut -f1)
@@ -337,13 +418,13 @@ report_outcome() {
fi
if test -z "${the_shell}"; then
the_shell='File is not executable.'
the_shell='File is not executable'
fi
echo "${not}ok $n - ${path} (${the_shell}) ${skip}"
echo "${not}ok $n - ${path} (${the_shell}${onhost}) ${skip}"
if { test "${result}" = not_ok && "${print_not_ok_stdout}"; } ||
{ test "${result}" = ok && "${print_ok_stdout}"; }; then
echo '# ------------ Begin output ------------'
sed 's/^/# /' "$(stdout_file "${abspath}" "${the_shell}")"
sed 's/^/# /' "$(stdout_file "${abspath}" "${the_shell}" "${host}")"
echo '# ------------ End output ------------'
fi
echo "# Previous test took ${file_elapsed} seconds."
@@ -356,45 +437,44 @@ report_outcome() {
printf "$(dirname -- "${path}")/\n> $(basename -- "${path}")\n"
fi
}
case "${result}" in
ok)
if "${print_ok}"; then
header
# On success, print a green '✓'
if "${print_in_color}"; then
printf '\033[32m \033[0m'
printf "\033[32m${success_mark} \033[0m"
else
printf '✓ '
printf "${success_mark} "
fi
echo "${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
echo "${the_shell}${onhost} (${file_elapsed} $(plural second $file_elapsed))"
fi
;;
not_ok)
if "${print_not_ok}"; then
header
# On not_ok, print a red '✗'
if "${print_in_color}"; then
printf '\033[31m \033[0m'
printf "\033[31m${fail_mark} \033[0m"
else
printf '✗ '
printf "${fail_mark} "
fi
echo "${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
echo "${the_shell}${onhost} (${file_elapsed} $(plural second $file_elapsed))"
fi
;;
skip)
if "${print_ok}"; then
header
if test -z "${the_shell}"; then
echo ' (File is not executable.)'
echo " (File is not executable${onhost}.)"
else
echo " ${the_shell} ("${file_elapsed}" $(plural second $file_elapsed))"
echo " ${the_shell}${onhost} (${file_elapsed} $(plural second $file_elapsed))"
fi
fi
;;
esac
if { test "${result}" = not_ok && "${print_not_ok_stdout}"; } ||
{ test "${result}" = ok && "${print_ok_stdout}"; }; then
sed 's/^/ # /' "$(stdout_file "${abspath}" "${the_shell}")"
sed 's/^/ | /' "$(stdout_file "${abspath}" "${the_shell}" "${host}")"
fi
fi
@@ -424,10 +504,10 @@ report_outcome() {
}
has_shebang_line() {
head -n 1 "${1}" | grep -qE '^#!'
head -n 1 "${1}" | grep -v '^#!/bin/sh$' | grep -q '^#!'
}
USAGE="usage: $0 [<options>] <test directory>"
USAGE="usage: $0 [options]... [test file or directory]..."
urchin_help() {
cat <<EOF
@@ -442,14 +522,15 @@ particular test file once per shell.
On each run,
1. The TEST_SHELL environment variable is set to the particular shell.
2. If the test file lacks a shebang line, the test script is also
executed in that shell.
2. If the test file lacks a shebang line or has a shebang line of
"#!/bin/sh", the test script is also executed in that shell.
The following flags affect how this multiple-shell testing is handled.
-s, --shell <shell> Tell Urchin to use a different list of shells.
(You can pass this flag multiple times.)
-d, --disable-cycling Disable the cycling of shells; Urchin will
-n, --disable-cycling Disable the cycling of shells; Urchin will
execute test files ordinarily, implicitly using
sh for files that lack shebang lines. It will
set the TEST_SHELL variable to "/bin/sh" if and
@@ -459,10 +540,9 @@ The following flags affect how Urchin processes tests.
-b, --run-in-series Run tests in series. The default is to run tests
in parallel where possible.
-n, --max-forks Maximum number of parallel tests (Default is 50.)
-e, --exit-on-fail Stop running if any single test fails.
This is useful if you are running something
configuration files with Urchin.
This can be useful if you are running something
other than test files with Urchin.
-T, --timeout <seconds> Kill a test if it runs for longer than the
specified duration. The default is no timeout.
-f, --force Force running even if the test directory's name
@@ -472,9 +552,11 @@ These options affect how results are formatted. Options -q, and -v
have no effect when combined with --tap. -vv, -vvv, and -vvvv do have
effect when combined with --tap.
-c, --color Print results in color.
-p, --pretty Print results in color and with fancy symbols.
-t, --tap Format output in Test Anything Protocol (TAP)
And these options affect how much is printed.
-q, --quiet Print nothing to stdout;
the only output is the exit code.
(default verbosity) Print names of failed tests and counts
@@ -482,14 +564,21 @@ effect when combined with --tap.
-v Print stdout from failing tests.
-vv Print names of passed tests.
-vvv, --verbose Print stdout from all tests.
-vvvv, --debug Print debugging messages (XXX not implemented)
-vvvv, --debug Run with set -x.
The remaining flags provide information about urchin.
-h, --help Display this help.
--version Display the version number.
Go to https://github.com/tlevine/urchin for documentation on writing tests.
Urchin recognizes certain environment variables.
TEST_SHELL This is sometimes over-ridden; see -s.
RUN_IN_SERIES Set this to true to have the same effect as
-b/--run-in-series. This is helpful if you are
calling urchin inside an urchin test suite.
Go to https://thomaslevine.com/!/urchin/ for documentation on writing tests.
EOF
}
@@ -515,10 +604,13 @@ validate_strings() {
}
main() {
max_forks=50
argv="$@"
cycle_shell=true
shell_list="${urchin_tmp}"/shell_list
test_arg_list="${urchin_tmp}"/test_list
> "${test_arg_list}"
remotes_list="${urchin_tmp}"/remotes
run_in_series=false
force=false
exit_on_not_ok=false
@@ -534,8 +626,6 @@ main() {
do
case "${1}" in
-b|--run-in-series) run_in_series=true;;
-n|--max-forks) shift
max_forks="${1}";;
-e|--exit-on-fail) exit_on_not_ok=true;;
-f|--force) force=true;;
-s|--shell)
@@ -549,17 +639,26 @@ main() {
}
validate_strings "${shell_for_sh_tests}" 'Shell paths'
if echo "${shell_for_sh_tests}" | grep \ > /dev/null; then
if echo "${shell_for_sh_tests}" | grep -q \ ; then
echo "Warning: It is best if shell paths contain no spaces so that
you don't need to quote the TEST_SHELL variable." > /dev/stderr
you don't need to quote the TEST_SHELL variable." >&2
fi
echo "${shell_for_sh_tests}" >> "${shell_list}"
;;
-d|--disable-cycling) cycle_shell=false;;
-n|--disable-cycling) cycle_shell=false;;
-t|--tap) tap_format=true;;
-T|--timeout) shift; urchin_timeout="${1}" ;;
-c|--color) print_in_color=true;;
-T|--timeout)
shift
urchin_timeout="${1}"
if ! {
echo "${urchin_timeout}" |
grep '[0-9][0-9.]*\(s\|m\|h\|d\|\)'
}; then
echo Bad timeout argument: "${urchin_timeout}" >&2
urchin_exit 1
fi ;;
-p|--pretty) print_in_color=true;;
-q|--quiet) print_not_ok=false
print_margins=false;;
@@ -569,7 +668,10 @@ main() {
-vvv|--verbose)print_not_ok_stdout=true
print_ok=true;
print_ok_stdout=true;;
-vvvv|--debug) echo 'Not implemented' > /dev/stderr && exit 1;;
-vvvv|--debug) print_not_ok_stdout=true
print_ok=true;
print_ok_stdout=true
set -x;;
-h|--help) urchin_help
urchin_exit 0;;
@@ -583,6 +685,17 @@ main() {
esac
shift
done
if "${RUN_IN_SERIES}" 2> /dev/null; then
run_in_series=true
fi
if $print_in_color; then
success_mark=✓
fail_mark=✗
else
success_mark=.
fail_mark=F
fi
# -------------------- VALIDATE INPUT -------------------- #
if ! "${cycle_shell}" && test -f "${shell_list}"; then
@@ -594,7 +707,7 @@ main() {
if ! test -f "${shell_list}"; then
if $cycle_shell; then
for shell in $DEFAULT_SHELLS; do
if which $shell > /dev/null; then
if which $shell 1> /dev/null 2> /dev/null; then
echo $shell >> "$shell_list"
fi
done
@@ -603,30 +716,14 @@ main() {
fi
fi
# Warn about strange sort commands
weird_string='@ b\n- d\n? a\n~ c\n! e\n'
sort_result="$(printf "${weird_string}" | sort | cut -d\ -f2 | tr -d '\n')"
if test "${sort_result}" = edacb; then
sort=sort
else
if which python > /dev/null; then
sort=sort_python
else
echo 'Your version of sort sorts in dictionary order (-d) by default.
Depending on how you name your tests, your Urchin output may look strange.
If this is a problem, install BusyBox or BSD coreutils.' >&2
sort=sort
fi
fi
if test -n "${urchin_timeout}"; then
# Choose the timeout command
if timeout -t 0 true; then
if timeout -t 0 true 2> /dev/null; then
TIMEOUT="timeout -t ${urchin_timeout}"
elif timeout 0 true; then
elif timeout 0 true 2> /dev/null; then
TIMEOUT="timeout ${urchin_timeout}"
else
echo I couldn\'t figure out to use your version of timeout >&2
echo I couldn\'t figure out how to use your version of timeout >&2
urchin_exit 1
fi
fi
@@ -637,7 +734,7 @@ main() {
fi
# -------------------- REALLY RUN -------------------- #
start=$(date +%s)
start=$("${epoch}")
# 1 test file or folder to run
# 2 urchin root
@@ -647,16 +744,42 @@ main() {
recurse "$(fullpath "${seed}")" "${root}" "${cycle_shell}" \
"${TEST_SHELL}" || break
done < "${test_arg_list}"
finish=$(date +%s)
finish=$("${epoch}")
test $(cat "${urchin_tmp}"/log | wc -l) -gt 0 || {
echo 'No tests found' > /dev/stderr
if test $(cat "${urchin_tmp}"/log | wc -l) -eq 0; then
echo 'No tests found' >&2
urchin_exit 1
}
fi
report_outcome "${root}" "${tap_format}" "${urchin_tmp}"/log "${start}" \
"${finish}"
urchin_exit "${?}"
if test -n "${RUNNING_ON_REMOTE}"; then
echo "${urchin_tmp}"
elif test -f "${remotes_list}"; then
while read remote; do
remote_main $remote "${argv}"
done < "${remotes_list}"
else
report_outcome "${root}" "${tap_format}" "${urchin_tmp}"/log \
"${start}" "${finish}"
urchin_exit "${?}"
fi
}
remote_main() {
hostname="${1}"; shift
flags="${1}"; shift
urchin_dir=.urchin-cross-shell-test
rsync --archive -e "ssh ${flags}"
../urchin ../tests "${hostname}":"${urchin_dir}" ||
scp -r ${flags} ../urchin ../tests "${hostname}":"${urchin_dir}"
remote_tmp="$(ssh "${hostname}" ${flags} "cd ${urchin_dir} && ./urchin $@")"
remotedir="${hostname}":"${remote_tmp}"
localdir="${urchin_tmp}/remote/${host}"
rsync --archive -e "ssh ${flags}" "${remotedir}" "${localdir}" ||
scp -r ${flags} "${remotedir}"
}
test -n "${TESTING_URCHIN_INTERNALS}" || main "$@"

Binary file not shown.

Before

Width:  |  Height:  |  Size: 99 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 115 KiB