102 Commits

Author SHA1 Message Date
Thomas Levine
40950a0b52 add -- after readlink 2016-02-27 19:10:28 +00:00
Thomas Levine
dc53523deb fix / detection 2016-02-27 18:41:57 +00:00
Thomas Levine
a797fc2c8b typo 2016-02-27 18:31:41 +00:00
Thomas Levine
e07b340906 Automatic commit with j 2016-02-27 18:26:14 +00:00
Thomas Levine
a4a0aabbef cleaner comment 2016-02-27 18:13:12 +00:00
Thomas Levine
f4d3ffa0e4 Automatic commit with j 2016-02-27 18:12:33 +00:00
Thomas Levine
28832f22a1 check hidden files better 2016-02-27 18:09:40 +00:00
Thomas Levine
f5ef61633e better check 2016-02-27 18:06:19 +00:00
Thomas Levine
2836d4b9a9 woo 2016-02-27 17:53:15 +00:00
Thomas Levine
972fe6bc35 aoeu 2016-02-27 17:51:14 +00:00
Thomas Levine
633d6d32c9 better than main 2016-02-27 17:44:35 +00:00
Thomas Levine
b9d72aef72 remove debug statements 2016-02-27 17:41:37 +00:00
Thomas Levine
92cec52c97 main function 2016-02-27 17:33:46 +00:00
Thomas Levine
b9a067c68e make the .urhin directory 2016-02-27 17:14:32 +00:00
Thomas Levine
e9d6b73dbd urchin_root 2016-02-27 17:13:55 +00:00
Thomas Levine
b8bd097f5c Automatic commit with j 2016-02-27 16:59:47 +00:00
Thomas Levine
f83df14868 set +e 2016-02-27 16:56:44 +00:00
Thomas Levine
fdc7129921 urchin root 2016-02-27 16:53:27 +00:00
Thomas Levine
e5ba45ae96 nicer log file location 2016-02-27 16:05:48 +00:00
Thomas Levine
31e0b9fcb7 quote 2016-02-27 16:01:18 +00:00
Thomas Levine
3efcf0aa33 blah 2016-02-27 15:59:17 +00:00
Thomas Levine
97faea610c error message when no root is set 2016-02-27 14:50:04 +00:00
Thomas Levine
f9ddefcf54 fix test 2016-02-27 14:45:57 +00:00
Thomas Levine
54b9e5887d more urchin_root 2016-02-27 14:45:37 +00:00
Thomas Levine
c9af70b947 simpler urchin root 2016-02-27 14:41:37 +00:00
Thomas Levine
83af249dcd test urchin_root 2016-02-27 14:39:32 +00:00
Thomas Levine
160222c0eb test suite to test .urchin 2016-02-27 14:14:15 +00:00
Thomas Levine
4cd9d41a1e Automatic commit with j 2016-02-26 20:38:31 +00:00
Thomas Levine
e14d8edc05 .urchin in root idea 2016-02-26 18:30:33 +00:00
Thomas Levine
15acd676f9 nagios explanation 2016-02-26 17:43:05 +00:00
Thomas Levine
0c73e5eb44 move skip tests to history 2016-02-26 17:39:18 +00:00
Thomas Levine
bb103757e4 implement exit code 3 thing 2016-02-26 17:37:39 +00:00
Thomas Levine
dce0adc768 test skip on exit code 3 2016-02-26 17:37:12 +00:00
Thomas Levine
80ebf28da9 test skipping idea 2016-02-26 16:39:53 +00:00
Thomas Levine
3f0e925975 move molly-guard change from todo to history 2016-02-26 16:26:22 +00:00
Thomas Levine
43674f12e2 more accepting molly-guard 2016-02-26 16:23:41 +00:00
Thomas Levine
9bb1366549 simpler truth test 2016-02-26 16:19:32 +00:00
Thomas Levine
643a1bf576 test a more accepting molly-guard 2016-02-26 16:18:49 +00:00
Thomas Levine
08ea7bb88f specify version in readme 2016-02-26 16:14:54 +00:00
Thomas Levine
e5ada4899a task runner ideas 2016-02-26 15:57:21 +00:00
Thomas Levine
c818033faa list alternatives 2016-02-14 16:38:02 +00:00
Thomas Levine
88572b6c90 to support single file runs 2016-02-14 07:44:07 +00:00
Thomas Levine
3e5e7334e8 sourcing idea 2016-02-11 20:18:35 +00:00
Thomas Levine
87bb9f4660 link to COPYING file on web 2016-02-08 16:09:00 +00:00
Thomas Levine
caabcd25ff update HISTORY 2016-02-08 16:06:22 +00:00
Thomas Levine
8f82cf5aeb make long lines shorter 2016-02-08 16:05:56 +00:00
Thomas Levine
385ec3fe81 add a copying notice to the urchin excutable 2016-02-08 16:02:03 +00:00
Thomas Levine
d4eb99ffb0 rename LICENSE -> COPYING 2016-02-08 15:59:56 +00:00
Thomas Levine
ed939ff39a update history 2016-02-08 15:59:47 +00:00
Thomas Levine
d24a71d0d7 copyright notice
add Thomas Levine and Michael Klement
2016-02-08 15:59:16 +00:00
Thomas Levine
ceae1811e2 nagios todo 2016-02-08 15:51:53 +00:00
Thomas Levine
2ac3a52707 fix printf to work in mksh 2016-02-08 15:50:12 +00:00
Thomas Levine
2d4de62729 msys64 test results 2016-02-08 15:48:57 +00:00
Thomas Levine
3b3ba0e1fa windows bash implementations list 2016-02-08 15:48:57 +00:00
Thomas Levine
1bc9bb2671 explain license thing 2016-02-08 15:48:57 +00:00
Thomas Levine
fafd0c1801 note desire to test on windows 2016-02-08 15:48:57 +00:00
Thomas Levine
d85b0ee352 packaging ideas 2016-02-08 15:48:57 +00:00
Thomas Levine
cf793387e9 licensing 2016-02-08 15:48:57 +00:00
Thomas Levine
1f79db874b environment variables 2016-02-08 15:48:57 +00:00
Thomas Levine
c770479884 TODO list 2016-02-08 15:48:57 +00:00
Thomas Levine
96ab43c233 explain why urchin is called urchin 2016-01-29 17:28:26 +00:00
Thomas Levine
2be0151213 document the --help flag 2016-01-29 17:20:18 +00:00
Thomas Levine
655daf4b74 show all counts in cross-shell-tests runner
It previously just showed the last three lines. It needs to show the
last four lines now because urchin now displays skipped tests.
2016-01-29 17:19:15 +00:00
Thomas Levine
31023bce21 update HISTORY 2016-01-29 17:17:35 +00:00
Thomas Levine
d96f3707e9 implement -v 2016-01-29 17:17:31 +00:00
Thomas Levine
3f05794d7e add -v to usage 2016-01-29 17:16:01 +00:00
Thomas Levine
d7c8549206 urchin -v test 2016-01-29 17:14:34 +00:00
Thomas Levine
bc74b20c5c add authors 2016-01-28 15:33:43 +00:00
Thomas Levine
8bdd00c3e9 explain the branching 2016-01-28 15:30:59 +00:00
Thomas Levine
42dd0947e8 more changes 2016-01-28 15:28:36 +00:00
Thomas Levine
da6d89aa56 add to change log 2016-01-28 15:27:32 +00:00
Thomas Levine
3ea035d188 rename test to be more clear 2016-01-28 15:24:24 +00:00
Thomas Levine
8e669c40a6 add + indent symbol so directories line up 2016-01-28 15:23:14 +00:00
Thomas Levine
66c010b694 fix fixture to not expect stderr 2016-01-28 15:23:14 +00:00
Thomas Levine
7339e84d7c test stdout stderr for TAP 2016-01-28 15:23:14 +00:00
Thomas Levine
e364d40f3a TAP test 2016-01-28 15:23:14 +00:00
Thomas Levine
13e748d684 remove awk dependency in test suite
I'm on NixOS, so awk isn't in /usr/bin. I can search for it with
/usr/bin/env, but then I can't use -f in the shebang line.
2016-01-28 15:23:14 +00:00
Thomas Levine
bb42d27917 test plan 2016-01-28 15:23:14 +00:00
Thomas Levine
c1ec06c6fe logfile 2016-01-28 15:23:14 +00:00
Thomas Levine
5382946639 error on fail 2016-01-28 15:23:14 +00:00
Thomas Levine
144013c71c print skip count at end 2016-01-28 15:23:13 +00:00
Thomas Levine
009fd7ea05 print stdout for tap and not-tap 2016-01-28 15:23:13 +00:00
Thomas Levine
c3901fe175 tap indentation comments for directories 2016-01-28 15:23:13 +00:00
Thomas Levine
538c8437a7 assorted tap stuff 2016-01-28 15:23:13 +00:00
Thomas Levine
cd124e09ee more tap 2016-01-28 15:23:13 +00:00
Thomas Levine
d963f10be1 more tap 2016-01-28 15:23:13 +00:00
Thomas Levine
0ca55d155b oops 2016-01-28 15:23:13 +00:00
Thomas Levine
8d55550248 test test successes better 2016-01-28 15:23:13 +00:00
Thomas Levine
0f86778d79 convert indents to comments 2016-01-28 15:23:13 +00:00
Thomas Levine
92cb735bb7 start writing non-tap cases 2016-01-28 15:23:13 +00:00
Thomas Levine
c817355e94 not working 2016-01-28 15:23:13 +00:00
Thomas Levine
d4bedbbfcf first pass implementation of exit on fail 2016-01-28 15:23:13 +00:00
Thomas Levine
f6b57772f2 document unimplemented -e flag 2016-01-28 15:23:13 +00:00
Thomas Levine
72fa30e787 Test die on fail. 2016-01-28 15:23:13 +00:00
Thomas Levine
eda4a6e42c call urchin -s in cross-shell tests 2016-01-28 15:23:12 +00:00
Thomas Levine
df80586d79 remove "urchin -x" test 2016-01-28 15:23:12 +00:00
Thomas Levine
d525c1793b remove "urchin -x"
unnecessary now that shall exists
2016-01-28 15:23:12 +00:00
Thomas Levine
56a4e5d8af introduction paragraph 2016-01-25 14:06:33 +00:00
Thomas Levine
d81b7efdc3 asciibetical 2016-01-25 13:56:33 +00:00
Thomas Levine
7f5455739d update readme
* move shall reference to the right place
* further explain cross-shell testing methods
2016-01-25 13:49:36 +00:00
Thomas Levine
48e5090091 start writing alternatives 2015-12-09 07:46:16 -05:00
Thomas Levine
5abf088eb8 add new ideas 2015-10-19 12:52:24 -04:00
43 changed files with 612 additions and 158 deletions

9
AUTHORS Normal file
View File

@@ -0,0 +1,9 @@
Authors
-------
David Jones
Michael Klement
Thomas Levine
Maintainer
-------
Thomas Levine <_@thomaslevine.com>

View File

@@ -1,3 +1,5 @@
Copyright (c) 2013, 2014, 2015, 2016 Thomas Levine
Copyright (c) 2014, Michael Klement
Copyright (c) 2012, ScraperWiki Limited
All rights reserved.

46
HISTORY
View File

@@ -1,5 +1,49 @@
HISTORY
-------
=======
Version 0.0.7
---------------------
The Molly-guard is now more accepting. For example, you no longer need to
pass -f in this case: https://github.com/creationix/nvm/issues/357
Previously, tests were run if they were executable and were otherwise marked
as skipped. Now, an executable script can indicate that it is skipped by
exiting with code 3. For example, if a test requires some dependancy, it
might look for the dependency and then skip if it does not see the dependency.
It might look like this.
#!/bin/sh
if which inkscape; then
exit 3 # status code 3 for skip
fi
inkscape blah blah ...
I chose status code 3 sort of arbitrarily at first, but it turns out that it
would the appropriate status code if these tests were Nagios plugins, as the
concept of skipping a test is similar to the Nagios concept of unknown service
status (https://nagios-plugins.org/doc/guidelines.html#AEN78).
Version 0.0.6
---------------------
* Produce TAP output with the -t flag.
* Add a + sign in front of directories in the normal output so that they
line up with non-directories.
* Display skipped tests in the normal output and in the TAP output.
* Correct some things in the documentation.
* Rearrange things in the documentation to be more clear.
* Pass the -e flag to exit urchin if any single test fails.
* Remove the undocumented, experimental -x flag now that shall exists.
* Display version number with the -v flag.
* Document why Urchin is called "Urchin"
* Update TODO
* Support mksh (Change a printf command.)
* Make long lines shorter.
These changes are made somewhat separately in the branches "exit-on-fail",
"remove-urchin-x", "tap", and "update-readme". They are rebased into one
branch, "tlevine-2016-02", for merging into "master".
Version 0.0.5
---------------------

164
TODO Normal file
View File

@@ -0,0 +1,164 @@
Things I want
=============
Test speed
-------------
Make tests run faster.
https://github.com/bike-barn/hermit/issues/62
First, easier thing is probably to run tests in parallel.
Second, also easier thing is to tell people to save things to RAM rather than
disk whenever they can.
Third, harder thing is to put the test suite in RAM automatically. Maybe the
whole test directory, which includes fixtures, gets copied to a tmpfs if one
exists.
Hmm or maybe there's a compromise: Tell people to mount /tmp as a tmpfs so
that temp files are fast. Maybe allow people to set some other directory as
the temporary file place, in case they want a different tmpfs location.
In order to run things in parallel, we have to change how we do the
stdout_file. I think it's easiest to create separate files for each test and
to save them in testroot/.urchin/stdout/$filename. The test root would be
defined as the closest ancestor containing a .urchin directory.
Options
-------------
I want long options. For example, there's presently -f and -e.
I want to make them -f|--force and -e|--exit.
Environment variables
-------------
Do something to make it easier to debug environment variables, because that is
often confusing.
https://github.com/creationix/nvm/issues/719
https://github.com/creationix/nvm/issues/589
Documenting that people should run "env" when their tests fail might be good
enough.
Licensing and copyright
------------------------
* Reference all owners and years in the Copyright file
* Consider copyleft licenses
* Add license notices to other files if necessary
Packaging
------------
Package for package managers.
* I want NixOS, of course.
* Debian is probably the big one.
Other interesting package managers
* Update the npm package
* Homebrew (for Mac)
Windows
----------
Try running Urchin in Windows somehow. Interpreters include
* CygWin (https://www.cygwin.com/)
* MSYS (http://mingw.org/wiki/msys)
* GNU on Windows (https://github.com/bmatzelle/gow/wiki)
* Git for Windows (https://git-scm.com/download/win)
* win-bash (http://win-bash.sourceforge.net/)
Consider copyleft licenses
----------
ScraperWiki owns the original version of Urchin (Thomas Levine did the early
work as part of his work for ScraperWiki.) and originally licensed it under an
MIT-style license. Other people made changes after this original ScraperWiki
version. As of January 2016, they are just Thomas Levine (when he wasn't
working for ScraperWiki) and Michael Klement.
The original license was MIT just because that's what ScraperWiki put on
everything. Should we change the license?
The MIT-style license grants pretty much all rights. It says that you need
to attribute when you redistribute source code, but you don't
necessarily have to redistribute source code.
A copyleft license adds the restriction that modified versions of the
code need to be licensed under the same license. GNU licenses in
particular require that source code be released if non-source versions are
released, and the different GNU licenses differ in what how the
non-source version is defined. (The original, GPL, discusses compiled
binaries.) Copyleft doesn't mean anything specific for commercial use.
MIT-licensed code can be modified and then licensed as GPL, because MIT
license allows that, but GPL code can't be modified as MIT, because MIT
doesn't allow that. And if we get all of the authors to agree on it, we
can always add whatever crazy license we want, regardless of what we
have already.
The distinction between MIT-style and GNU-something might matter quite little
in the case of Urchin.
1. Urchin is written in an interpreted language (shell), so it might be
hard to distribute usefully without providing the source code.
2. Urchin just runs tests; it doesn't get compiled with the rest of the
code (also because it's in shell). Thus, I think a GPL license on
Urchin wouldn't infect the code being tested.
This is as far as I have gotten with contemplating license changes. For now
we're sticking with the original MIT-style license, but it's easy to change
licenses later.
Nagios plugins
-----------------
It would be cool to run Nagios plugins with Urchin. This is already possible,
actually, but it might be worth giving some special thought to it.
https://nagios-plugins.org/doc/guidelines.html
Source setup and teardown
--------------------
If setup and teardown are sourced instead of executed, maybe we can more
cleanly create and teardown temporary files.
(
. ./setup
./$thetestfile
. ./teardown
)
On the other hand, this could just be sourced explicitly in the test file,
without the special setup and teardown feature.
Run on a file
----------------
Presently you can run urchin only on a directory.
It would be neat if you could run it on a file as well.
This occurred to me when I wanted to run
urchin test/fast/Unit\ tests/nvm_ls_current
on the nvm tests. I wound up running this instead.
urchin test/fast/Unit\ tests/ | grep nvm_ls_current
The Molly guard would be assessed, and the corresponding setup, setup_dir,
teardown, and teardown_dir files would be run in the appropriate order.
In order to know how far up the tree to evaluate the setup, &c. files,
I think it would make sense to require that a ".urchin" file be placed in the
root of the tests. Urchin would keep going up until it sees this file, and it
would evaluate the appropriate setup, &c. files from there down to the
particular test file of interest. We would also use this for testing
directtories more correctly.
Running automated tasks
-------------------------
Urchin might be appropriate for if you have lots of tasks that you want to run
periodically; add an urchin call to your crontab, and call all of your other
tasks with urchin. Here are some features that might make urchin better for
this sort of thing.
* Time how long each test/job takes
* Optionally kill tests/jobs after a specific timeout threshold
* Send output of different tests/jobs to different files for each file
descriptor (STDOUT, STDERR)

9
alternatives Normal file
View File

@@ -0,0 +1,9 @@
Totally different syntax and similar features, plus TAP output
https://github.com/sstephenson/bats
Relatively similar interface
https://github.com/mlafeldt/sharness
Lists of alternatives
https://thomaslevine.com/!/shell-testing/
https://github.com/mlafeldt/sharness#alternatives

View File

@@ -2,11 +2,11 @@
# Run urchin in a bunch of different shells,
# including a shell that isn't quite POSIX-compatible (zsh)
for shell in dash bash ksh zsh; do
for shell in dash bash mksh ksh zsh; do
if which $shell > /dev/null 2> /dev/null; then
echo
echo Running urchin tests in $shell
$shell urchin tests | tail -n 3
$shell urchin -s $shell tests | tail -n 4
else
echo
echo Skipping $shell because it is not in the PATH

View File

@@ -4,9 +4,13 @@
/ /_/ / / / /__/ / / / / / / /
\__,_/_/ \___/_/ /_/_/_/ /_/
Urchin is a test framework for shell. It is implemented in
portable /bin/sh and should work on GNU/Linux, Mac OS X, and
other Unix platforms.
Urchin is a file-based test harness, normally used for testing shell programs.
It is written in portable shell and should thus work on GNU/Linux, BSD
(including Mac OS X), and other Unix-like platforms.
Urchin is called "Urchin" because
[sea urchins](https://en.wikipedia.org/wiki/Sea_urchin)
have shells called "tests".
## Try it out
Urchin's tests are written in Urchin, so you can run them to see what Urchin
@@ -27,14 +31,15 @@ run this:
cd urchin
./cross-shell-tests
## Globally
Download Urchin like so (as root) (or use npm, below):
## Install
Urchin is contained in a single file, so you can install it by copying it to a
directory in your `PATH`. For example, you can run the following as root.
cd /usr/local/bin
wget https://raw.github.com/tlevine/urchin/master/urchin
wget https://raw.githubusercontent.com/tlevine/urchin/v0.0.6/urchin
chmod +x urchin
Can be installed with npm too:
Urchin can be installed with npm too.
npm install -g urchin
@@ -86,15 +91,14 @@ Files are only run if they are executable, and files beginning with `.` are
ignored. Thus, fixtures and libraries can be included sloppily within the test
directory tree. The test passes if the file exits 0; otherwise, it fails.
In case you care about the order in which your tests execute, consider that
Tests files and subdirectories are run in ASCIIbetical order within each
directory; that is,
urchin looks for files within a directory in the following manner.
for file in *; do
do_something_with_test_file $file
done
Tests within a directory are executed in whatever order `*` returns.
### Writing cross-shell compatibility tests for testing shell code
While you could write your test scripts to explicitly invoke the functionality
@@ -106,24 +110,20 @@ The specific approach depends on your test scenario:
* (b) Your scripts _source_ scripts containing portable shell code.
#### (a) Cross-shell tests with test scripts that _invoke_ shell scripts
Urchin sets the `TEST_SHELL` environment variable so that you may change the
shell with which your tests call other shell programs. To run your test
scripts in multiple shells you must call `$TEST_SHELL` in your tests and then
run urchin with the appropriate option.
First, consider using [shall](https://github.com/mklement0/shall).
#!/usr/bin/env shall
echo This is a test file.
Alternatively, you can use urchin's built-in recognition of the
`TEST_SHELL` environment variable.
In your test scripts, invoke the shell scripts to test via the shell
specified in environment variable `TEST_SHELL` rather than directly;
e.g.: `$TEST_SHELL ../foo bar` (rather than just `../foo bar`).
Note that if you alsow want your test scripts to work when run directly,
outside of Urchin, be sure to target scripts that happen to be in the
current directory with prefix `./`; e.g., `$TEST_SHELL ./baz`
(rather than `$TEST_SHELL baz`).
Then, on invocation of Urchin, prepend a definition of environment variable
`TEST_SHELL` specifying the shell to test with, e.g.: `TEST_SHELL=zsh urchin ./tests`.
On invocation of Urchin, prepend a definition of environment variable
`TEST_SHELL` specifying the shell to test with, e.g.,
TEST_SHELL=zsh urchin ./tests
To test with multiple shells in sequence, use something like:
for shell in sh bash ksh zsh; do
@@ -131,14 +131,20 @@ To test with multiple shells in sequence, use something like:
done
If `TEST_SHELL` has no value, Urchin defines it as `/bin/sh`, so the test
scripts can rely on `$TEST_SHELL` always containing a value.
scripts can rely on `$TEST_SHELL` always containing a value when Urchin runs
them.
That said, we still recommand that you account for the possibility that
`$TEST_SHELL` does not contain a value so that you may run your test scripts
without Urchin. Supporting this case is very simple; when you invoke scripts
that happen to be in the current directory, be sure to use the prefix `./`,
e.g., `$TEST_SHELL ./baz` rather than `$TEST_SHELL baz`.
#### (b) Cross-shell tests with test scripts that _source_ shell scripts
If you _source_ shell code in your test scripts, it is the test scripts
themselves that must be run with the shell specified.
To that end, Urchin supports the `-s <shell>` option, which instructs
Urchin supports the `-s <shell>` option, which instructs
Urchin to invoke the test scripts with the specified shell; e.g., `-s bash`.
(In addition, Urchin sets environment variable `TEST_SHELL` to the specified
shell.)
@@ -154,21 +160,12 @@ To test with multiple shells in sequence, use something like:
urchin -s $shell ./tests
done
<!--
#### (c) Cross shell tests with `urchin -x` (experimental)
If you run urchin with the `-x` flag, it will be as if you ran
`$TEST_SHELL`. Unless `$TEST_SHELL` isn't set, in which case it'll
be as if you ran `/bin/sh`. Putting this in she shebang line might
eventually work out to be a cleaner way of doing cross-shell testing.
Also consider using [shall](https://github.com/mklement0/shall).
It does something similar, but the interface may be more intuitive.
#!/usr/bin/env urchin -x
test a = a
#!/usr/bin/env shall
echo This is a test file.
It might make sense if you do this.
export TEST_SHELL=zsh && urchin -x
export TEST_SHELL=bash && urchin -x
-->
## Alternatives to Urchin
Alternatives to Urchin are discussed in
[this blog post](https://blog.scraperwiki.com/2012/12/how-to-test-shell-scripts/).

View File

View File

@@ -0,0 +1 @@
false

View File

@@ -0,0 +1 @@
false

View File

@@ -0,0 +1 @@
false

2
tests/.urchin/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
*
!.gitignore

View File

@@ -1,5 +1,3 @@
#!/usr/bin/awk -f
# This script will only succeed if it is indeed processed by awk.
BEGIN { print "ok" }
#!/usr/bin/env true
true will processed the contents of this script, but that
means that nothing will happen and the script will exit 0

View File

@@ -3,6 +3,5 @@
# Tests the `-s <shell> option, which invokes shebang-less and sh-shebang-line test scripts with the specified shell (for testing *sourced* shell code).
which bash >/dev/null || { echo "Cannot test -s option: bash cannot be located." >&2; exit 2; }
which /usr/bin/awk >/dev/null || { echo "Cannot test -s option: /usr/bin/awk not found." >&2; exit 2; }
../../urchin -s bash ./.test-run-by-specified-shell

View File

@@ -0,0 +1,11 @@
tmp=$(mktemp)
../urchin -e -f ./.die-on-fail > $tmp
result=$?
grep '1 should run.' $tmp
grep '2 should run.' $tmp
grep -v '3 should not run.' $tmp
grep -v '4 should not run.' $tmp
rm $tmp
exit $result

2
tests/Print version on -v. Executable file
View File

@@ -0,0 +1,2 @@
#!/bin/sh
../urchin -v | grep '[0-9.]\{3,\}'

View File

@@ -0,0 +1,10 @@
# Begin - .testsuite/
not ok 1 - a
# ------------ Begin output ------------
# This is stdout from a.
# ------------ End output ------------
ok 2 - b
ok 3 - # SKIP c
# End - .testsuite/
# Took 0 seconds.
1..3

4
tests/TAP/.testsuite/a Executable file
View File

@@ -0,0 +1,4 @@
#!/bin/sh
echo This is stderr from a. > /dev/stderr
echo This is stdout from a. > /dev/stdout
false

4
tests/TAP/.testsuite/b Executable file
View File

@@ -0,0 +1,4 @@
#!/bin/sh
echo This is stderr from b. > /dev/stderr
echo This is stdout from b. > /dev/stdout
true

1
tests/TAP/.testsuite/c Normal file
View File

@@ -0,0 +1 @@
This should not be run.

View File

@@ -0,0 +1,4 @@
tmp=$(mktemp)
../../urchin -t .testsuite/ | sed 1d > $tmp
diff $tmp .expected-output

View File

@@ -0,0 +1,6 @@
#!/bin/sh
observed=$(../../urchin "$1" .testsuite/a/b)
expected=.testsuite/a/b/../..
test "$observed" = "$expected"

View File

@@ -0,0 +1,2 @@
*
!.gitignore

View File

@@ -0,0 +1 @@
#!/usr/bin/env true

View File

@@ -0,0 +1,2 @@
#!/bin/sh
../../urchin --root .testsuite/a/b/c/testcase

View File

@@ -0,0 +1,2 @@
#!/bin/sh
./.meta-assess --root

View File

@@ -0,0 +1,2 @@
#!/bin/sh
./.meta-assess -r

View File

@@ -0,0 +1,2 @@
#!/bin/sh
../../urchin --root /bin 2>&1 | grep "'/bin/.urchin'"

View File

@@ -0,0 +1,2 @@
#!/bin/sh
! ../../urchin --root .testsuite/a/not-a-file

View File

@@ -1,3 +1,5 @@
#!/bin/sh
! ../../urchin ./.chainsaw
tmp=$(mktemp -d)/blah
echo '#!/usr/bin/env true' > $tmp
! ../../urchin $tmp

4
tests/get_stdout_file/t Executable file
View File

@@ -0,0 +1,4 @@
#!/bin/sh
I_AM_SOURCING_URCHIN_IN_A_TEST=true . ../../urchin
test $(get_stdout_file t) = aoeu

1
tests/skip/.test/fail Executable file
View File

@@ -0,0 +1 @@
exit 1

1
tests/skip/.test/skip Executable file
View File

@@ -0,0 +1 @@
exit 3

1
tests/skip/.test/succeed Executable file
View File

@@ -0,0 +1 @@
exit 0

View File

@@ -0,0 +1 @@
test $(../../urchin -t .test/ | grep -c SKIP) -eq 1

View File

@@ -1,3 +0,0 @@
#!/bin/sh
test c = $(../urchin -x .print-arg-3 a 'b b b b' c d e)

366
urchin
View File

@@ -1,19 +1,84 @@
#!/bin/sh
# Make sure that CDPATH isn't set, as it causes `cd` to behave unpredictably - notably, it can produce output,
# which breaks fullpath().
# This file is part of urchin. It is subject to the license terms in the
# COPYING file found in the top-level directory of this distribution or at
# https://raw.githubusercontent.com/tlevine/urchin/master/COPYING
# No part of urchin, including this file, may be copied, modified, propagated,
# or distributed except according to the terms contained in the COPYING file.
set +e
# Make sure that CDPATH isn't set, as it causes `cd` to behave unpredictably -
# notably, it can produce output, which breaks fullpath().
unset CDPATH
fullpath() {
(
cd -- "$1"
pwd
)
# Urchin version number
VERSION=0.0.6
urchin_root() {
# Call recursively but remember the original argument.
current="$(remove_trailing_slash "$1")"
if test -n "$2"; then
orig="$2"
else
orig="$1"
fi
if test "$(readlink -f -- "$1")" = /; then
# Stop traversing upwards at /
if test -d "$orig"; then
origdir="$orig"
else
origdir=$(dirname "$orig")
fi
echo "You need to create the .urchin directory in the root of your tests,
maybe like this:
mkdir '$(readlink -f -- "$(remove_trailing_slash "$origdir")")/.urchin'
" >&2
return 1
elif ! test -e "$current"; then
echo "$current: No such file or directory">&2
return 1
elif test -f "$current"; then
urchin_root "$(dirname "$current")" "$orig"
elif test -d "$current"/.urchin; then
remove_trailing_slash "$current"
elif test "$current" != . && test "$current" != .. &&
echo "$current" | grep '^\.' && > /dev/null; then
# Stop traversing upwards at hidden directories.
urchin_root / "$orig"
else
urchin_root "$current"/.. "$orig"
fi
}
indent() {
level="$1"
printf "%$((2 * ${level}))s"
if test "$level" -gt 0; then
printf "%$((2 * ${level}))s"
fi
}
remove_trailing_slash() {
echo "$1" | sed s/\\/$//
}
escape_slashes() {
echo "$1" | sed s+/+\\\\/+g
}
get_stdout_file() {
root="$(readlink -f -- "$(urchin_root "$1")")"
test_file="$(readlink -f -- "$1")"
eroot="$(escape_slashes "$root")"
base="$(echo "$test_file" | sed "s/^$eroot/$eroot\/.urchin/")"
if test -d "$1"; then
echo "$base"/.index
else
echo "$base"
fi
}
recurse() {
@@ -21,18 +86,29 @@ recurse() {
indent_level="$2"
shell_for_sh_tests="$3"
[ "$potential_test" = '.urchin' ] && return
[ "$potential_test" = 'setup_dir' ] && return
[ "$potential_test" = 'teardown_dir' ] && return
[ "$potential_test" = 'setup' ] && return
[ "$potential_test" = 'teardown' ] && return
stdout_file="$(get_stdout_file "$potential_test")"
mkdir -p "$(dirname "$stdout_file")"
[ $indent_level -eq 0 ] && : > "$stdout_file"
if [ -d "$potential_test" ]
then
(
if $tap_format; then
indent $indent_level | sed 's/ /#/g'
echo "# Begin - ${potential_test}"
else
indent $indent_level
echo " ${potential_test}"
echo "+ ${potential_test}"
fi
(
cd -- "$potential_test"
[ -f setup_dir ] && [ -x setup_dir ] && ./setup_dir >> "$stdout_file"
@@ -46,53 +122,112 @@ recurse() {
[ -f setup ] && [ -x setup ] && ./setup >> "$stdout_file"
# $2 instead of $indent_level so it doesn't clash
recurse "${test}" $(( $2 + 1 )) "$shell_for_sh_tests"
set +e
recurse "${test}" "$(( $2 + 1 ))" "$shell_for_sh_tests"
exit_code=$?
set -e
if $exit_on_fail && test $exit_code -ne 0; then
[ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
[ -f teardown_dir ] && [ -x teardown_dir ] && ./teardown_dir >> "$stdout_file"
return 1
fi
[ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
done
[ -f teardown_dir ] && [ -x teardown_dir ] && ./teardown_dir >> "$stdout_file"
echo
)
elif [ -x "$potential_test" ]
then
[ -f setup ] && [ -x setup ] && ./setup >> "$stdout_file"
# Run the test
if [ -n "$shell_for_sh_tests" ] && has_sh_or_no_shebang_line ./"$potential_test"
then
TEST_SHELL="$TEST_SHELL" "$shell_for_sh_tests" ./"$potential_test" > "$stdout_file" 2>&1
if $tap_format; then
indent $indent_level | sed 's/ /#/g'
echo "# End - ${potential_test}"
else
TEST_SHELL="$TEST_SHELL" ./"$potential_test" > "$stdout_file" 2>&1
echo
fi
exit_code="$?"
[ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
indent $indent_level
if [ $exit_code -eq 0 ]
else
if [ -x "$potential_test" ]
then
# On success, print a green '✓'
printf '\033[32m✓ \033[0m'
printf '%s\n' "${potential_test}"
printf '%s\n' "${potential_test} passed" >> "$logfile"
[ -f setup ] && [ -x setup ] && ./setup >> "$stdout_file"
# Run the test
if [ -n "$shell_for_sh_tests" ] && has_sh_or_no_shebang_line ./"$potential_test"
then
TEST_SHELL="$TEST_SHELL" "$shell_for_sh_tests" ./"$potential_test" > "$stdout_file" 2>&1
else
TEST_SHELL="$TEST_SHELL" ./"$potential_test" > "$stdout_file" 2>&1
fi
exit_code="$?"
[ -f teardown ] && [ -x teardown ] && ./teardown >> "$stdout_file"
if [ $exit_code -eq 0 ]; then
result=success
elif [ $exit_code -eq 3 ]; then
result=skip
else
result=fail
fi
else
# On fail, print a red '✗'
printf '\033[31m✗ \033[0m'
printf '%s\n' "${potential_test}"
printf '%s\n' "${potential_test} failed" >> "$logfile"
printf '\033[31m' # Print output captured from failed test in red.
cat "$stdout_file"
printf '\033[0m'
result=skip
fi
echo "${result}" >> "$logfile"
if $tap_format; then
n=$(grep -ce '^\(success\|fail\|skip\)' "$logfile")
if [ "$result" == fail ]; then
not='not '
else
not=''
fi
if [ "$result" == skip ]; then
skip='# SKIP '
else
skip=''
fi
echo "${not}ok $n - ${skip}${potential_test}"
if [ "$result" == fail ]; then
echo '# ------------ Begin output ------------'
sed 's/^/# /' "$stdout_file"
echo '# ------------ End output ------------'
fi
else
indent $indent_level
case "$result" in
success)
# On success, print a green '✓'
printf '\033[32m✓ \033[0m'
printf '%s\n' "${potential_test}"
;;
fail)
# On fail, print a red '✗'
printf '\033[31m✗ \033[0m'
printf '%s\n' "${potential_test}"
# Print output captured from failed test in red.
printf '\033[31m'
cat "$stdout_file"
printf '\033[0m'
;;
skip)
printf ' %s\n' "${potential_test}"
;;
esac
fi
if $exit_on_fail && test 0 -ne $exit_code; then
return 1
fi
fi
[ $indent_level -eq 0 ] && rm "$stdout_file"
}
has_sh_or_no_shebang_line() {
head -n 1 "$1" | grep -vqE '^#!' && return 0 # no shebang line at all
head -n 1 "$1" | grep -qE '^#![[:blank:]]*/bin/sh($|[[:blank:]])' && return 0 # shebang line is '#!/bin/sh' or legal variations thereof
# no shebang line at all
head -n 1 "$1" | grep -vqE '^#!' && return 0
# shebang line is '#!/bin/sh' or legal variations thereof
head -n 1 "$1" | grep -qE '^#![[:blank:]]*/bin/sh($|[[:blank:]])' &&
return 0
return 1
}
@@ -105,17 +240,19 @@ $USAGE
-s <shell> Invoke test scripts that either have no shebang line at all or
have shebang line "#!/bin/sh" with the specified shell.
-e Stop running if any single test fails. This is helpful if you want
to use Urchin to run things other than tests, such as a set of
configuration scripts.
-f Force running even if the test directory's name does not
contain the word "test".
-h This help.
-t Format output in Test Anything Protocol (TAP)
-h, --help This help.
-r, --root Print the Urchin root for a particular file or directory.
-v Display the version number.
Go to https://github.com/tlevine/urchin for documentation on writing tests.
EOF
# [Experimental -x option left undocumented for now.]
# -x [Experimental; not meant for direct invocation, but for use in
# the shebang line of test scripts]
# Run with "\$TEST_SHELL", falling back on /bin/sh.
}
plural () {
@@ -131,8 +268,11 @@ plural () {
}
urchin_go() {
echo Running tests at $(date +%Y-%m-%dT%H:%M:%S) | tee "$logfile"
start=$(date +%s)
if "$tap_format"; then
printf \#\
fi
echo Running tests at $(date +%Y-%m-%dT%H:%M:%S)
start="$(date +%s)"
# Determine the environment variable to define for test scripts
# that reflects the specified or implied shell to use for shell-code tests.
@@ -147,17 +287,33 @@ urchin_go() {
TEST_SHELL='/bin/sh'
fi
recurse "$1" 0 "$2" # test folder -- indentation level -- [shell to invoke test scripts with]
# 1 test folder
# 2 indentation level
# 3 shell to invoke test scripts with
recurse "$1" 0 "$2"
finish=$(date +%s)
elapsed=$(($finish - $start))
echo "Done, took $elapsed $(plural second $elapsed)."
set -- $(grep -e 'passed$' "$logfile"|wc -l) $(grep -e 'failed$' "$logfile"|wc -l)
printf '%s\n' "$1 $(plural test "$1") passed."
[ $2 -gt 0 ] && printf '\033[31m' || printf '\033[32m' # If tests failed, print the message in red, otherwise in green.
printf '%s\n' "$2 $(plural test "$2") failed."
printf '\033[m'
return "$2"
passed=$(grep -c '^success' "$logfile")
failed=$(grep -c '^fail' "$logfile")
skipped=$(grep -c '^skip' "$logfile")
if $tap_format; then
echo "# Took $elapsed $(plural second $elapsed)."
echo 1..$(($passed + $failed + $skipped))
else
echo "Done, took $elapsed $(plural second $elapsed)."
printf '%s\n' "$passed $(plural test "$passed") passed."
printf '%s\n' "$skipped $(plural test "$skipped") skipped."
# If tests failed, print the message in red, otherwise in green.
[ $failed -gt 0 ] && printf '\033[31m' || printf '\033[32m'
printf '%s\n' "$failed $(plural test "$failed") failed."
printf '\033[m'
fi
# rm -f "$logfile"
test -z "$failed" || test "$failed" -eq '0'
}
urchin_molly_guard() {
@@ -172,47 +328,57 @@ urchin_molly_guard() {
exit 1
}
shell_for_sh_tests=
force=false
while [ $# -gt 0 ]
do
case "$1" in
-f) force=true;;
-s)
shift
shell_for_sh_tests=$1
which "$shell_for_sh_tests" >/dev/null || { echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2; urchin_help >&2; exit 2; }
;;
-x) # [EXPERIMENTAL; UNDOCUMENTED FOR NOW] `urchin -x <test-script>` in a test script's shebang line is equivalent to invoking that script with `"$TEST_SHELL" <test-script>`
shift
urchinsh=${TEST_SHELL:-/bin/sh}
"$urchinsh" "$@"
exit $?;;
-h|--help) urchin_help
exit 0;;
-*) urchin_help >&2
exit 1;;
*) break;;
esac
shift
done
if test -z "$I_AM_SOURCING_URCHIN_IN_A_TEST"; then
shell_for_sh_tests=
force=false
exit_on_fail=false
tap_format=false
while [ $# -gt 0 ]
do
case "$1" in
-e) exit_on_fail=true;;
-f) force=true;;
-r|--root)
shift
urchin_root "$1"
exit;;
-s)
shift
shell_for_sh_tests=$1
which "$shell_for_sh_tests" > /dev/null || {
echo "Cannot find specified shell: '$shell_for_sh_tests'" >&2
urchin_help >&2
exit 11
} ;;
-t) tap_format=true;;
-h|--help) urchin_help
exit 0;;
-v) echo "$VERSION"
exit;;
-*) urchin_help >&2
exit 1;;
*) break;;
esac
shift
done
# Verify argument for main stuff
if [ "$#" != '1' ] || [ ! -d "$1" ]
then
[ -n "$1" ] && [ ! -d "$1" ] && echo "Not a directory: '$1'" >&2
echo "$USAGE" >&2
exit 2
fi
# Constants
logfile=$(fullpath "$1")/.urchin.log
stdout_file=$(fullpath "$1")/.urchin_stdout
# Run or present the Molly guard.
if basename "$(fullpath "$1")" | grep -Fi 'test' > /dev/null || $force
then
urchin_go "$1" "$shell_for_sh_tests"
else
urchin_molly_guard
# Verify argument for main stuff
if [ "$#" != '1' ] || [ ! -d "$1" ]
then
[ -n "$1" ] && [ ! -d "$1" ] && echo "Not a directory: '$1'" >&2
echo "$USAGE" >&2
exit 11
fi
# Run or present the Molly guard.
if ! urchin_root "$1" > /dev/null; then
exit 1
elif basename "$(readlink -f -- "$(urchin_root "$1")")" |
grep -Fi 'test' > /dev/null || $force; then
logfile="$(readlink -f -- "$(urchin_root "$1")/.urchin/.log")"
printf '' > "$logfile"
urchin_go "$1" "$shell_for_sh_tests"
else
urchin_molly_guard
fi
fi

BIN
urchin_test_on_msys64_1.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 99 KiB

BIN
urchin_test_on_msys64_2.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 115 KiB