diff --git a/test/sharness/.gitignore b/test/sharness/.gitignore new file mode 100755 index 000000000..782418fd1 --- /dev/null +++ b/test/sharness/.gitignore @@ -0,0 +1,4 @@ +lib/sharness/ +test-results/ +trash directory.*/ +plugins diff --git a/test/sharness/Makefile b/test/sharness/Makefile new file mode 100755 index 000000000..39af9aa2b --- /dev/null +++ b/test/sharness/Makefile @@ -0,0 +1,50 @@ +# Run tests + +SHELL_PATH ?= $(SHELL) +SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH)) +RM ?= rm -f +PROVE ?= prove +LIBDIR = lib +SHARNESSDIR = sharness +AGGREGATE_SCRIPT ?= $(LIBDIR)/$(SHARNESSDIR)/aggregate-results.sh +DEFAULT_TEST_TARGET ?= test + +T = $(sort $(wildcard *.t)) + +all: $(DEFAULT_TEST_TARGET) + +test: pre-clean aggregate-results-and-cleanup + +$(T): sharness + + @echo "*** $@ ***"; '$(SHELL_PATH_SQ)' $@ $(TEST_OPTS) + +pre-clean: + @echo "*** $@ ***" + $(RM) -r test-results + +clean: + @echo "*** $@ ***" + $(RM) -r test-results + +aggregate-results-and-cleanup: $(T) aggregate-results clean + +aggregate-results: + @echo "*** $@ ***" + @for f in test-results/*.counts; do \ + echo "$$f"; \ + done | '$(SHELL_PATH_SQ)' '$(AGGREGATE_SCRIPT)' + +sharness: + @echo "*** checking $@ ***" + lib/install-sharness.sh + +prove: clean + @echo "*** prove ***"; $(PROVE) --exec '$(SHELL_PATH_SQ)' $(PROVE_OPTS) $(T) :: $(TEST_OPTS) + $(MAKE) clean-prove-cache + +clean-prove-cache: + $(RM) .prove + +.PHONY: all test prove $(T) pre-clean clean sharness +.PHONY: aggregate-results-and-cleanup aggregate-results diff --git a/test/sharness/README.md b/test/sharness/README.md new file mode 100755 index 000000000..e787824dd --- /dev/null +++ b/test/sharness/README.md @@ -0,0 +1,71 @@ +# aragonCLI-sharness-tests + +AragonCLI whole tests using the [Sharness framework](https://github.com/chriscool/sharness/) + +## Running all the tests + +Just use `make` in this directory to run all the tests. + +## Running just one test + +You can run only one test script by launching it like a regular shell +script: + +``` +$ ./sharness.t +``` + +## Command-line options + +The `*.t` test scripts have the following options: + +- `--debug`, `-d`: helps debugging +- `--immediate`, `-i`: stop execution after the first failing test +- `--long-tests`, `-l`: run tests marked with prereq EXPENSIVE +- `--interactive-tests`: run tests marked with prereq INTERACTIVE +- `--help`, `-h`: show test description +- `--verbose`, `-v`: show additional debug output +- `--quiet`, `-q`: show less output +- `--chain-lint`/`--no-chain-lint`: check &&-chains in scripts +- `--no-color`: don't color the output +- `--tee`: also write output to a file +- `--verbose-log`: write output to a file, but not on stdout +- `--root=`: create trash directories in `` instead of current directory. + +## Sharness + +When running sharness tests from main Makefile, dependencies for sharness +will be downloaded from its github repo and installed in a "lib/sharness" +directory. + +Please do not change anything in the "lib/sharness" directory. + +## Writing Tests + +Start with the [sharness API](https://github.com/chriscool/sharness/blob/master/API.md) and then please have a look at existing tests and try to follow their example. + +It should be possible to put most of the code inside `test_expect_success`, +or sometimes `test_expect_failure`, blocks, and to chain all the commands +inside those blocks with `&&`, or `||` for diagnostic commands. + +### Diagnostics + +Make your test case output helpful for when running sharness verbosely. +This means cating certain files, or running diagnostic commands. +For example: + +``` +test_expect_success ".ipfs/ has been created" ' + test -d ".ipfs" && + test -f ".ipfs/config" && + test -d ".ipfs/datastore" && + test -d ".ipfs/blocks" || + test_fsh ls -al .ipfs +' +``` + +The `|| ...` is a diagnostic run when the preceding command fails. +test*fsh is a shell function that echoes the args, runs the cmd, +and then also fails, making sure the test case fails. (wouldnt want +the diagnostic accidentally returning true and making it \_seem* like +the test case succeeded!). diff --git a/test/sharness/aragon_help.t b/test/sharness/aragon_help.t new file mode 100644 index 000000000..dd58cf7ae --- /dev/null +++ b/test/sharness/aragon_help.t @@ -0,0 +1,46 @@ +#!/bin/sh + +test_description="Test aragon --help command" + +. ./lib/test-lib.sh + +TEXT_IT_SHOULD_INCLUDE="For more information, check out https://hack.aragon.org" + +test_expect_success "'aragon --help' succeeds" ' + aragon --help +' + +# test_expect_success "'aragon --help' is fast" ' +# # silent the output of aragon --help by redirecting it to: /dev/null +# # redirect the output of time (which goes to stderr/2 normally) to stdout/1 with: 2>&1 + +# OUTPUT=$(TIMEFORMAT='%lR'; time (aragon --help > /dev/null) 2>&1) && +# echo "$OUTPUT" + +# if [[ $OUTPUT == "0m5.883s" ]]; #TODO find a way to parse this and compare it +# then return 0 +# else return 1 +# fi +# ' + +# pipe output to grep +test_expect_success "'aragon --help' output includes the hack website (1)" ' + aragon --help | grep "$TEXT_IT_SHOULD_INCLUDE" +' + +# save output to file and use grep +test_expect_success "'aragon --help' output includes the hack website (2)" ' + aragon --help > actual && + grep "$TEXT_IT_SHOULD_INCLUDE" actual +' + +# save output to variable and use grep +test_expect_success "'aragon --help' output includes the hack website (3)" ' + OUTPUT=$(aragon --help) && + if [[ $OUTPUT =~ "$TEXT_IT_SHOULD_INCLUDE" ]]; + then return 0 + else return 1 + fi +' + +test_done diff --git a/test/sharness/aragon_init.t b/test/sharness/aragon_init.t new file mode 100755 index 000000000..212d000a1 --- /dev/null +++ b/test/sharness/aragon_init.t @@ -0,0 +1,26 @@ +#!/bin/sh + +test_description="Test aragon init command" + +. ./lib/test-lib.sh + +APP_NAME="test-app" + +test_expect_success "'aragon init' succeeds" ' + test_might_fail aragon init "$APP_NAME" +' + +# test arapp.json have app name updated +test_expect_success "arapp.json appName updated" ' + echo 3 > result + grep "\"appName\": \""$APP_NAME".*.aragonpm.eth" "$APP_NAME"/arapp.json -c > matchs + test_cmp result matchs +' + +#test if project already exists +test_expect_success "project already exists" ' + test_must_fail aragon init "$APP_NAME" > output.txt && + grep "Project with name "$APP_NAME" already exists" output.txt +' + +test_done \ No newline at end of file diff --git a/test/sharness/aragon_ipfs.t b/test/sharness/aragon_ipfs.t new file mode 100644 index 000000000..1f644807d --- /dev/null +++ b/test/sharness/aragon_ipfs.t @@ -0,0 +1,37 @@ +#!/bin/sh + +test_description="Test aragon ipfs command" + +. ./lib/test-lib.sh + +test_launch_aragon_ipfs() { + test_expect_success "'aragon ipfs' succeeds" ' + aragon ipfs > out_file 2> err_file & + PID=$! && + echo "Process ID of the shell running this: $PID" && + PGID=$(test_get_pgid_from_pid $PID) && + echo "Process Group ID: $PGID" + ' + # TODO Memory leak: tail never finishes + # TODO process substitution is not portable, only works in bash + test_expect_success "'aragon ipfs' says all good" ' + timeout 180 grep -m 1 "running" <(tail -f out_file) + ' + # TODO test_curl_resp_http_code "http://localhost:5001/api/v0/version" "HTTP/1.1 200 OK" + # TODO test check aragon-cache "http://127.0.0.1:$API_PORT/ipfs/$HASH" "HTTP/1.1 200 OK" +} + +test_kill_ipfs() { + # -- is the default, which also means -15 or -TERM, sending the Termination signal + # consider doing a graceful shutdown with -INT (-2) (interrupt - same as CTRL+C) + # or using -KILL (-9) if a clean termination does not work + test_expect_failure "'aragon ipfs' can be terminated" ' + kill -9 -$PGID + ' +} + +test_launch_aragon_ipfs + +test_kill_ipfs + +test_done diff --git a/test/sharness/aragon_version.t b/test/sharness/aragon_version.t new file mode 100644 index 000000000..396afa0b8 --- /dev/null +++ b/test/sharness/aragon_version.t @@ -0,0 +1,30 @@ +#!/bin/sh + +test_description="Test aragon --version command" + +. ./lib/test-lib.sh + +EXPECTED_OUTPUT="5.3.3" + +test_expect_success "'aragon --version' succeeds" ' + aragon --version +' + +# save outputs to files and compare +test_expect_success "'aragon --version' output looks good (1)" ' + aragon --version > actual && + echo $EXPECTED_OUTPUT > expected && + test_cmp expected actual +' + +# save the output in a variable and compare +test_expect_success "'aragon --version' output looks good (2)" ' + OUTPUT=$(aragon --version) && + + if [[ $OUTPUT == $EXPECTED_OUTPUT ]]; + then return 0 + else return 1 + fi +' + +test_done \ No newline at end of file diff --git a/test/sharness/create-aragon-app.t b/test/sharness/create-aragon-app.t new file mode 100755 index 000000000..39ea501a1 --- /dev/null +++ b/test/sharness/create-aragon-app.t @@ -0,0 +1,26 @@ +#!/bin/sh + +test_description="Test create-aragon-app command" + +. ./lib/test-lib.sh + +APP_NAME="test-app" + +test_expect_success "'create-aragon-app' succeeds" ' + test_might_fail npx create-aragon-app "$APP_NAME" +' + +# test arapp.json have app name updated +test_expect_success "arapp.json appName updated" ' + echo 3 > result + grep "\"appName\": \""$APP_NAME".*.aragonpm.eth" "$APP_NAME"/arapp.json -c > matchs + test_cmp result matchs +' + +#test if project already exists +test_expect_success "project already exists" ' + test_must_fail npx create-aragon-app "$APP_NAME" > output.txt && + grep "Project with name "$APP_NAME" already exists" output.txt +' + +test_done \ No newline at end of file diff --git a/test/sharness/lib/install-sharness.sh b/test/sharness/lib/install-sharness.sh new file mode 100755 index 000000000..6511a9223 --- /dev/null +++ b/test/sharness/lib/install-sharness.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# install sharness.sh +# + +# settings +version=827ec00027125fc0ba56397beb05c15b06a67606 +urlprefix=https://github.com/chriscool/sharness.git +if test ! -n "$clonedir" ; then + clonedir=lib +fi +sharnessdir=sharness + +if test -f "$clonedir/$sharnessdir/SHARNESS_VERSION_$version" +then + # There is the right version file. Great, we are done! + exit 0 +fi + +die() { + echo >&2 "$@" + exit 1 +} + +checkout_version() { + git checkout "$version" || die "Could not checkout '$version'" + rm -f SHARNESS_VERSION_* || die "Could not remove 'SHARNESS_VERSION_*'" + touch "SHARNESS_VERSION_$version" || die "Could not create 'SHARNESS_VERSION_$version'" + echo "Sharness version $version is checked out!" +} + +if test -d "$clonedir/$sharnessdir/.git" +then + # We need to update sharness! + cd "$clonedir/$sharnessdir" || die "Could not cd into '$clonedir/$sharnessdir' directory" + git fetch || die "Could not fetch to update sharness" + checkout_version +else + # We need to clone sharness! + mkdir -p "$clonedir" || die "Could not create '$clonedir' directory" + cd "$clonedir" || die "Could not cd into '$clonedir' directory" + + git clone "$urlprefix" || die "Could not clone '$urlprefix'" + cd "$sharnessdir" || die "Could not cd into '$sharnessdir' directory" + checkout_version +fi +exit 0 diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh new file mode 100644 index 000000000..33c035745 --- /dev/null +++ b/test/sharness/lib/test-lib.sh @@ -0,0 +1,41 @@ + +SHARNESS_LIB="lib/sharness/sharness.sh" + +. "$SHARNESS_LIB" || { + echo >&2 "Cannot source: $SHARNESS_LIB" + echo >&2 "Please check Sharness installation." + exit 1 +} + +# Quote arguments for sh eval +shellquote() { + _space='' + for _arg + do + # On Mac OS, sed adds a newline character. + # With a printf wrapper the extra newline is removed. + printf "$_space'%s'" "$(printf "%s" "$_arg" | sed -e "s/'/'\\\\''/g;")" + _space=' ' + done + printf '\n' +} + +# Echo the args, run the cmd, and then also fail, +# making sure a test case fails. +test_fsh() { + echo "> $@" + eval $(shellquote "$@") + echo "" + false +} + +# Same as sharness' test_cmp but using test_fsh (to see the output). +# We have to do it twice, so the first diff output doesn't show unless it's +# broken. +test_cmp() { + diff -q "$@" >/dev/null || test_fsh diff -u "$@" +} + +test_get_pgid_from_pid() { + awk '{print $5}' < /proc/$1/stat +} diff --git a/test/sharness/sharness.t b/test/sharness/sharness.t new file mode 100755 index 000000000..fa48184e0 --- /dev/null +++ b/test/sharness/sharness.t @@ -0,0 +1,33 @@ +#!/bin/sh + +test_description="Show basic features of Sharness" + +. ./lib/sharness/sharness.sh + + +test_expect_success "Success is reported like this" " + echo hello world | grep hello +" + +test_expect_success "Commands are chained this way" " + test x = 'x' && + test 2 -gt 1 && + echo success +" + +return_42() { + echo "Will return soon" + return 42 +} + +test_expect_success "You can test for a specific exit code" " + test_expect_code 42 return_42 +" + +test_expect_failure "We expect this to fail" " + test 1 = 2 +" + +test_done + +# vi: set ft=sh :