(PDB-3993) Add ./test-config; add pgver/pgbin/pgport opts

Move test config handling to ext/bin/test-config, which allows you to
--set, --get, or --reset (undo any previous --set-ting for the source
tree) various test config values including the previously existing
puppet and puppetserver integration test refs, and new pgport, pgver,
and pgbin settings.

The general idea is that tests can use test-config --get to determine
what default value to use for these settings, which can be adjusted
for the local source tree via --set.

By default, try to detect the pgbin directory automatically, based on
the pgver value, which also allows "test-config --get pgbin" to
replace the previously travis-specific pgbin-from-spec command.

Always require the pgport to be specified somehow, either via a
--pgport argument to a given test, or for the whole tree via
"test-config --set pgport PORT".

Record the tree's default pgver in ext ext/test-conf/pgver-default.

.gitignore

@@ -52,3 +52,13 @@ checkouts
 /resources/locales.clj
 /mp-*
 /dev-resources/i18n/bin
+
+/.bundle/
+/ext/test-conf/pgbin-requested
+/ext/test-conf/pgport-requested
+/ext/test-conf/pgver-requested
+/ext/test-conf/puppet-ref-requested
+/ext/test-conf/puppetserver-dep
+/ext/test-conf/puppetserver-ref-requested
+/puppet/state/
+/puppetserver/

.travis.yml

@@ -11,28 +11,43 @@ sudo: required
 # prep-os-essentials-for:
 # https://github.com/travis-ci/travis-ci/issues/9927
 
+# We explicitly set up lein and pgbox at the top level so that we can
+# use them in commands like test-config, and so we can't end up doing
+# it multiple times if any of the (e.g. boxed-) sub-commands also make
+# the attempt.
+
 aliases:
   - &run-core-tests |
     set -e
     jdk="$(ext/travisci/jdk-from-spec "$PDB_TEST")"
     if test "$(uname)" = Linux; then jdk_switcher use "$jdk"; fi
     ext/travisci/prep-os-essentials-for "$PDB_TEST"
-    pgbin="$(ext/travisci/pgbin-from-spec "$PDB_TEST")"
-    ext/bin/boxed-core-tests --pgbin "$pgbin" -- lein test
+    mkdir -p ext/travisci/local
+    export PATH="$(pwd)/ext/travisci/local/bin:$PATH"
+    ext/bin/require-leiningen default ext/travisci/local
+    ext/bin/require-pgbox default ext/travisci/local
+    pgver="$(ext/travisci/prefixed-ref-from-spec "$PDB_TEST" pg-)"
+    ext/bin/test-config --set pgver "$pgver"
+    ext/bin/test-config --set pgport 34335
+    ext/bin/boxed-core-tests -- lein test
 
   - &run-integration-tests |
     set -e
     jdk="$(ext/travisci/jdk-from-spec "$PDB_TEST")"
     if test "$(uname)" = Linux; then jdk_switcher use "$jdk"; fi
     ext/travisci/prep-os-essentials-for "$PDB_TEST"
-    pgbin="$(ext/travisci/pgbin-from-spec "$PDB_TEST")"
+    mkdir -p ext/travisci/local
+    export PATH="$(pwd)/ext/travisci/local/bin:$PATH"
+    ext/bin/require-leiningen default ext/travisci/local
+    ext/bin/require-pgbox default ext/travisci/local
+    pgver="$(ext/travisci/prefixed-ref-from-spec "$PDB_TEST" pg-)"
     puppet="$(ext/travisci/prefixed-ref-from-spec "$PDB_TEST" pup-)"
     server="$(ext/travisci/prefixed-ref-from-spec "$PDB_TEST" srv-)"
-    ext/bin/boxed-integration-tests \
-      --pgbin "$pgbin" \
-      --puppet "$puppet" \
-      --server "$server" \
-      -- lein test :integration
+    ext/bin/test-config --set pgver "$pgver"
+    ext/bin/test-config --set pgport 34335
+    ext/bin/test-config --set puppet-ref "$puppet"
+    ext/bin/test-config --set puppetserver-ref "$server"
+    ext/bin/boxed-integration-tests -- lein test :integration
 
 jobs:
   include:

Gemfile

@@ -5,9 +5,9 @@ oldest_supported_puppet = "5.0.0"
 beaker_version = ENV['BEAKER_VERSION']
 
 begin
-  puppet_ref = File.read(gemfile_home + '/ext/test-conf/puppet-requested').strip
+  puppet_ref = File.read(gemfile_home + '/ext/test-conf/puppet-ref-requested').strip
 rescue Errno::ENOENT
-  puppet_ref = File.read(gemfile_home + '/ext/test-conf/puppet-default').strip
+  puppet_ref = File.read(gemfile_home + '/ext/test-conf/puppet-ref-default').strip
 end
 
 def location_for(place, fake_version = nil)

documentation/CONTRIBUTING.md

@@ -56,48 +56,91 @@ top of things.
 ### Testing
 
 The easiest way to run the tests until you need to do it often is to
-use the built-in sandbox harness.  Assuming you the PostgreSQL
-binaries installed in /usr/lib/posgresql/9.6/bin, then you should be
-able to run the core tests like this:
+use the built-in sandbox harness.  You'll need to either specify the
+PostgreSQL port you'd like the sandbox to use by providing a `--pgport
+PORT` argument to each relevant test invocation, or you can set a
+default for the source tree:
 
-    $ ext/bin/boxed-core-tests \
-        --pgbin /usr/lib/posgresql/9.6/bin \
-        -- lein test
+    $ ./ext/bin/test-config --set pgport 34335
+
+Once you've set the default pgport, you should be able to run the core
+tests like this:
+
+    $ ext/bin/boxed-core-tests -- lein test
 
 Copies of tools like `lein` and `pgbox` may be downloaded and
 installed to a temporary directory during the process, if you don't
 already have the expected versions.
 
-Similarly you should be able to run the integration tests against the
-default Puppet and Puppetserver versions like this:
+Similarly you should be able to configure and run the integration
+tests against the default Puppet and Puppetserver versions like this:
 
+    $ ext/bin/test-config --reset puppet-ref
+    $ ext/bin/test-config --reset puppetserver-ref
     $ ext/bin/boxed-integration-tests \
-        --pgbin /usr/lib/posgresql/9.6/bin \
         -- lein test :integration
 
-You can also select the integration test versions of puppet and
-puppetserver with the `--puppet REF` and `--server REF` arguments.
+Note that you only need to configure the puppet-ref and
+puppetserver-ref once for each tree, but you can also change the refs
+when you like with `test-config`:
+
+    $ ext/bin/test-config --set puppet-ref 5.3.x
+    $ ext/bin/test-config --set puppetserver-ref 5.1.x
+
+Running `--reset` for an option resets it to the tree default, and at
+the moment, you'll need to do that manually whenever you're using the
+default and the relevant `*-default` file in ext/test-conf changes in
+the source.
 
 The sandboxes are destroyed when the commands finish, but you can
 arrange to inspect the environment after a failure like this:
 
     $ ext/bin/boxed-integration-tests \
-        --pgbin /usr/lib/posgresql/9.6/bin \
         -- bash -c 'lein test || bash'
 
 which will drop you into a shell if anything goes wrong.
 
+To run the local rspec tests (e.g. for the PuppetDB terminus code),
+you must have configured the `puppet-ref` via `ext/bin/test-config` as
+described above, and then from within the `puppet/` directory you can
+run:
+
+    $ bundle exec rspec spec
+
+If you'd like to preserve the temporary test databases on failure, you can
+set `PDB_TEST_PRESERVE_DB_ON_FAIL` to true:
+
+    $ PDB_TEST_KEEP_DB_ON_FAIL=true lein test
+
+The sandboxed tests will try to find and use the version of PostgreSQL
+specified by:
+
+    $ ./ext/bin/test-config --get pgver
+
+Unless you override that with `test-config`:
+
+    $ ext/bin/test-config --set pgver 9.6
+
+Given just the version, the tests will try to find a suitable
+PostgreSQL installation, but you can specify one directly like this:
+
+    $ ext/bin/test-config --set pgbin /usr/lib/postgresql/9.6/bin
+
+at which point the pgver setting will be irrelevant until/unless you
+reset pgbin:
+
+    $ ext/bin/test-config --reset pgbin
+
 If you're running the tests all the time, you might want to set up
 your own persistent sandbox instead (`ext/bin/with-pdbbox` does
-something similar) so you can run them directly:
+something similar) so you can run tests directly against that:
 
     $ ext/bin/pdbbox-init \
       --sandbox ./test-sandbox \
       --pgbin /usr/lib/postgresql-9.6/bin \
       --pgport 17961
 
-After that you can start and stop the included database server like
-this:
+Then you can start and stop the included database server like this:
 
     $ export PDBBOX="$(pwd)/test-sandbox"
     $ ext/bin/pdbbox-env pg_ctl start -w
@@ -114,15 +157,15 @@ configure the puppet and puppetserver versions you want to use.
 Assuming you have suitable versions of Ruby and Bundler available, you
 can do this:
 
-    $ ext/bin/config-puppet-test-ref
-    $ ext/bin/config-puppetserver-test-ref
+    $ ext/bin/test-config --reset puppet-ref
+    $ ext/bin/test-config --reset puppetserver-ref
 
 The default puppet and puppetserver versions are recorded in
 `ext/test-conf/`.  You can request specific versions of puppet or
-puppetserver by specifying arguments to the config tools like this:
+puppetserver like this:
 
-    $ ext/bin/config-puppet-test-ref 5.3.x
-    $ ext/bin/config-puppetserver-test-ref 5.1.x
+    $ ext/bin/test-config --set puppet-ref 5.3.x
+    $ ext/bin/test-config --set puppetserver-ref 5.3.x
 
 Run the tools again to change the requested versions, and `lein
 distclean` will completely undo the configurations.
@@ -140,17 +183,6 @@ sandbox:
     $ ext/bin/pdbbox-env lein run services \
         -c test-sandbox/pdb.ini
 
-To run the local rspec tests (e.g. for the PuppetDB terminus code),
-you must have run `config-puppet-test-ref` as described above, and
-then from within the `puppet/` directory run:
-
-    $ bundle exec rspec spec
-
-If you'd like to preserve the temporary test databases on failure, you can
-set `PDB_TEST_PRESERVE_DB_ON_FAIL` to true:
-
-    $ PDB_TEST_KEEP_DB_ON_FAIL=true lein test
-
 And finally, you can of course set up and [configure your own
 PostgreSQL server][configure_postgres] for testing, but then you'll
 need to create the test users:

ext/bin/boxed-core-tests

@@ -2,18 +2,17 @@
 
 set -uxeo pipefail
 
-usage() { echo 'Usage: $(basename "$0") --pgbin DIR -- CMD ...'; }
+usage() { echo 'Usage: $(basename "$0") [--pgbin DIR] [--pgport PORT] -- CMD ...'; }
 misuse() { usage 1>&2; exit 2; }
 
-pgbin=''
+declare -A opt
 
 while test $# -gt 0; do
     case "$1" in
-        --pgbin)
-            shift
-            test $# -gt 0 || misuse
-            pgbin="$1"
-            shift
+        --pgbin|--pgport)
+            test $# -gt 1 || misuse
+            opt["${1:2}"]="$2"
+            shift 2
             ;;
         --)
             shift
@@ -25,7 +24,21 @@ while test $# -gt 0; do
 done
 cmd=("$@")
 
-test "$pgbin" || misuse
+if test -z "${opt[pgbin]:-}"; then
+    opt[pgbin]="$(ext/bin/test-config --get pgbin)"
+    if test  -z "${opt[pgbin]:-}"; then
+        echo 'Please specify --pgbin or set pgbin with ext/bin/test-config' 1>&2
+        exit 2
+    fi
+fi
+
+if test -z "${opt[pgport]:-}"; then
+    opt[pgport]="$(ext/bin/test-config --get pgport)"
+     if test  -z "${opt[pgport]:-}"; then
+        echo 'Please specify --pgport or set pgport with ext/bin/test-config' 1>&2
+        exit 2
+    fi
+fi
 
 tmpdir="$(mktemp -d "core-test-XXXXXX")"
 tmpdir="$(cd "$tmpdir" && pwd)"
@@ -36,5 +49,7 @@ ext/bin/require-leiningen default "$tmpdir/local"
 ext/bin/require-pgbox default "$tmpdir/local"
 export PATH="$tmpdir/local/bin:$PATH"
 ext/bin/host-info
-ext/bin/with-pdbbox --box "$tmpdir/box" --pgbin "$pgbin" --pgport 34335 \
+ext/bin/with-pdbbox --box "$tmpdir/box" \
+                    --pgbin "${opt[pgbin]}" \
+                    --pgport "${opt[pgport]}" \
                     -- "${cmd[@]}"

ext/bin/boxed-integration-tests

@@ -3,34 +3,19 @@
 set -ueo pipefail
 
 usage() {
-    echo 'Usage: $(basename "$0") --pgbin DIR --puppet REF --server REF -- CMD ...'
+    echo 'Usage: $(basename "$0") [--pgbin DIR] [--pgport PORT] -- CMD ...'
 }
 
 misuse() { usage 1>&2; exit 2; }
 
-pup_ref=''
-pupsrv_ref=''
-pgbin=''
+declare -A opt
 
 while test $# -gt 0; do
     case "$1" in
-        --pgbin)
-            shift
-            test $# -gt 0 || misuse
-            pgbin="$1"
-            shift
-            ;;
-        --puppet)
-            shift
-            test $# -gt 0 || misuse
-            pup_ref="$1"
-            shift
-            ;;
-        --server)
-            shift
-            test $# -gt 0 || misuse
-            pupsrv_ref="$1"
-            shift
+        --pgbin|--pgport)
+            test $# -gt 1 || misuse
+            opt["${1:2}"]="$2"
+            shift 2
             ;;
         --)
             shift
@@ -42,9 +27,21 @@ while test $# -gt 0; do
 done
 cmd=("$@")
 
-test "$pup_ref" || misuse
-test "$pupsrv_ref" || misuse
-test "$pgbin" || misuse
+if test -z "${opt[pgbin]:-}"; then
+    opt[pgbin]="$(ext/bin/test-config --get pgbin)"
+    if test  -z "${opt[pgbin]:-}"; then
+        echo 'Please specify --pgbin or set pgbin with ext/bin/test-config' 1>&2
+        exit 2
+    fi
+fi
+
+if test -z "${opt[pgport]:-}"; then
+    opt[pgport]="$(ext/bin/test-config --get pgport)"
+     if test  -z "${opt[pgport]:-}"; then
+        echo 'Please specify --pgport or set pgport with ext/bin/test-config' 1>&2
+        exit 2
+    fi
+fi
 
 set -x
 
@@ -57,7 +54,7 @@ ext/bin/require-leiningen default "$tmpdir/local"
 ext/bin/require-pgbox default "$tmpdir/local"
 export PATH="$tmpdir/local/bin:$PATH"
 ext/bin/host-info
-ext/bin/config-puppet-test-ref "$pup_ref"
-ext/bin/config-puppetserver-test-ref "$pupsrv_ref"
-ext/bin/with-pdbbox --box "$tmpdir/box" --pgbin "$pgbin" --pgport 34335 \
+ext/bin/with-pdbbox --box "$tmpdir/box" \
+                    --pgbin "${opt[pgbin]}" \
+                    --pgport "${opt[pgport]}" \
                     -- "${cmd[@]}"

ext/bin/config-puppet-test-ref

@@ -20,15 +20,15 @@ fi
 
 set -x
 
-rm -f ext/test-conf/puppet-requested
+rm -f ext/test-conf/puppet-ref-requested
 
 case "$#" in
     0)
-        puppet_ref="$(cat ext/test-conf/puppet-default)"
+        puppet_ref="$(cat ext/test-conf/puppet-ref-default)"
         ;;
     1)
         puppet_ref="$1"
-        echo "$puppet_ref" > ext/test-conf/puppet-requested
+        echo "$puppet_ref" > ext/test-conf/puppet-ref-requested
         ;;
     *)
         usage 1>&2

ext/bin/config-puppetserver-test-ref

@@ -26,15 +26,15 @@ fi
 
 set -x
 
-rm -f ext/test-conf/puppetserver-requested ext/test-conf/puppetserver-dep
+rm -f ext/test-conf/puppetserver-ref-requested ext/test-conf/puppetserver-dep
 
 case "$#" in
     0)
-        pupsrv_ref="$(cat ext/test-conf/puppetserver-default)"
+        pupsrv_ref="$(cat ext/test-conf/puppetserver-ref-default)"
         ;;
     1)
         pupsrv_ref="$1"
-        echo "$pupsrv_ref" > ext/test-conf/puppetserver-requested
+        echo "$pupsrv_ref" > ext/test-conf/puppetserver-ref-requested
         ;;
     *)
         usage 1>&2