travis: fixes, tidy, feedback and rebases

.travis.yml

@@ -7,31 +7,35 @@ python:
     - 3.7
 dist: xenial
 
-
 before_install:
   - export PATH="$PWD/.travis:$PATH"
 
 after_success:
   - now report_coverage
 
+after_failure:
+  - printenv PATH PYTHONPATH
+  - rose check-software
+
 jobs:
   include:
     - name: "Unit Tests"
       install:
-        - now install coverage cylc linters
+        - now install coverage cylc linters pytest rose
       script:
-        - now style_test
-        - now unit_test
+        - now test style
+        - now test units
 
     - name: "Test Battery"
-      install:
-        - now install coverage cylc fcm
+      before_install:
+        - export PATH="$PWD/.travis:$PATH"
+        - now install coverage cylc fcm rose tut_suite
       script:
-        - now test_battery
+        - now test battery
 
     - name: "Documentation"
       install:
-        - now install sphinx coverage
+        - now install coverage rose sphinx tut_suite
       script:
-        - docs_test
-        - docs_build
+        - now test docs
+        - now build docs

.travis/now

@@ -1,24 +1,81 @@
-#!/bin/bash
+#!/usr/bin/env bash
+#-------------------------------------------------------------------------------
+# Copyright (C) 2012-2019 British Crown (Met Office) & Contributors.
+#
+# This file is part of Rose, a framework for meteorological suites.
+#
+# Rose is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Rose is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Rose. If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+
+# Bash script for use with Travis-CI builds.
+#
+# usage: now command [args...]
+#
+# commands:
+#     build
+#     install
+#     report_coverage
+#     test
+
+# source the .bashrc because for some reason this doesn't get done for us
+# do this before the set -eu to avoid bailing on the build for hardcoded
+# bashrc issues
+if [[ -f "${HOME}/.bashrc" ]]; then
+    source "${HOME}/.bashrc"
+fi
+
+
+set -eu
 
 APT=()
 PIP=()
 NPM=()
+PY_PATH=()
+RC_PATH=("${HOME}")
 
-install_coverage () {
+_join () {
+    local IFS="$1";
+    shift;
+    echo "$*";
+}
+
+_build_docs () {
+    rose make-docs --strict clean html slides latexpdf
+}
+
+_install_coverage () {
     export COVERAGE_PROCESS_START="${TRAVIS_BUILD_DIR}/.coveragerc"
-    export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis"
     PIP+=(coverage pytest-cov)
+    PY_PATH+=("${TRAVIS_BUILD_DIR}/.travis")
+}
+
+_install_rose () {
+    PIP+=(aiofiles)
+    PY_PATH+=("${TRAVIS_BUILD_DIR}/lib/python/")
+    RC_PATH+=("${TRAVIS_BUILD_DIR}/bin")
 }
 
-install_cylc () {
+_install_cylc () {
     wget 'https://github.com/cylc/cylc-flow/archive/master.tar.gz' \
         -O - | tar -xz -C "${HOME}"
     pip install -e "${HOME}/cylc-flow-master"
     APT+=(at heirloom-mailx)
     PIP+=(colorama python-jose pyzmq Jinja2)
+    RC_PATH+=("${HOME}/cylc-master/bin")
 }
 
-install_fcm () {
+_install_fcm () {
     sudo sh -c 'echo "deb http://opensource.wandisco.com/ubuntu \
         `lsb_release -cs` svn19" >> /etc/apt/sources.list.d/subversion19.list'
     sudo wget -q http://opensource.wandisco.com/wandisco-debian.gpg -O- | \
@@ -28,40 +85,80 @@ install_fcm () {
     tar -xvf '/tmp/fcm-master.tar.gz' -C "${HOME}"
     APT+=(subversion build-essential gfortran libxml-parser-perl \
           libconfig-inifiles-perl libdbi-perl libdbd-sqlite3-perl)
+    RC_PATH+=("${HOME}/fcm-master/bin")
 }
 
-install_sphinx () {
+_install_sphinx () {
+    # sphinx documentation and its extensions
     APT+=(latexmk texlive texlive-generic-extra texlive-latex-extra \
-          texlive-fonts-recommended)
-    PIP+=(sphinx sphinx_rtd_theme sphinxcontrib-httpdomain hieroglyph pillow)
+          texlive-fonts-recommended graphviz)
+    PIP+=(sphinx sphinx_rtd_theme sphinxcontrib-httpdomain hieroglyph)
 }
 
-install_unittests () {
+_install_tut_suite () {
+    # cylc tutorial suite
+    PIP+=(pillow)
+}
+
+_install_pytest () {
+    # pytest and its extensions
     PIP+=(pytest)
 }
 
-install_linters () {
+_install_linters () {
     PIP+=(pycodestyle)
     NPM+=(eslint)
 }
 
+_test_units () {
+    pytest --cov-append lib/python/rose/tests/*
+}
+
+_test_style () {
+    pycodestyle
+    eslint .
+}
+
+_test_battery () {
+    cp "${TRAVIS_BUILD_DIR}/.travis/sitecustomize.py" ./lib/python
+    coverage run .travis/cover.py
+    rm ./lib/python/sitecustomize.py
+}
+
+_test_docs () {
+    rose make-docs --strict clean linkcheck doctest
+}
+
+build () {
+    for arg in "$@"; do
+        "_build_${arg}"
+    done
+}
+
 install () {
     for arg in "$@"; do
-        "install_${arg}"
+        "_install_${arg}"
     done
 
+    if [[ ${#PIP[@]} -gt 0 ]]; then
+        pip install "${PIP[@]}" &
+    fi
+
+    if [[ ${#NPM[@]} -gt 0 ]]; then
+        npm install -g "${NPM[@]}" &
+    fi
+
     if [[ ${#APT[@]} -gt 0 ]]; then
         sudo apt-get update
         sudo apt-get install -y "${APT[@]}"
     fi
 
-    if [[ ${#PIP[@]} -gt 0 ]]; then
-        pip install "${PIP[@]}"
-    fi
+    wait
 
-    if [[ ${#NPM[@]} > 0 ]]; then
-        npm install -g "${NPM[@]}"
-    fi
+    cat >"${HOME}/.bashrc" \
+        <<<"export PATH=\"$(_join ':' "${RC_PATH[@]}"):\$PATH\";"
+    cat >>"${HOME}/.bashrc" \
+        <<<"export PYTHONPATH=\"$(_join ':' "${PY_PATH[@]}"):\$PYTHONPATH\";"
 }
 
 report_coverage () {
@@ -70,28 +167,15 @@ report_coverage () {
     bash <(curl -s https://codecov.io/bash)
 }
 
-unit_test () {
-    PYTHONPATH=$PYTHONPATH:lib/python/ pytest lib/python/rose/tests/*
-    return
-}
-
-style_test () {
-    pycodestyle
-    eslint
-}
-
-test_battery () {
-    cp "${TRAVIS_BUILD_DIR}/.travis/sitecustomize.py" ./lib/python
-    coverage run .travis/cover.py
-    rm ./lib/python/sitecustomize.py
-}
-
-docs_test () {
-    rose make-docs --strict clean linkcheck doctest
+test () {
+    PS1='$' . "${HOME}/.bashrc"
+    for arg in "$@"; do
+        "_test_${arg}"
+    done
 }
 
-docs_build () {
-    rose make-docs --strict clean html slides latexpdf
-}
+# do this here so we only trace the commands we are interested in
+set -o xtrace
 
+# run the specified function
 "$@"

bin/rose-check-software

@@ -319,8 +319,6 @@ def docs(check=check):
     ret = check_all('Documentation Builder', [
         check('graphviz-dot <dot>', command_template='-V',
                version_template='graphviz version ([^\s]+)', outfile=2),
-        # TODO remove Sphinx version dependency
-        # https://github.com/nyergler/hieroglyph/issues/148
         check('py:sphinx', (1, 5, 3)),
         check('py:sphinx_rtd_theme', (0, 2, 4)),
         check('py:sphinxcontrib.httpdomain'),

sphinx/extract-pdf-documents.py

@@ -39,15 +39,7 @@ def main():
         sys.exit('usage: extract-pdf-documents build_dir')
     latex_dir = os.path.join(build_dir, 'latex')
     pdf_dir = os.path.join(build_dir, 'pdf')
-    try:
-        os.mkdir(pdf_dir)
-    except OSError as exc:
-        if exc.errno == errno.EEXIST:
-            # directory exists -> no further action required
-            pass
-        else:
-            # meaningful os error -> raise
-            raise
+    os.makedirs(pdf_dir, exist_ok=True)
 
     # the index html file
     html = (