added cylc install and functional test

stub tests for rose fileinstall

added cylc install and functional test

.github/workflows/test_fast.yml

@@ -25,7 +25,6 @@ jobs:
 
       - name: Install
         run: |
-          pip install git+https://github.com/metomi/rose@master
           pip install ."[all]"
 
       - name: Style

.github/workflows/test_functional.yml

@@ -48,7 +48,7 @@ jobs:
       - name: Install
         run: |
           pip install git+https://github.com/metomi/rose@master
-          pip install --no-deps git+https://github.com/wxtim/cylc-rose.git@c49a2cbbf7f86de924a5591d81b8089aa3b1f001
+          pip install --no-deps git+https://github.com/cylc/cylc-rose.git@master
           pip install ."[all]"
           mkdir "$HOME/cylc-run"
 

CHANGES.md

@@ -123,8 +123,9 @@ hierarchy and ability to set site config directory.
 config option `[scheduling]stop after cycle point`.
 
 [#3913](https://github.com/cylc/cylc-flow/pull/3913) - Added the ability to
-parse `[jinja2:suite.rc]` section from a `rose-suite.conf` file in the same
-folder as the `flow.cylc` file.
+use plugins to parse suite templating variables and additional files to
+install. Only one such plugin exists at the time of writing, designed to
+parse ``rose-suite.conf`` files in repository "cylc-rose".
 
 ### Fixes
 

cylc/flow/config.py

@@ -222,8 +222,6 @@ def __init__(
         self.mem_log("config.py: before RawSuiteConfig init")
         if output_fname:
             output_fname = os.path.expandvars(output_fname)
-        # Copy template_vars to prevent addition of non serializable template
-        # vars from file being added by parsec.
         self.pcfg = RawSuiteConfig(
             fpath,
             output_fname,

cylc/flow/parsec/fileparse.py

@@ -31,8 +31,8 @@
 """
 
 import os
-import sys
 import re
+import sys
 
 import pkg_resources
 from ast import literal_eval
@@ -240,7 +240,7 @@ def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
         'jinja2:suite.rc': None
     }
     for entry_point in pkg_resources.iter_entry_points(
-        'cylc.configure'
+        'cylc.pre_configure'
     ):
         rose_vars = entry_point.resolve()(Path(fpath).parent)
 

cylc/flow/parsec/jinja2support.py

@@ -20,9 +20,9 @@
 
 import importlib
 import os
-import sys
 import pkgutil
 import re
+import sys
 import traceback
 from glob import glob
 
@@ -182,7 +182,6 @@ def jinja2environment(dir_=None):
     env.globals['environ'] = os.environ
     env.globals['raise'] = raise_helper
     env.globals['assert'] = assert_helper
-
     return env
 
 

cylc/flow/scripts/install.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""cylc install [OPTIONS] ARGS
+
+Test communication with a running suite.
+
+If suite REG is running or TASK in suite REG is currently running,
+exit with success status, else exit with error status."""
+
+import os
+import pkg_resources
+
+from cylc.flow.option_parsers import CylcOptionParser as COP
+from cylc.flow.terminal import cli_function
+from cylc.flow.pathutil import get_suite_run_dir
+
+
+def get_option_parser():
+    parser = COP(
+        __doc__, comms=True,
+        argdoc=[('REG', 'Suite name')])
+
+    return parser
+
+
+@cli_function(get_option_parser)
+def main(parser, options, suite):
+    for entry_point in pkg_resources.iter_entry_points(
+        'cylc.post_install'
+    ):
+        entry_point.resolve()(
+            dir_=os.getcwd(),
+            opts=options,
+            dest_root=get_suite_run_dir(suite)
+        )
+
+
+if __name__ == "__main__":
+    main()

tests/functional/rose-conf/00-jinja2/rose-suite.conf

@@ -5,4 +5,4 @@ TASK1="respond"
 TASK2="plunge"
 TASK3="allow"
 MEMBERS=["control", "yan", "tan", "tethera"]
-SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3}
+SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3}

tests/functional/rose-conf/01-empy/rose-suite.conf

@@ -5,4 +5,4 @@ TASK1="respond"
 TASK2="plunge"
 TASK3="allow"
 MEMBERS=["control", "yan", "tan", "tethera"]
-SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3}
+SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3}

tests/functional/rose-conf/02-env/rose-suite.conf

@@ -3,4 +3,4 @@ EG_STRING = Sphinx of black quartz, judge my vow!
 EG_INTEGER = 42
 EG_FLOAT = 99.99
 [jinja2:suite.rc]
-jinja2_var = 12
+jinja2_var = 12

tests/functional/rose-conf/03-fileinstall.t

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test jinja2 from rose-suite.conf file is processed into a suite.
+. "$(dirname "$0")/test_header"
+#-------------------------------------------------------------------------------
+python -c "import cylc.rose" > /dev/null 2>&1 ||
+  skip_all "cylc.rose not installed in environment."
+
+set_test_number 3
+install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+
+sed -i "s@REPLACE_THIS@${CYLC_REPO_DIR}/tests/functional/rose-conf/fileinstall_data@g" rose-suite.conf
+
+
+run_ok "${TEST_NAME_BASE}-validate" cylc install "${SUITE_NAME}"
+
+# Test that data files have been concatenated.
+DATA_INSTALLED_PATH="${SUITE_RUN_DIR}/data"
+DATA_ORIGIN_PATH="${CYLC_REPO_DIR}/tests/functional/rose-conf/fileinstall_data/"
+if [[ $(cat "${DATA_ORIGIN_PATH}randoms1.data"; cat "${DATA_ORIGIN_PATH}randoms3.data") == $(cat "${DATA_INSTALLED_PATH}") ]]; then
+  ok "${TEST_NAME_BASE}.File installed from wildcards name."
+else
+  fail "${TEST_NAME_BASE}.File not installed from wildcards name."
+fi
+
+# Test that lion.py has been installed in a sub directory.
+LION_INSTALLED_PATH="${SUITE_RUN_DIR}/lib/python/lion.py"
+LION_ORIGIN_PATH="${CYLC_REPO_DIR}/tests/functional/rose-conf/fileinstall_data/lion.py"
+if [[ $(cat "${LION_INSTALLED_PATH}") == $(cat "${LION_ORIGIN_PATH}") ]]; then
+  ok "${TEST_NAME_BASE}.File installed to subfolder"
+else
+  fail "${TEST_NAME_BASE}.File not installed to subfolder"
+fi
+
+purge
+exit