Rose conf.read jinja2

.github/workflows/test_functional.yml

@@ -47,7 +47,9 @@ jobs:
 
       - name: Install
         run: |
+          pip install git+https://github.com/metomi/rose@master
           pip install ."[all]"
+          pip install --no-deps git+https://github.com/cylc/cylc-rose.git@master
           mkdir "$HOME/cylc-run"
 
       - name: Configure Atrun

CHANGES.md

@@ -141,6 +141,11 @@ config option `[scheduling]stop after cycle point`.
 [#3961](https://github.com/cylc/cylc-flow/pull/3961) - Added a new command:
 `cylc clean`.
 
+[#3913](https://github.com/cylc/cylc-flow/pull/3913) - Added the ability to
+use plugins to parse suite templating variables and additional files to
+install. Only one such plugin exists at the time of writing, designed to
+parse ``rose-suite.conf`` files in repository "cylc-rose".
+
 [#3955](https://github.com/cylc/cylc-flow/pull/3955) - Global config options
 to control the job submission environment.
 

cylc/flow/parsec/fileparse.py

@@ -31,9 +31,13 @@
 """
 
 import os
-import sys
 import re
+import sys
+
+import pkg_resources
+from pathlib import Path
 
+from cylc.flow import __version__
 from cylc.flow import LOG
 from cylc.flow.parsec.exceptions import ParsecError, FileParseError
 from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
@@ -209,6 +213,55 @@ def multiline(flines, value, index, maxline):
     return quot + newvalue + line, index
 
 
+def process_plugins(fpath):
+    # Load Rose Vars, if a ``rose-suite.conf`` file is present.
+    extra_vars = {
+        'env': {},
+        'template_variables': {},
+        'templating_detected': None
+    }
+    for entry_point in pkg_resources.iter_entry_points(
+        'cylc.pre_configure'
+    ):
+        plugin_result = entry_point.resolve()(fpath)
+        for section in ['env', 'template_variables']:
+            if section in plugin_result and plugin_result[section] is not None:
+                # Raise error if multiple plugins try to update the same keys.
+                section_update = plugin_result.get(section, {})
+                keys_collision = (
+                    extra_vars[section].keys() & section_update.keys()
+                )
+                if keys_collision:
+                    raise ParsecError(
+                        f"{entry_point.name} is trying to alter "
+                        f"[{section}]{', '.join(sorted(keys_collision))}."
+                    )
+                extra_vars[section].update(section_update)
+
+        if (
+            'templating_detected' in plugin_result and
+            plugin_result['templating_detected'] is not None and
+            extra_vars['templating_detected'] is not None and
+            extra_vars['templating_detected'] !=
+                plugin_result['templating_detected']
+        ):
+            # Don't allow subsequent plugins with different templating_detected
+            raise ParsecError(
+                "Can't merge templating languages "
+                f"{extra_vars['templating_detected']} and "
+                f"{plugin_result['templating_detected']}"
+            )
+        elif(
+            'templating_detected' in plugin_result and
+            plugin_result['templating_detected'] is not None
+        ):
+            extra_vars['templating_detected'] = plugin_result[
+                'templating_detected'
+            ]
+
+    return extra_vars
+
+
 def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
     """
     Read a cylc parsec config file (at fpath), inline any include files,
@@ -233,6 +286,12 @@ def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
     do_empy = True
     do_jinja2 = True
     do_contin = True
+
+    extra_vars = process_plugins(Path(fpath).parent)
+
+    if not template_vars:
+        template_vars = {}
+
     if viewcfg:
         if not viewcfg['empy']:
             do_empy = False
@@ -248,27 +307,72 @@ def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
         flines = inline(
             flines, fdir, fpath, False, viewcfg=viewcfg, for_edit=asedit)
 
+    template_vars['CYLC_VERSION'] = __version__
+
+    # Push template_vars into extra_vars so that duplicates come from
+    # template_vars.
+    if extra_vars['templating_detected'] is not None:
+        will_be_overwritten = (
+            template_vars.keys() &
+            extra_vars['template_variables'].keys()
+        )
+        for key in will_be_overwritten:
+            LOG.warning(
+                f'Overriding {key}: {extra_vars["template_variables"][key]} ->'
+                f' {template_vars[key]}'
+            )
+        extra_vars['template_variables'].update(template_vars)
+        template_vars = extra_vars['template_variables']
+
     # process with EmPy
     if do_empy:
+        if (
+            extra_vars['templating_detected'] == 'empy' and
+            not re.match(r'^#![Ee]m[Pp]y\s*', flines[0])
+        ):
+            if not re.match(r'^#!', flines[0]):
+                flines.insert(0, '#!empy')
+            else:
+                raise FileParseError(
+                    "Plugins set templating engine = "
+                    f"{extra_vars['templating_detected']}"
+                    f" which does not match {flines[0]} set in flow.cylc."
+                )
         if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
             LOG.debug('Processing with EmPy')
             try:
                 from cylc.flow.parsec.empysupport import empyprocess
             except (ImportError, ModuleNotFoundError):
                 raise ParsecError('EmPy Python package must be installed '
                                   'to process file: ' + fpath)
-            flines = empyprocess(flines, fdir, template_vars)
+            flines = empyprocess(
+                flines, fdir, template_vars
+            )
 
     # process with Jinja2
     if do_jinja2:
+        if (
+            extra_vars['templating_detected'] == 'jinja2' and
+            not re.match(r'^#![jJ]inja2\s*', flines[0])
+        ):
+            if not re.match(r'^#!', flines[0]):
+                flines.insert(0, '#!jinja2')
+            else:
+                raise FileParseError(
+                    "Plugins set templating engine = "
+                    f"{extra_vars['templating_detected']}"
+                    f" which does not match {flines[0]} set in flow.cylc."
+                )
         if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
             LOG.debug('Processing with Jinja2')
             try:
                 from cylc.flow.parsec.jinja2support import jinja2process
             except (ImportError, ModuleNotFoundError):
                 raise ParsecError('Jinja2 Python package must be installed '
                                   'to process file: ' + fpath)
-            flines = jinja2process(flines, fdir, template_vars)
+            flines = jinja2process(
+                flines, fdir, template_vars
+            )
 
     # concatenate continuation lines
     if do_contin:

cylc/flow/parsec/jinja2support.py

@@ -93,7 +93,7 @@ def raise_helper(message, error_type='Error'):
 def assert_helper(logical, message):
     """Provides a Jinja2 function for asserting logical expressions."""
     if not logical:
-        raise_helper(message, 'Assertation Error')
+        raise_helper(message, 'Assertion Error')
     return ''  # Prevent None return value polluting output.
 
 
@@ -182,6 +182,7 @@ def jinja2environment(dir_=None):
     env.globals['environ'] = os.environ
     env.globals['raise'] = raise_helper
     env.globals['assert'] = assert_helper
+
     return env
 
 

cylc/flow/scripts/install.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""cylc install [OPTIONS] ARGS
+
+Test communication with a running suite.
+
+If suite REG is running or TASK in suite REG is currently running,
+exit with success status, else exit with error status."""
+
+import os
+import pkg_resources
+from pathlib import Path
+
+from cylc.flow.option_parsers import CylcOptionParser as COP
+from cylc.flow.terminal import cli_function
+from cylc.flow.pathutil import get_suite_run_dir
+from cylc.flow.suite_files import parse_suite_arg
+
+
+def get_option_parser():
+    parser = COP(
+        __doc__, comms=True, prep=True,
+        argdoc=[('REG', 'Suite name')])
+
+    return parser
+
+
+@cli_function(get_option_parser)
+def main(parser, options, reg):
+    suite, flow_file = parse_suite_arg(options, reg)
+
+    for entry_point in pkg_resources.iter_entry_points(
+        'cylc.pre_configure'
+    ):
+        entry_point.resolve()(Path(flow_file).parent)
+
+    for entry_point in pkg_resources.iter_entry_points(
+        'cylc.post_install'
+    ):
+        entry_point.resolve()(
+            dir_=os.getcwd(),
+            opts=options,
+            dest_root=get_suite_run_dir(suite)
+        )
+
+
+if __name__ == "__main__":
+    main()

setup.cfg

@@ -86,6 +86,7 @@ cylc.command =
     get-suite-version = cylc.flow.scripts.get_suite_version:main
     graph = cylc.flow.scripts.graph:main
     hold = cylc.flow.scripts.hold:main
+    install = cylc.flow.scripts.install:main
     jobs-kill = cylc.flow.scripts.jobs_kill:main
     jobs-poll = cylc.flow.scripts.jobs_poll:main
     jobs-submit = cylc.flow.scripts.jobs_submit:main

tests/functional/jinja2/10-builtin-functions.t

@@ -30,7 +30,7 @@ TEST_NAME="${TEST_NAME_BASE}"-fail-assert
 run_fail "${TEST_NAME}" cylc validate "${SUITE_NAME}" \
     -s 'FOO="True"' \
     -s 'ANSWER="43"'
-grep_ok 'Jinja2 Assertation Error: Universal' "${TEST_NAME}.stderr"
+grep_ok 'Jinja2 Assertion Error: Universal' "${TEST_NAME}.stderr"
 TEST_NAME="${TEST_NAME_BASE}"-fail-raise
 run_fail "${TEST_NAME}" cylc validate "${SUITE_NAME}" -s 'ANSWER="42"'
 grep_ok 'Jinja2 Error: FOO must be defined' "${TEST_NAME}.stderr"

tests/functional/rose-conf/00-jinja2.t

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test jinja2 from rose-suite.conf file is processed into a suite.
+. "$(dirname "$0")/test_header"
+#-------------------------------------------------------------------------------
+python -c "import cylc.rose" > /dev/null 2>&1 ||
+  skip_all "cylc.rose not installed in environment."
+
+set_test_number 2
+
+install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+
+run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
+
+cylc view -p --stdout "${SUITE_NAME}" > processed.conf.test
+
+cmp_ok processed.conf.test processed.conf.control
+
+purge
+exit

tests/functional/rose-conf/00-jinja2/flow.cylc

@@ -0,0 +1,36 @@
+#!jinja2
+[meta]
+    title = "Add jinja2 vars from a rose-suite.conf"
+    description = """
+    Natively, in Cylc!
+    """
+
+[scheduling]
+    initial cycle point = {{ICP}}
+    final cycle point = {{FCP}}
+    cycling mode = integer
+    [[graph]]
+{% for member in MEMBERS %}
+        P1 = {{TASK1}} => {{TASK2}}_{{member}} => {{TASK3}}
+{% endfor %}
+{% for key, value in SAMUELJOHNSON.items() %}
+        P1 = {{TASK3}} => {{value}}_auf_deutsch_ist_{{key}} => fin
+{% endfor %}
+
+[runtime]
+    [[root]]
+        script = echo "This task is ${CYLC_TASK_ID}"
+
+    [[{{ TASK1 }}]]
+
+{% for member in MEMBERS %}
+    [[{{ TASK2 }}_{{member}}]]
+{% endfor %}
+
+    [[{{ TASK3 }}]]
+
+{% for key, value in SAMUELJOHNSON.items() %}
+    [[{{value}}_auf_deutsch_ist_{{key}}]]
+{% endfor %}
+
+    [[fin]]

tests/functional/rose-conf/00-jinja2/processed.conf.control

@@ -0,0 +1,30 @@
+[meta]
+    title = "Add jinja2 vars from a rose-suite.conf"
+    description = """
+    Natively, in Cylc!
+    """
+[scheduling]
+    initial cycle point = 1
+    final cycle point = 1
+    cycling mode = integer
+    [[graph]]
+        P1 = respond => plunge_control => allow
+        P1 = respond => plunge_yan => allow
+        P1 = respond => plunge_tan => allow
+        P1 = respond => plunge_tethera => allow
+        P1 = allow => 1_auf_deutsch_ist_ein => fin
+        P1 = allow => 2_auf_deutsch_ist_zwei => fin
+        P1 = allow => 3_auf_deutsch_ist_drei => fin
+[runtime]
+    [[root]]
+        script = echo "This task is ${CYLC_TASK_ID}"
+    [[respond]]
+    [[plunge_control]]
+    [[plunge_yan]]
+    [[plunge_tan]]
+    [[plunge_tethera]]
+    [[allow]]
+    [[1_auf_deutsch_ist_ein]]
+    [[2_auf_deutsch_ist_zwei]]
+    [[3_auf_deutsch_ist_drei]]
+    [[fin]]

tests/functional/rose-conf/00-jinja2/rose-suite.conf

@@ -0,0 +1,8 @@
+[jinja2:suite.rc]
+ICP=1
+FCP=1
+TASK1="respond"
+TASK2="plunge"
+TASK3="allow"
+MEMBERS=["control", "yan", "tan", "tethera"]
+SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3}