From 36a20a0eab69eafadb0a31e1a8615fd627efa993 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Fri, 13 Nov 2020 11:34:38 +0000 Subject: [PATCH] added functional tests to check dealing with optional configs set by ENV or rose-suite.conf:opts= tests for rose fileinstall added skips for tests of rose.cylc if rose-cylc not installed --- .github/workflows/test_functional.yml | 1 + cylc/flow/parsec/fileparse.py | 12 +++- cylc/flow/parsec/rose_utils.py | 1 + setup.cfg | 1 + tests/functional/rose-conf/00-jinja2.t | 4 ++ tests/functional/rose-conf/01-empy.t | 3 + tests/functional/rose-conf/02-env.t | 3 + .../rose-conf/04-opts-set-from-env.t | 34 +++++++++ .../rose-conf/04-opts-set-from-env/flow.cylc | 36 ++++++++++ .../opt/rose-suite-Cymraeg.conf | 8 +++ .../opt/rose-suite-Gaelige.conf | 8 +++ .../processed.conf.control | 30 ++++++++ .../04-opts-set-from-env/rose-suite.conf | 8 +++ .../05-opts-set-from-rose-suite.conf.t | 34 +++++++++ .../flow.cylc | 36 ++++++++++ .../opt/rose-suite-Cymraeg.conf | 8 +++ .../opt/rose-suite-Gaelige.conf | 8 +++ .../processed.conf.control | 30 ++++++++ .../rose-suite.conf | 9 +++ tests/unit/parsec/test_rose_utils.py | 70 ++++++++++++++++++- 20 files changed, 341 insertions(+), 3 deletions(-) create mode 100755 tests/functional/rose-conf/04-opts-set-from-env.t create mode 100644 tests/functional/rose-conf/04-opts-set-from-env/flow.cylc create mode 100644 tests/functional/rose-conf/04-opts-set-from-env/opt/rose-suite-Cymraeg.conf create mode 100644 tests/functional/rose-conf/04-opts-set-from-env/opt/rose-suite-Gaelige.conf create mode 100644 tests/functional/rose-conf/04-opts-set-from-env/processed.conf.control create mode 100644 tests/functional/rose-conf/04-opts-set-from-env/rose-suite.conf create mode 100755 tests/functional/rose-conf/05-opts-set-from-rose-suite.conf.t create mode 100644 tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/flow.cylc create mode 100644 tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/opt/rose-suite-Cymraeg.conf create mode 100644 tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/opt/rose-suite-Gaelige.conf create mode 100644 tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/processed.conf.control create mode 100644 tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/rose-suite.conf diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml index 0f01ebc625e..c9429f5568a 100644 --- a/.github/workflows/test_functional.yml +++ b/.github/workflows/test_functional.yml @@ -48,6 +48,7 @@ jobs: - name: Install run: | pip install git+https://github.com/metomi/rose@master + pip install --no-deps git+https://github.com/wxtim/cylc-rose.git@c49a2cbbf7f86de924a5591d81b8089aa3b1f001 pip install ."[all]" mkdir "$HOME/cylc-run" diff --git a/cylc/flow/parsec/fileparse.py b/cylc/flow/parsec/fileparse.py index 3338e127000..87670677990 100644 --- a/cylc/flow/parsec/fileparse.py +++ b/cylc/flow/parsec/fileparse.py @@ -34,13 +34,13 @@ import sys import re +import pkg_resources from ast import literal_eval from copy import copy from pathlib import Path from cylc.flow import LOG from cylc.flow.parsec.exceptions import ParsecError, FileParseError -from cylc.flow.parsec.rose_utils import get_rose_vars from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults from cylc.flow.parsec.include import inline from cylc.flow.parsec.util import itemstr @@ -234,7 +234,15 @@ def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False): template_vars = {} # Load Rose Vars, if a ``rose-suite.conf`` file is present. - rose_vars = get_rose_vars(Path(fpath).parent) + rose_vars = { + 'env': None, + 'empy:suite.rc': None, + 'jinja2:suite.rc': None + } + for entry_point in pkg_resources.iter_entry_points( + 'cylc.configure' + ): + rose_vars = entry_point.resolve()(Path(fpath).parent) if viewcfg: if not viewcfg['empy']: diff --git a/cylc/flow/parsec/rose_utils.py b/cylc/flow/parsec/rose_utils.py index 2c086610b21..fe62d7be284 100644 --- a/cylc/flow/parsec/rose_utils.py +++ b/cylc/flow/parsec/rose_utils.py @@ -87,6 +87,7 @@ def rose_fileinstall(dir_=None, opts=None, dest_root=None): dir_(string or pathlib.Path): Search for a ``rose-suite.conf`` file in this location. dest_root (string or pathlib.Path) + """ if not rose_config_exists(dir_): return False diff --git a/setup.cfg b/setup.cfg index e3743500c23..5f973bbcd80 100644 --- a/setup.cfg +++ b/setup.cfg @@ -85,6 +85,7 @@ cylc.command = get-suite-version = cylc.flow.scripts.get_suite_version:main graph = cylc.flow.scripts.graph:main hold = cylc.flow.scripts.hold:main + install = cylc.flow.scripts.install:main jobs-kill = cylc.flow.scripts.jobs_kill:main jobs-poll = cylc.flow.scripts.jobs_poll:main jobs-submit = cylc.flow.scripts.jobs_submit:main diff --git a/tests/functional/rose-conf/00-jinja2.t b/tests/functional/rose-conf/00-jinja2.t index c3b870716f0..a41bda93e83 100755 --- a/tests/functional/rose-conf/00-jinja2.t +++ b/tests/functional/rose-conf/00-jinja2.t @@ -18,7 +18,11 @@ # Test jinja2 from rose-suite.conf file is processed into a suite. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- +python -c "import cylc.rose" > /dev/null 2>&1 || + skip_all "cylc.rose not installed in environment." + set_test_number 1 + install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" cylc view -p --stdout "${SUITE_NAME}" > processed.conf.test diff --git a/tests/functional/rose-conf/01-empy.t b/tests/functional/rose-conf/01-empy.t index fc6f42a1df0..251d7247027 100755 --- a/tests/functional/rose-conf/01-empy.t +++ b/tests/functional/rose-conf/01-empy.t @@ -18,6 +18,9 @@ # Test jinja2 from rose-suite.conf file is processed into a suite. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- +python -c "import cylc.rose" > /dev/null 2>&1 || + skip_all "cylc.rose not installed in environment." + set_test_number 1 install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" diff --git a/tests/functional/rose-conf/02-env.t b/tests/functional/rose-conf/02-env.t index c3b870716f0..66cf44f5813 100755 --- a/tests/functional/rose-conf/02-env.t +++ b/tests/functional/rose-conf/02-env.t @@ -18,6 +18,9 @@ # Test jinja2 from rose-suite.conf file is processed into a suite. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- +python -c "import cylc.rose" > /dev/null 2>&1 || + skip_all "cylc.rose not installed in environment." + set_test_number 1 install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" diff --git a/tests/functional/rose-conf/04-opts-set-from-env.t b/tests/functional/rose-conf/04-opts-set-from-env.t new file mode 100755 index 00000000000..a14fc58af42 --- /dev/null +++ b/tests/functional/rose-conf/04-opts-set-from-env.t @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test jinja2 from rose-suite.conf file is processed into a suite. +. "$(dirname "$0")/test_header" +#------------------------------------------------------------------------------- +python -c "import cylc.rose" > /dev/null 2>&1 || + skip_all "cylc.rose not installed in environment." + +set_test_number 1 +install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" + +export ROSE_SUITE_OPT_CONF_KEYS=Gaelige + +cylc view -p --stdout "${SUITE_NAME}" > processed.conf.test + +cmp_ok processed.conf.test processed.conf.control + +purge +exit diff --git a/tests/functional/rose-conf/04-opts-set-from-env/flow.cylc b/tests/functional/rose-conf/04-opts-set-from-env/flow.cylc new file mode 100644 index 00000000000..714c04070ed --- /dev/null +++ b/tests/functional/rose-conf/04-opts-set-from-env/flow.cylc @@ -0,0 +1,36 @@ +#!jinja2 +[meta] + title = "Add jinja2 vars from a rose-suite.conf" + description = """ + Natively, in Cylc! + """ + +[scheduling] + initial cycle point = {{ICP}} + final cycle point = {{FCP}} + cycling mode = integer + [[graph]] +{% for member in MEMBERS %} + P1 = {{TASK1}} => {{TASK2}}_{{member}} => {{TASK3}} +{% endfor %} +{% for key, value in SAMUELJOHNSON.items() %} + P1 = {{TASK3}} => {{value}}_auf_deutsch_ist_{{key}} => fin +{% endfor %} + +[runtime] + [[root]] + script = echo "This task is ${CYLC_TASK_ID}" + + [[{{ TASK1 }}]] + +{% for member in MEMBERS %} + [[{{ TASK2 }}_{{member}}]] +{% endfor %} + + [[{{ TASK3 }}]] + +{% for key, value in SAMUELJOHNSON.items() %} + [[{{value}}_auf_deutsch_ist_{{key}}]] +{% endfor %} + + [[fin]] diff --git a/tests/functional/rose-conf/04-opts-set-from-env/opt/rose-suite-Cymraeg.conf b/tests/functional/rose-conf/04-opts-set-from-env/opt/rose-suite-Cymraeg.conf new file mode 100644 index 00000000000..b5a34962370 --- /dev/null +++ b/tests/functional/rose-conf/04-opts-set-from-env/opt/rose-suite-Cymraeg.conf @@ -0,0 +1,8 @@ +[jinja2:suite.rc] +ICP=1 +FCP=1 +TASK1="mynd" +TASK2="bwyta" +TASK3="prynu" +MEMBERS=["control", "un", "dau", "tri"] +SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3} \ No newline at end of file diff --git a/tests/functional/rose-conf/04-opts-set-from-env/opt/rose-suite-Gaelige.conf b/tests/functional/rose-conf/04-opts-set-from-env/opt/rose-suite-Gaelige.conf new file mode 100644 index 00000000000..3076e16af21 --- /dev/null +++ b/tests/functional/rose-conf/04-opts-set-from-env/opt/rose-suite-Gaelige.conf @@ -0,0 +1,8 @@ +[jinja2:suite.rc] +ICP=1 +FCP=1 +TASK1="cuir" # to put +TASK2="tog" # to lift +TASK3="gabh" # to take +MEMBERS=["control", "aon", "dhà", "trì"] +SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3} \ No newline at end of file diff --git a/tests/functional/rose-conf/04-opts-set-from-env/processed.conf.control b/tests/functional/rose-conf/04-opts-set-from-env/processed.conf.control new file mode 100644 index 00000000000..995afacce4a --- /dev/null +++ b/tests/functional/rose-conf/04-opts-set-from-env/processed.conf.control @@ -0,0 +1,30 @@ +[meta] + title = "Add jinja2 vars from a rose-suite.conf" + description = """ + Natively, in Cylc! + """ +[scheduling] + initial cycle point = 1 + final cycle point = 1 + cycling mode = integer + [[graph]] + P1 = cuir => tog_control => gabh + P1 = cuir => tog_aon => gabh + P1 = cuir => tog_dhà => gabh + P1 = cuir => tog_trì => gabh + P1 = gabh => 1_auf_deutsch_ist_ein => fin + P1 = gabh => 2_auf_deutsch_ist_zwei => fin + P1 = gabh => 3_auf_deutsch_ist_drei => fin +[runtime] + [[root]] + script = echo "This task is ${CYLC_TASK_ID}" + [[cuir]] + [[tog_control]] + [[tog_aon]] + [[tog_dhà]] + [[tog_trì]] + [[gabh]] + [[1_auf_deutsch_ist_ein]] + [[2_auf_deutsch_ist_zwei]] + [[3_auf_deutsch_ist_drei]] + [[fin]] diff --git a/tests/functional/rose-conf/04-opts-set-from-env/rose-suite.conf b/tests/functional/rose-conf/04-opts-set-from-env/rose-suite.conf new file mode 100644 index 00000000000..16bcd0d2188 --- /dev/null +++ b/tests/functional/rose-conf/04-opts-set-from-env/rose-suite.conf @@ -0,0 +1,8 @@ +[jinja2:suite.rc] +ICP=1 +FCP=1 +TASK1="respond" +TASK2="plunge" +TASK3="allow" +MEMBERS=["control", "yan", "tan", "tethera"] +SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3} \ No newline at end of file diff --git a/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf.t b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf.t new file mode 100755 index 00000000000..5a19ebbff4e --- /dev/null +++ b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf.t @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test jinja2 from rose-suite.conf file is processed into a suite. +. "$(dirname "$0")/test_header" +#------------------------------------------------------------------------------- +python -c "import cylc.rose" > /dev/null 2>&1 || + skip_all "cylc.rose not installed in environment." + +set_test_number 1 +install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" + +export ROSE_SUITE_OPT_CONF_KEYS="" + +cylc view -p --stdout "${SUITE_NAME}" > processed.conf.test + +cmp_ok processed.conf.test processed.conf.control + +purge +exit diff --git a/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/flow.cylc b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/flow.cylc new file mode 100644 index 00000000000..714c04070ed --- /dev/null +++ b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/flow.cylc @@ -0,0 +1,36 @@ +#!jinja2 +[meta] + title = "Add jinja2 vars from a rose-suite.conf" + description = """ + Natively, in Cylc! + """ + +[scheduling] + initial cycle point = {{ICP}} + final cycle point = {{FCP}} + cycling mode = integer + [[graph]] +{% for member in MEMBERS %} + P1 = {{TASK1}} => {{TASK2}}_{{member}} => {{TASK3}} +{% endfor %} +{% for key, value in SAMUELJOHNSON.items() %} + P1 = {{TASK3}} => {{value}}_auf_deutsch_ist_{{key}} => fin +{% endfor %} + +[runtime] + [[root]] + script = echo "This task is ${CYLC_TASK_ID}" + + [[{{ TASK1 }}]] + +{% for member in MEMBERS %} + [[{{ TASK2 }}_{{member}}]] +{% endfor %} + + [[{{ TASK3 }}]] + +{% for key, value in SAMUELJOHNSON.items() %} + [[{{value}}_auf_deutsch_ist_{{key}}]] +{% endfor %} + + [[fin]] diff --git a/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/opt/rose-suite-Cymraeg.conf b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/opt/rose-suite-Cymraeg.conf new file mode 100644 index 00000000000..b5a34962370 --- /dev/null +++ b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/opt/rose-suite-Cymraeg.conf @@ -0,0 +1,8 @@ +[jinja2:suite.rc] +ICP=1 +FCP=1 +TASK1="mynd" +TASK2="bwyta" +TASK3="prynu" +MEMBERS=["control", "un", "dau", "tri"] +SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3} \ No newline at end of file diff --git a/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/opt/rose-suite-Gaelige.conf b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/opt/rose-suite-Gaelige.conf new file mode 100644 index 00000000000..3076e16af21 --- /dev/null +++ b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/opt/rose-suite-Gaelige.conf @@ -0,0 +1,8 @@ +[jinja2:suite.rc] +ICP=1 +FCP=1 +TASK1="cuir" # to put +TASK2="tog" # to lift +TASK3="gabh" # to take +MEMBERS=["control", "aon", "dhà", "trì"] +SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3} \ No newline at end of file diff --git a/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/processed.conf.control b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/processed.conf.control new file mode 100644 index 00000000000..1daf7eacfe9 --- /dev/null +++ b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/processed.conf.control @@ -0,0 +1,30 @@ +[meta] + title = "Add jinja2 vars from a rose-suite.conf" + description = """ + Natively, in Cylc! + """ +[scheduling] + initial cycle point = 1 + final cycle point = 1 + cycling mode = integer + [[graph]] + P1 = mynd => bwyta_control => prynu + P1 = mynd => bwyta_un => prynu + P1 = mynd => bwyta_dau => prynu + P1 = mynd => bwyta_tri => prynu + P1 = prynu => 1_auf_deutsch_ist_ein => fin + P1 = prynu => 2_auf_deutsch_ist_zwei => fin + P1 = prynu => 3_auf_deutsch_ist_drei => fin +[runtime] + [[root]] + script = echo "This task is ${CYLC_TASK_ID}" + [[mynd]] + [[bwyta_control]] + [[bwyta_un]] + [[bwyta_dau]] + [[bwyta_tri]] + [[prynu]] + [[1_auf_deutsch_ist_ein]] + [[2_auf_deutsch_ist_zwei]] + [[3_auf_deutsch_ist_drei]] + [[fin]] diff --git a/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/rose-suite.conf b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/rose-suite.conf new file mode 100644 index 00000000000..fa5aaeac357 --- /dev/null +++ b/tests/functional/rose-conf/05-opts-set-from-rose-suite.conf/rose-suite.conf @@ -0,0 +1,9 @@ +opts=Cymraeg +[jinja2:suite.rc] +ICP=1 +FCP=1 +TASK1="respond" +TASK2="plunge" +TASK3="allow" +MEMBERS=["control", "yan", "tan", "tethera"] +SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3} \ No newline at end of file diff --git a/tests/unit/parsec/test_rose_utils.py b/tests/unit/parsec/test_rose_utils.py index 813ee87605d..2889e01646a 100644 --- a/tests/unit/parsec/test_rose_utils.py +++ b/tests/unit/parsec/test_rose_utils.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import hashlib import os import pytest @@ -22,7 +23,8 @@ from cylc.flow.parsec.rose_utils import ( rose_config_exists, get_rose_vars, - rose_config_tree_loader + rose_config_tree_loader, + rose_fileinstall ) @@ -213,3 +215,69 @@ def test_rose_config_tree_loader( 'JINJA2_VAR': f'{exp_JINJA2_VAR}' } assert results == expected + + +@pytest.fixture +def rose_fileinstall_config_template(tmp_path, scope='module'): + def wrapped_function(section): + """Fixture which returns a tmp_path containing a rose config tree. + + uses ``wrapped_function`` to allow passing either "empy" or "jinja2" + section types. + + Creates: + . + `--tmp_path + |-- rose-suite.conf + `-- opt + |-- rose-suite-gravy.conf + `-- rose-suite-chips.conf + """ + with open(tmp_path / 'rose-suite.conf', 'w+') as testfh: + # The [env] section is there to make sure I don't load it with + # the jinja2 method. + testfh.write( + "[file]\n" + "Dontwantthis_ENV_VAR=Jelly\n" + f"[{section}:suite.rc]\n" + "JINJA2_VAR=64\n" + "Another_Jinja2_var=Defined in config\n" + ) + return tmp_path + return wrapped_function + + +def test_rose_fileinstall(tmp_path): + """Check that we can install files specified in a rose-suite.conf. + + """ + othersource_dir = tmp_path / "sources" + workflow_dir = tmp_path / "workflow" + destination_dir = tmp_path / "destination" + for dir_ in [othersource_dir, workflow_dir, destination_dir]: + dir_.mkdir() + + york_content = "Now is the winter of our discontent..." + lancaster_content = "Thus did ever rebellion find rebuke." + york = othersource_dir / "York" + lancaster = othersource_dir / "Lancaster" + (york).write_text(york_content) + (lancaster).write_text(lancaster_content) + + red = destination_dir / "Red" + white = destination_dir / "White" + + with open(workflow_dir / 'rose-suite.conf', 'w+') as testfh: + # The [env] section is there to make sure I don't load it with + # the jinja2 method. + testfh.write( + "[file:White]\n" + f"source={str(york)}\n" + "[file:Red]\n" + f"source={str(lancaster)}\n" + ) + + rose_fileinstall(dir_=str(workflow_dir), dest_root=str(destination_dir)) + + assert red.read_text() == lancaster.read_text() + assert white.read_text() == york.read_text()