<datetime>-<start/restart/reload>.cylc contain processed, sparse co…

…nfig. (#4521)

* Allow processed config idump to take a handle to pass to printcfg
Made the ``<datetime>-<start/restart/reload>.cylc use pcfg.idump()
  to have the processed and sparse config dumped.

* ensure the start.cylc file is --sparse

* CHANGElog

* made file handler use 'w' not 'wb

* Update cylc/flow/parsec/util.py

Co-authored-by: Ronnie Dutta <[email protected]>

* Update CHANGES.md

* Update cylc/flow/parsec/util.py

Co-authored-by: Melanie Hall <[email protected]>

* Update CHANGES.md

Co-authored-by: Ronnie Dutta <[email protected]>
Co-authored-by: Melanie Hall <[email protected]>

CHANGES.md

@@ -48,7 +48,7 @@ creating a new release entry be sure to copy & paste the span tag with the
 updated. Only the first match gets replaced, so it's fine to leave the old
 ones in. -->
 -------------------------------------------------------------------------------
-## __cylc-8.0rc1 (<span actions:bind='release-date'></span>)__
+## __cylc-8.0rc1 (<span actions:bind='release-date'>Upcoming</span>)__
 
 First Release Candidate for Cylc 8.
 
@@ -66,6 +66,10 @@ being allowed as installation target names.
 [#4442](https://github.com/cylc/cylc-flow/pull/4442) - Prevent installation
 of workflows inside other installed workflows.
 
+[#4521](https://github.com/cylc/cylc-flow/pull/4521) - The workflow config
+logs (that get written in `log/flow-config/` on start/restart/reload) 
+are now sparse, i.e. they will no longer be fleshed-out with defaults.
+
 ### Fixes
 
 -------------------------------------------------------------------------------
@@ -132,7 +136,6 @@ error for a config item that isn't valid, to one that isn't set.
 [#4457](https://github.com/cylc/cylc-flow/pull/4457) - Cylc 8
 `cycle point time zone` now defaults to UTC, except in Cylc 7 compatibility mode.
 
-
 ### Fixes
 
 [#4493](https://github.com/cylc/cylc-flow/pull/4493) - handle late job

cylc/flow/parsec/config.py

@@ -142,7 +142,7 @@ def get(self, keys=None, sparse=False):
         return cfg
 
     def idump(self, items=None, sparse=False, prefix='',
-              oneline=False, none_str=''):
+              oneline=False, none_str='', handle=None):
         """
         items is a list of --item style inputs:
            '[runtime][foo]script'.
@@ -158,10 +158,10 @@ def idump(self, items=None, sparse=False, prefix='',
                 mkeys.append(j)
         if null:
             mkeys = [[]]
-        self.mdump(mkeys, sparse, prefix, oneline, none_str)
+        self.mdump(mkeys, sparse, prefix, oneline, none_str, handle=handle)
 
     def mdump(self, mkeys=None, sparse=False, prefix='',
-              oneline=False, none_str=''):
+              oneline=False, none_str='', handle=None):
         if oneline:
             items = []
             if mkeys:
@@ -176,13 +176,18 @@ def mdump(self, mkeys=None, sparse=False, prefix='',
             print(prefix + ' '.join(items))
         elif mkeys:
             for keys in mkeys:
-                self.dump(keys, sparse, prefix, none_str)
+                self.dump(keys, sparse, prefix, none_str, handle=handle)
 
-    def dump(self, keys=None, sparse=False, prefix='', none_str=''):
+    def dump(
+        self, keys=None, sparse=False, prefix='', none_str='', handle=None
+    ):
         if not keys:
             keys = []
         cfg = self.get(keys, sparse)
-        printcfg(cfg, prefix=prefix, level=len(keys), none_str=none_str)
+        printcfg(
+            cfg, prefix=prefix, level=len(keys),
+            none_str=none_str, handle=handle
+        )
 
     def _get_namespace_parents(self) -> List[List[str]]:
         """Get a list of the parents of config items which can be user defined.

cylc/flow/parsec/util.py

@@ -122,6 +122,18 @@ def printcfg(cfg, level=0, indent=0, prefix='', none_str='',
              handle=None):
     """Pretty-print a parsec config item or section (nested dict).
 
+    Args:
+        cfg: The config to be printed.
+        level: Each level of the hierarchy is printed with this many extra
+            square brackets: E.g. if ``indent=2`` then ``[runtime]`` will be
+            printed as ``[[[runtime]]].
+        indent (int): Indentation of top level sections - if set the whole
+            document will have a left margin this many 4 space indents wide.
+        prefix (str): Prefix each line with this.
+        none_str (str): Value to insert instead of blank if no value is set in
+            config.
+        handle (stream handler): Where to write the output.
+
     As returned by parse.config.get().
     """
     # Note: default handle=sys.stdout, but explicitly setting this as a kwarg

cylc/flow/scheduler.py

@@ -71,7 +71,6 @@
 from cylc.flow.option_parsers import verbosity_to_env
 from cylc.flow.parsec.exceptions import TemplateVarLanguageClash
 from cylc.flow.parsec.OrderedDict import DictTree
-from cylc.flow.parsec.util import printcfg
 from cylc.flow.parsec.validate import DurationFloat
 from cylc.flow.pathutil import (
     get_workflow_run_dir,
@@ -1121,9 +1120,9 @@ def load_flow_file(self, is_reload=False):
             load_type = "start"
         file_name = get_workflow_run_config_log_dir(
             self.workflow, f"{time_str}-{load_type}.cylc")
-        with open(file_name, "wb") as handle:
-            handle.write(b"# cylc-version: %s\n" % CYLC_VERSION.encode())
-            printcfg(self.config.cfg, none_str=None, handle=handle)
+        with open(file_name, "w") as handle:
+            handle.write("# cylc-version: %s\n" % CYLC_VERSION)
+            self.config.pcfg.idump(sparse=True, handle=handle)
 
         if not self.config.initial_point and not self.is_restart:
             LOG.warning('No initial cycle point provided - no cycling tasks '

tests/functional/startup/01-log-flow-config.t

@@ -17,7 +17,7 @@
 #-------------------------------------------------------------------------------
 # Test "log/flow-config/*-<mode>.cylc" files that are generated on workflow start up.
 . "$(dirname "$0")/test_header"
-set_test_number 9
+set_test_number 10
 
 init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__'
 #!Jinja2
@@ -58,20 +58,23 @@ start.cylc
 __OUT__
 
 LOGD="${RUN_DIR}/${WORKFLOW_NAME}/log/flow-config"
-RUN_CONFIG="$(ls "${LOGD}/"*-start.cylc)"
+START_CONFIG="$(ls "${LOGD}/"*-start.cylc)"
 REL_CONFIG="$(ls "${LOGD}/"*-reload.cylc)"
 RES_CONFIG="$(ls "${LOGD}/"*-restart.cylc)"
 # The generated *-run.cylc and *-reload.cylc should be identical
 # The generated *.cylc files should validate
-cmp_ok "${RUN_CONFIG}" "${REL_CONFIG}"
-run_ok "${TEST_NAME_BASE}-validate-start-config" cylc validate "${RUN_CONFIG}"
+cmp_ok "${START_CONFIG}" "${REL_CONFIG}"
+run_ok "${TEST_NAME_BASE}-validate-start-config" cylc validate "${START_CONFIG}"
 run_ok "${TEST_NAME_BASE}-validate-restart-config" cylc validate "${RES_CONFIG}"
 
-diff -u "${RUN_CONFIG}" "${RES_CONFIG}" >'diff.out'
+diff -u "${START_CONFIG}" "${RES_CONFIG}" >'diff.out'
 contains_ok 'diff.out' <<'__DIFF__'
 -    description = the weather is bad
 +    description = the weather is good
 __DIFF__
 
+# Ensure that the start config is sparse.
+grep_fail "\[\[mail\]\]" "${START_CONFIG}"
+
 purge
 exit