From 3790acf932593d26b131d1747d022074b1ea3410 Mon Sep 17 00:00:00 2001 From: Mel Hall <37735232+datamel@users.noreply.github.com> Date: Mon, 7 Sep 2020 13:55:04 +0100 Subject: [PATCH] Fix docstrings and install_target for restart --- CHANGES.md | 3 +++ cylc/flow/cfgspec/globalcfg.py | 11 +++++++++- cylc/flow/cfgspec/suite.py | 13 +++++++----- cylc/flow/platforms.py | 3 ++- cylc/flow/remote.py | 9 ++++++-- cylc/flow/scheduler.py | 23 +++++++++++++++------ tests/functional/remote/03-install-target.t | 2 +- 7 files changed, 48 insertions(+), 16 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ec60e66f409..b82808fa7d2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -48,6 +48,9 @@ compatibility, the `cylc run` command will automatically symlink an existing ### Enhancements +[#3796](https://github.com/cylc/cylc-flow/pull/3796) - Remote installation is +now on a per install target rather than a per platform basis. app/ bin/ etc/ lib/ directories are now installed on the target, configurable in flow.cylc. + [#3724](https://github.com/cylc/cylc-flow/pull/3724) - Re-implemented the `cylc scan` command line interface and added a Python API for accessing workflow scanning functionality. diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 40fd334ffb7..a4e60ed08ce 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -466,7 +466,16 @@ Conf('owner', VDR.V_STRING) Conf('install target', VDR.V_STRING, desc=''' This defaults to the platform name. This will be used as the - target for remote file installation.''') + target for remote file installation. + For example, to indicate to Cylc that Platform_A shares a file + system with localhost, we would configure as follows: + + .. code-block:: cylc + + [platforms] + [[Platform_A]] + install target = localhost + ''') with Conf('localhost', meta=Platform): Conf('hosts', VDR.V_STRING_LIST, ['localhost']) diff --git a/cylc/flow/cfgspec/suite.py b/cylc/flow/cfgspec/suite.py index 8feb4cd2a71..8cfadd8c4d8 100644 --- a/cylc/flow/cfgspec/suite.py +++ b/cylc/flow/cfgspec/suite.py @@ -90,11 +90,12 @@ Configure the directories and files to be included in the remote file installation. .. note:: + These, as standard, include the following directories: - app - bin - etc - lib + * app + * bin + * etc + * lib And include the server.key file (from the .service directory), this is required for authentication. These should be located in the top level of your Cylc workflow, @@ -102,7 +103,8 @@ Directories must have a trailing slash. For example, to add the following items to your file installation: - .. code-block:: bash + .. code-block:: none + ~/cylc-run/workflow_x |__dir1/ |__dir2/ @@ -110,6 +112,7 @@ |__file2 .. code-block:: cylc + [scheduler] includes = dir/, dir2/, file1, file2 ''') diff --git a/cylc/flow/platforms.py b/cylc/flow/platforms.py index 82533fb236f..47298ee9ca3 100644 --- a/cylc/flow/platforms.py +++ b/cylc/flow/platforms.py @@ -247,7 +247,8 @@ def get_host_from_platform(platform, method=None): def get_install_target_from_platform(platform): - """Sets install target to default platform name. + """Sets install target to configured or default platform name. + Args: platform (dict): A dict representing a platform. diff --git a/cylc/flow/remote.py b/cylc/flow/remote.py index aa2d8544b55..1c83e337365 100644 --- a/cylc/flow/remote.py +++ b/cylc/flow/remote.py @@ -174,13 +174,18 @@ def get_includes_to_rsync(rsync_includes=None): def construct_rsync_over_ssh_cmd( src_path, dst_path, platform, logfile=None, rsync_includes=None): - """Constructs the rsync command used for remote file installation - Args: + """Constructs the rsync command used for remote file installation. + + Includes as standard the directories: app, bin, etc, lib; and the server + key, used for ZMQ authentication. + + Args: src_path(string): source path dst_path(string): path of target platform(dict)): contains info relating to platform logfile(str): the path to the file logging the rsync rsync_includes(list): files and directories to be included in the rsync + """ dst_host = platform['name'] rsync_cmd = "rsync -v --perms --recursive --links --checksum --delete" diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py index d352e190de1..b9259efacb0 100644 --- a/cylc/flow/scheduler.py +++ b/cylc/flow/scheduler.py @@ -98,7 +98,9 @@ get_time_string_from_unix_time as time2str, get_utc_mode) from cylc.flow.xtrigger_mgr import XtriggerManager -from cylc.flow.platforms import platform_from_name +from cylc.flow.platforms import ( + get_install_target_from_platform, + platform_from_name) class SchedulerStop(CylcError): @@ -721,18 +723,26 @@ def restart_remote_init(self): """ - def is_in_list(platform, distinct_platform): - for distinct_platform in distinct_platform: - if(platform['install target'] + def is_platform_with_target_in_list( + install_target, distinct_platforms_list): + """Determines whether install target is in the list of platforms""" + for distinct_platform in distinct_platforms_list: + if(install_target == distinct_platform['install target']): return True distinct_install_target_platforms = [] for itask in self.pool.get_rh_tasks(): + itask.platform['install target'] = ( + get_install_target_from_platform(itask.platform)) if itask.state(*TASK_STATUSES_ACTIVE): - if not is_in_list(itask.platform, - distinct_install_target_platforms): + if not ( + is_platform_with_target_in_list( + itask.platform['install target'], + distinct_install_target_platforms + ) + ): distinct_install_target_platforms.append(itask.platform) incomplete_init = False @@ -743,6 +753,7 @@ def is_in_list(platform, distinct_platform): incomplete_init = True if incomplete_init: + # TODO: Review whether this sleep is needed. sleep(1.0) # Remote init is done via process pool self.proc_pool.process() diff --git a/tests/functional/remote/03-install-target.t b/tests/functional/remote/03-install-target.t index b333e7da4c9..22b7c86e3b3 100644 --- a/tests/functional/remote/03-install-target.t +++ b/tests/functional/remote/03-install-target.t @@ -45,7 +45,7 @@ __FLOW_CONFIG__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" \ -s "CYLC_TEST_PLATFORM=${CYLC_TEST_PLATFORM}" -suite_run_ok "${TEST_NAME_BASE}-run" cylc run --debug --no-detach \ +suite_run_ok "${TEST_NAME_BASE}-run" cylc run --debug \ "${SUITE_NAME}" -s "CYLC_TEST_PLATFORM=${CYLC_TEST_PLATFORM}" CYLC_SUITE_RUN_DIR="$RUN_DIR/${SUITE_NAME}" poll_grep_suite_log 'Suite held'