Skip to content

Commit

Permalink
reinstall: add --dry-run option
Browse files Browse the repository at this point in the history
  • Loading branch information
@oliver-sanders
oliver-sanders committed Jul 5, 2022
1 parent 81b5e0c commit 2f195de
Show file tree
Hide file tree
Showing 3 changed files with 180 additions and 52 deletions.
127 changes: 90 additions & 37 deletions cylc/flow/scripts/reinstall.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,23 @@
Examples:
# Having previously installed:
$ cylc install myflow
# To reinstall the latest run:
$ cylc reinstall myflow
# Or, to reinstall a specific run:
$ cylc reinstall myflow/run2
# Having previously installed:
$ cylc install myflow --no-run-name
# To reinstall this workflow run:
$ cylc reinstall myflow
# View the changes reinstall would make:
$ cylc reinstall myflow --dry-run
"""

from pathlib import Path
import sys
from typing import Optional, TYPE_CHECKING

from ansimarkup import parse as cparse

from cylc.flow import iter_entry_points
from cylc.flow.exceptions import PluginError, WorkflowFilesError
from cylc.flow.id_cli import parse_id
Expand Down Expand Up @@ -75,9 +78,42 @@ def get_option_parser() -> COP:
dest="clear_rose_install_opts"
)

parser.add_option(
'--dry', '--dry-run',
action='store_true',
help='Show the changes reinstallation would make.'
)

return parser


def format_rsync_out(out):
"""Format rsync stdout for presenting to users.
Note: Output formats of different rsync implementations may differ so keep
this code simple and robust.
"""
lines = []
for line in out.splitlines():
if line[0:4] == 'send':
# file added or updated
lines.append(cparse(f'<green>{line}</green>'))
elif line[0:4] == 'del.':
# file deleted
lines.append(cparse(f'<red>{line}</red>'))
elif line == 'cannot delete non-empty directory: opt':
# These "cannot delete non-empty directory" messages can arise
# as a result of excluding files within sub-directories.
# This opt dir message is likely to occur when a rose-suit.conf
# file is present.
continue
else:
# other uncategorised log line
lines.append(line)
return lines


@cli_function(get_option_parser)
def main(
parser: COP,
Expand All @@ -104,41 +140,58 @@ def main(
f'Restore the source or modify the "{source_symlink}"'
' symlink to continue.'
)
for entry_point in iter_entry_points(
'cylc.pre_configure'
):
try:
entry_point.resolve()(srcdir=source, opts=opts)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError(
'cylc.pre_configure',
entry_point.name,
exc
) from None

reinstall_workflow(

if not opts.dry:
for entry_point in iter_entry_points(
'cylc.pre_configure'
):
try:
entry_point.resolve()(srcdir=source, opts=opts)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError(
'cylc.pre_configure',
entry_point.name,
exc
) from None

stdout = reinstall_workflow(
source=Path(source),
named_run=workflow_id,
rundir=run_dir,
dry_run=False # TODO: ready for dry run implementation
dry_run=opts.dry,
)

for entry_point in iter_entry_points(
'cylc.post_install'
):
try:
entry_point.resolve()(
srcdir=source,
opts=opts,
rundir=str(run_dir)
)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError(
'cylc.post_install',
entry_point.name,
exc
) from None
if Path(source, 'rose-suite.conf').is_file():
print(
cparse(
'<blue>'
'NOTE: Files created by Rose file installation will show as'
' deleted.'
'\n They will be re-created during the reinstall'
' process.'
'</blue>',
),
file=sys.stderr,
)
print('\n'.join(format_rsync_out(stdout)), file=sys.stderr)

if not opts.dry:
for entry_point in iter_entry_points(
'cylc.post_install'
):
try:
entry_point.resolve()(
srcdir=source,
opts=opts,
rundir=str(run_dir)
)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError(
'cylc.post_install',
entry_point.name,
exc
) from None
56 changes: 42 additions & 14 deletions cylc/flow/workflow_files.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1389,12 +1389,24 @@ def get_cylc_run_abs_path(path: Union[Path, str]) -> Union[Path, str]:
return get_workflow_run_dir(path)


def _get_logger(rund, log_name):
"""Get log and create and open if necessary."""
def _get_logger(rund, log_name, open_file=True):
"""Get log and create and open if necessary.
Args:
rund:
The workflow run directory of the associated workflow.
log_name:
The name of the log to open.
open_file:
Open the appropriate log file and add it as a file handler to
the logger. I.E. Start writing the log to a file if not already
doing so.
"""
logger = logging.getLogger(log_name)
if logger.getEffectiveLevel != logging.INFO:
logger.setLevel(logging.INFO)
if not logger.hasHandlers():
if open_file and not logger.hasHandlers():
_open_install_log(rund, logger)
return logger

Expand Down Expand Up @@ -1471,7 +1483,7 @@ def reinstall_workflow(
named_run: str,
rundir: Path,
dry_run: bool = False
) -> None:
) -> str:
"""Reinstall workflow.
Args:
Expand All @@ -1480,29 +1492,45 @@ def reinstall_workflow(
rundir: run directory
dry_run: if True, will not execute the file transfer but report what
would be changed.
Returns:
Stdout from the rsync command.
"""
validate_source_dir(source, named_run)
check_nested_dirs(rundir)
reinstall_log = _get_logger(rundir, 'cylc-reinstall')
reinstall_log.info(f"Reinstalling \"{named_run}\", from "
f"\"{source}\" to \"{rundir}\"")
reinstall_log = _get_logger(
rundir,
'cylc-reinstall',
open_file=not dry_run, # don't open the log file for --dry-run
)
reinstall_log.info(
f'Reinstalling "{named_run}", from "{source}" to "{rundir}"'
)
rsync_cmd = get_rsync_rund_cmd(
source, rundir, reinstall=True, dry_run=dry_run)
source,
rundir,
reinstall=True,
dry_run=dry_run,
)
reinstall_log.info(cli_format(rsync_cmd))
# print(cli_format(rsync_cmd))
proc = Popen(rsync_cmd, stdout=PIPE, stderr=PIPE, text=True) # nosec
# * command is constructed via internal interface
stdout, stderr = proc.communicate()
reinstall_log.info(
f"Copying files from {source} to {rundir}"
f'\n{stdout}'
)

if proc.returncode != 0:
reinstall_log.warning(
f"An error occurred when copying files from {source} to {rundir}")
f"An error occurred when copying files from {source} to {rundir}"
)
reinstall_log.warning(f" Error: {stderr}")
check_flow_file(rundir)
reinstall_log.info(f'REINSTALLED {named_run} from {source}')
print(f'REINSTALLED {named_run} from {source}')
print(
f'REINSTALL{"ED" if not dry_run else ""} {named_run} from {source}'
)
close_log(reinstall_log)
return stdout


def abort_if_flow_file_in_path(source: Path) -> None:
Expand Down
49 changes: 48 additions & 1 deletion tests/functional/cylc-reinstall/00-simple.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
#------------------------------------------------------------------------------
# Test workflow re-installation
. "$(dirname "$0")/test_header"
set_test_number 11
set_test_number 17

# Test basic cylc reinstall, named run given
TEST_NAME="${TEST_NAME_BASE}-basic-named-run"
Expand All @@ -35,6 +35,7 @@ grep_ok "REINSTALLED ${RND_WORKFLOW_NAME}/run1 from ${RND_WORKFLOW_SOURCE}" "${R
popd || exit 1
purge_rnd_workflow

#------------------------------------------------------------------------------
# Test install/reinstall executed from elsewhere in filesystem
TEST_NAME="${TEST_NAME_BASE}-named-flow"
make_rnd_workflow
Expand All @@ -51,6 +52,7 @@ popd || exit 1
purge_rnd_workflow
rm -rf "${RUN_DIR:?}/${RND_WORKFLOW_NAME}/"

#------------------------------------------------------------------------------
# Test cylc reinstall succeeds if suite.rc file in source dir
TEST_NAME="${TEST_NAME_BASE}-no-flow-file"
make_rnd_workflow
Expand All @@ -64,4 +66,49 @@ __OUT__
run_ok "${TEST_NAME}-reinstall" cylc reinstall "${RND_WORKFLOW_NAME}/run1"
purge_rnd_workflow

#------------------------------------------------------------------------------
# Test --dry-run
TEST_NAME="${TEST_NAME_BASE}-dry-run"
make_rnd_workflow
touch "${RND_WORKFLOW_SOURCE}/a"
touch "${RND_WORKFLOW_SOURCE}/b"
touch "${RND_WORKFLOW_SOURCE}/rose-suite.conf"
run_ok "${TEST_NAME}" \
cylc install "${RND_WORKFLOW_SOURCE}" \
--workflow-name="${RND_WORKFLOW_NAME}" --no-run-name
rm "${RND_WORKFLOW_SOURCE}/a"
touch "${RND_WORKFLOW_SOURCE}/c"
# make sure the install log was created
RUN_DIR="${HOME}/cylc-run/${RND_WORKFLOW_NAME}"
if [[ "$(ls -1 "${RUN_DIR}/log/install"/* | wc -w)" == 1 ]]
then
ok "${TEST_NAME}-log"
else
fail "${TEST_NAME}-log"
fi
run_ok "${TEST_NAME}" \
cylc reinstall "${RND_WORKFLOW_NAME}" \
--dry-run --color=never
# the dry run output should go to stderr
cmp_ok "${TEST_NAME}.stderr" <<__OUT__
NOTE: Files created by Rose file installation will show as deleted.
They will be re-created during the reinstall process.
del. a
send c
__OUT__
# make sure that rsync was indeed run in dry mode!
if [[ -f "${RUN_DIR}/c" ]]; then
fail "${TEST_NAME}-transfer"
else
ok "${TEST_NAME}-transfer"
fi
# make sure no reinstall log file was created
if [[ "$(ls -1 "${RUN_DIR}/log/install"/* | wc -w)" == 1 ]]
then
ok "${TEST_NAME}-log"
else
fail "${TEST_NAME}-log"
fi


exit

0 comments on commit 2f195de

Please sign in to comment.