Merge branch 'master' into disallow_runN_and_runX_as_flow-names

CHANGES.md

@@ -59,6 +59,11 @@ First release candidate for Cylc 8.
 [#4526](https://github.com/cylc/cylc-flow/pull/4526) - Prevent runN and run\d+
 being allowed as installation target names.
 
+[#4442](https://github.com/cylc/cylc-flow/pull/4442) - Prevent installation
+of workflows inside other installed workflows.
+
+### Fixes
+
 -------------------------------------------------------------------------------
 ## __cylc-8.0b3 (<span actions:bind='release-date'>Released 2021-11-10</span>)__
 

MANIFEST.in

@@ -0,0 +1,2 @@
+include cylc/flow/py.typed
+recursive-include cylc/flow/etc/ *

conda-environment.yml

@@ -9,13 +9,14 @@ dependencies:
   - graphviz  # for static graphing
   - jinja2 ==2.11.0,<2.12
   - metomi-isodatetime >=1!2.0.2, <1!2.1.0
-  - protobuf >=3.15.0,<3.16.0
+  - protobuf >=3.19.0,<3.20.0
   - psutil >=5.6.0
   - python
   - pyuv >=1.4.0,<1.5.0
-  - pyzmq >=19.0.0,<19.1.0
+  - pyzmq >=22,<23
   - setuptools >=49
   - urwid >=2,<3
+
 # optional dependencies
   #- empy >=3.3,<3.4
   #- pandas >=1.0,<2

cylc/flow/scheduler.py

@@ -228,13 +228,13 @@ class Scheduler:
     auto_restart_time: Optional[float] = None
 
     # tcp / zmq
-    zmq_context: zmq.Context = None
+    zmq_context: Optional[zmq.Context] = None
     port: Optional[int] = None
     pub_port: Optional[int] = None
     server: Optional[WorkflowRuntimeServer] = None
     publisher: Optional[WorkflowPublisher] = None
     barrier: Optional[Barrier] = None
-    curve_auth: ThreadAuthenticator = None
+    curve_auth: Optional[ThreadAuthenticator] = None
     client_pub_key_dir: Optional[str] = None
 
     # queue-released tasks still in prep
@@ -1718,7 +1718,8 @@ async def _shutdown(self, reason: Exception) -> None:
                 [(b'shutdown', str(reason).encode('utf-8'))]
             )
             self.publisher.stop()
-        self.curve_auth.stop()  # stop the authentication thread
+        if self.curve_auth:
+            self.curve_auth.stop()  # stop the authentication thread
 
         # Flush errors and info before removing workflow contact file
         sys.stdout.flush()

cylc/flow/workflow_files.py

@@ -358,6 +358,11 @@ class RemoteCleanQueueTuple(NamedTuple):
     "This command will use ./{0}."
 )
 
+NESTED_DIRS_MSG = (
+    "Nested {dir_type} directories not allowed - cannot install workflow"
+    " in '{dest}' as '{existing}' is already a valid {dir_type} directory."
+)
+
 
 def _is_process_running(
     host: str,
@@ -1380,28 +1385,69 @@ def infer_latest_run(
     return (path, reg)
 
 
-def check_nested_run_dirs(run_dir: Union[Path, str]) -> None:
-    """Disallow nested run dirs e.g. trying to install foo/bar where foo is
-    already a valid workflow directory.
+def check_nested_dirs(
+    run_dir: Union[Path, str],
+    install_dir: Union[Path, str, None] = None
+) -> None:
+    """Disallow nested dirs:
+
+    - Nested installed run dirs
+    - Nested installed workflow dirs
 
     Args:
         run_dir: Absolute workflow run directory path.
+        install_dir: Absolute workflow install directory path
+            (contains _cylc-install). If None, will not check for nested
+            install dirs.
 
-    Raises WorkflowFilesError if reg dir is nested inside a run dir.
+    Raises:
+        WorkflowFilesError if reg dir is nested inside a run dir, or an
+            install dirs are nested.
     """
-    exc_msg = (
-        "Nested run directories not allowed - cannot install workflow in "
-        "'{0}' as '{1}' is already a valid run directory."
-    )
-    reg_path = Path(os.path.normpath(run_dir))
-    for parent_dir in reg_path.parents:
+    run_dir = Path(os.path.normpath(run_dir))
+    if install_dir is not None:
+        install_dir = Path(os.path.normpath(install_dir))
+    # Check parents:
+    for parent_dir in run_dir.parents:
+        # Stop searching at ~/cylc-run
         if parent_dir == Path(get_cylc_run_dir()):
             break
+        # check for run directories:
         if is_valid_run_dir(parent_dir):
             raise WorkflowFilesError(
-                exc_msg.format(reg_path, get_cylc_run_abs_path(parent_dir))
+                NESTED_DIRS_MSG.format(
+                    dir_type='run',
+                    dest=run_dir,
+                    existing=get_cylc_run_abs_path(parent_dir)
+                )
+            )
+        # Check for install directories:
+        if (
+            install_dir
+            and parent_dir in install_dir.parents
+            and (parent_dir / WorkflowFiles.Install.DIRNAME).is_dir()
+        ):
+            raise WorkflowFilesError(
+                NESTED_DIRS_MSG.format(
+                    dir_type='install',
+                    dest=run_dir,
+                    existing=get_cylc_run_abs_path(parent_dir)
+                )
             )
 
+    if install_dir:
+        # Search child tree for install directories:
+        for depth in range(MAX_SCAN_DEPTH):
+            search_pattern = f'*/{"*/" * depth}{WorkflowFiles.Install.DIRNAME}'
+            for result in install_dir.glob(search_pattern):
+                raise WorkflowFilesError(
+                    NESTED_DIRS_MSG.format(
+                        dir_type='install',
+                        dest=run_dir,
+                        existing=get_cylc_run_abs_path(result.parent)
+                    )
+                )
+
 
 def is_valid_run_dir(path):
     """Return True if path is a valid, existing run directory, else False.
@@ -1519,7 +1565,7 @@ def reinstall_workflow(named_run, rundir, source, dry_run=False):
             be changed.
     """
     validate_source_dir(source, named_run)
-    check_nested_run_dirs(rundir)
+    check_nested_dirs(rundir)
     reinstall_log = _get_logger(rundir, 'cylc-reinstall')
     reinstall_log.info(f"Reinstalling \"{named_run}\", from "
                        f"\"{source}\" to \"{rundir}\"")
@@ -1596,12 +1642,12 @@ def install_workflow(
     run_path_base = Path(get_workflow_run_dir(workflow_name))
     relink, run_num, rundir = get_run_dir_info(
         run_path_base, run_name, no_run_name)
+    check_nested_dirs(rundir, run_path_base)
     if Path(rundir).exists():
         raise WorkflowFilesError(
             f"\"{rundir}\" exists."
             " Try using cylc reinstall. Alternatively, install with another"
             " name, using the --run-name option.")
-    check_nested_run_dirs(rundir)
     symlinks_created = {}
     named_run = workflow_name
     if run_name:

setup.cfg

@@ -16,9 +16,16 @@
 
 [metadata]
 name = cylc-flow
+version = attr: cylc.flow.__version__
 author = Hilary Oliver
-url=https://cylc.github.io/
+url=https://cylc.org/
 description = A workflow engine for cycling systems
+long_description=file: README.md
+long_description_content_type=text/markdown
+project_urls =
+    Documentation = https://cylc.github.io/cylc-doc/latest/html/index.html
+    Source = https://github.com/cylc/cylc-flow
+    Tracker = https://github.com/cylc/cylc-flow/issues
 keywords =
     cycling-workflows
     hpc
@@ -46,20 +53,80 @@ classifiers =
     Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: Implementation :: CPython
     Topic :: Scientific/Engineering :: Atmospheric Science
+
+[options]
+packages = find_namespace:
+include_package_data = True
 python_requires = >=3.7
-setup_requires=pytest-runner==4.4
+install_requires =
+    aiofiles==0.7.*
+    ansimarkup>=1.0.0
+    colorama>=0.4,<=1
+    graphene>=2.1,<3
+    jinja2==2.11.*
+    metomi-isodatetime>=1!2.0.2, <1!2.1.0
+    protobuf==3.19.*
+    psutil>=5.6.0
+    pyuv==1.4.*
+    pyzmq==22.*
+    setuptools>=49
+    urwid==2.*
 
-[bdist_rpm]
-requires =
-    python3-colorama
-    python-isodatetime
-    python3-jinja2
-    python3-MarkupSafe
-    python3-zmq
+[options.packages.find]
+include = cylc*
 
-[aliases]
-# so that running python setup.py test invokes pytest
-test = pytest
+[options.extras_require]
+empy =
+    EmPy==3.3.*
+graph =
+    pillow
+main_loop-log_data_store =
+    pympler
+    matplotlib
+main_loop-log_main_loop =
+    matplotlib
+main_loop-log_memory =
+    pympler
+    matplotlib
+report-timings =
+    pandas==1.*
+    matplotlib
+tests =
+    async-timeout>=3.0.0
+    async_generator
+    bandit>=1.7.0
+    coverage>=5.0.0
+    flake8-broken-line>=0.3.0
+    flake8-bugbear>=21.0.0
+    flake8-builtins>=1.5.0
+    flake8-comprehensions>=3.5.0
+    flake8-debugger>=4.0.0
+    flake8-mutable>=1.2.0
+    flake8-simplify>=0.14.0
+    flake8>=3.0.0
+    mypy>=0.910
+    # TODO: https://github.com/pytest-dev/pytest-asyncio/issues/ 209
+    pytest-asyncio>=0.15.1
+    pytest-cov>=2.8.0
+    pytest-xdist>=2
+    pytest-env>=0.6.2
+    pytest>=6
+    testfixtures>=6.11.0
+    # Type annotation stubs
+    # http://mypy-lang.blogspot.com/2021/05/the-upcoming-switch-to-modular-typeshed.html
+    types-Jinja2>=0.1.3
+    types-aiofiles>=0.1.3
+    types-pkg_resources>=0.1.2
+    types-protobuf>=0.1.10
+    types-six>=0.1.6
+all =
+    %(empy)s
+    %(graph)s
+    %(main_loop-log_data_store)s
+    %(main_loop-log_main_loop)s
+    %(main_loop-log_memory)s
+    %(report-timings)s
+    %(tests)s
 
 [options.entry_points]
 # top level shell commands
@@ -102,7 +169,7 @@ cylc.command =
     remote-init = cylc.flow.scripts.remote_init:main
     remote-tidy = cylc.flow.scripts.remote_tidy:main
     remove = cylc.flow.scripts.remove:main
-    report-timings = cylc.flow.scripts.report_timings:main
+    report-timings = cylc.flow.scripts.report_timings:main [report-timings]
     scan = cylc.flow.scripts.scan:cli
     set-verbosity = cylc.flow.scripts.set_verbosity:main
     show = cylc.flow.scripts.show:main
@@ -118,12 +185,20 @@ cylc.command =
 cylc.main_loop =
     health_check = cylc.flow.main_loop.health_check
     auto_restart = cylc.flow.main_loop.auto_restart
-    log_data_store = cylc.flow.main_loop.log_data_store
-    log_main_loop = cylc.flow.main_loop.log_main_loop
-    log_memory = cylc.flow.main_loop.log_memory
+    log_data_store = cylc.flow.main_loop.log_data_store [main_loop-log_data_store]
+    log_main_loop = cylc.flow.main_loop.log_main_loop [main_loop-log_main_loop]
+    log_memory = cylc.flow.main_loop.log_memory [main_loop-log_memory]
     reset_bad_hosts = cylc.flow.main_loop.reset_bad_hosts
-# NOTE: all entry points should be listed here even if Cylc Flow does not
+# NOTE: all entry points should be listed here even if Cylc Flow does not
 # provide any implementations, to make entry point scraping easier
 cylc.pre_configure =
 cylc.post_install =
     log_vc_info = cylc.flow.install_plugins.log_vc_info:main
+
+[bdist_rpm]
+requires =
+    python3-colorama
+    python-isodatetime
+    python3-jinja2
+    python3-MarkupSafe
+    python3-zmq