Symlink dirs cli

CHANGES.md

@@ -72,6 +72,10 @@ Add an option for displaying source workflows in `cylc scan`.
 - Expose runahead limiting to UIs; restore correct force-triggering of queued
 tasks for Cylc 8.
 
+[#4250](https://github.com/cylc/cylc-flow/pull/4250) -
+Symlink dirs localhost symlinks are now overridable with cli option
+`--symlink-dirs`.
+
 [#4103](https://github.com/cylc/cylc-flow/pull/4103) -
 Expose runahead limiting to UIs; restore correct force-triggering of queued
 tasks for Cylc 8.
@@ -123,7 +127,6 @@ when initial cycle point is not valid for the cycling type.
 [#4228](https://github.com/cylc/cylc-flow/pull/4228) - Interacting with a
 workflow on the cli using `runN` is now supported.
 
-
 [#4193](https://github.com/cylc/cylc-flow/pull/4193) - Standard `cylc install`
 now correctly installs from directories with a `.` in the name. Symlink dirs
 now correctly expands environment variables on the remote. Fixes minor cosmetic

cylc/flow/cfgspec/globalcfg.py

@@ -369,6 +369,56 @@
                If workflow source directories of the same name exist in more
                than one of these paths, only the first one will be picked up.
         ''')
+        # Symlink Dirs
+        with Conf('symlink dirs',  # noqa: SIM117 (keep same format)
+                  desc="""
+            Configure alternate workflow run directory locations. Symlinks from
+            the the standard ``$HOME/cylc-run`` locations will be created.
+        """):
+            with Conf('<install target>'):
+                Conf('run', VDR.V_STRING, None, desc="""
+                    If specified, the workflow run directory will
+                    be created in ``<run dir>/cylc-run/<workflow-name>`` and a
+                    symbolic link will be created from
+                    ``$HOME/cylc-run/<workflow-name>``.
+                    If not specified the workflow run directory will be created
+                    in ``$HOME/cylc-run/<workflow-name>``.
+                    All the workflow files and the ``.service`` directory get
+                    installed into this directory.
+                """)
+                Conf('log', VDR.V_STRING, None, desc="""
+                    If specified the workflow log directory will be created in
+                    ``<log dir>/cylc-run/<workflow-name>/log`` and a symbolic
+                    link will be created from
+                    ``$HOME/cylc-run/<workflow-name>/log``. If not specified
+                    the workflow log directory will be created in
+                    ``$HOME/cylc-run/<workflow-name>/log``.
+                """)
+                Conf('share', VDR.V_STRING, None, desc="""
+                    If specified the workflow share directory will be
+                    created in ``<share dir>/cylc-run/<workflow-name>/share``
+                    and a symbolic link will be created from
+                    ``<$HOME/cylc-run/<workflow-name>/share``. If not specified
+                    the workflow share directory will be created in
+                    ``$HOME/cylc-run/<workflow-name>/share``.
+                """)
+                Conf('share/cycle', VDR.V_STRING, None, desc="""
+                    If specified the workflow share/cycle directory
+                    will be created in
+                    ``<share/cycle dir>/cylc-run/<workflow-name>/share/cycle``
+                    and a symbolic link will be created from
+                    ``$HOME/cylc-run/<workflow-name>/share/cycle``. If not
+                    specified the workflow share/cycle directory will be
+                    created in ``$HOME/cylc-run/<workflow-name>/share/cycle``.
+                """)
+                Conf('work', VDR.V_STRING, None, desc="""
+                    If specified the workflow work directory will be created in
+                    ``<work dir>/cylc-run/<workflow-name>/work`` and a symbolic
+                    link will be created from
+                    ``$HOME/cylc-run/<workflow-name>/work``. If not specified
+                    the workflow work directory will be created in
+                    ``$HOME/cylc-run/<workflow-name>/work``.
+                """)
 
     with Conf('editors', desc='''
         Choose your favourite text editor for editing workflow configurations.
@@ -711,60 +761,6 @@
     with Conf('platform groups'):  # noqa: SIM117 (keep same format)
         with Conf('<group>'):
             Conf('platforms', VDR.V_STRING_LIST)
-    # Symlink Dirs
-    with Conf('symlink dirs',   # noqa: SIM117 (keep same format)
-              desc="""
-        Define directories to be moved, symlinks from the the original
-        ``$HOME/cylc-run`` directories will be created.
-    """):
-        with Conf('<install target>'):
-            Conf('run', VDR.V_STRING, None, desc="""
-                Specifies the directory where the workflow run directories are
-                created. If specified, the workflow run directory will be
-                created in ``<run dir>/cylc-run/<workflow-name>`` and a
-                symbolic link will be created from
-                ``$HOME/cylc-run/<workflow-name>``.
-                If not specified the workflow run directory will be created in
-                ``$HOME/cylc-run/<workflow-name>``.
-                All the workflow files and the ``.service`` directory get
-                installed into this directory.
-            """)
-            Conf('log', VDR.V_STRING, None, desc="""
-                Specifies the directory where log directories are created. If
-                specified the workflow log directory will be created in
-                ``<log dir>/cylc-run/<workflow-name>/log`` and a symbolic link
-                will be created from ``$HOME/cylc-run/<workflow-name>/log``. If
-                not specified the workflow log directory will be created in
-                ``$HOME/cylc-run/<workflow-name>/log``.
-            """)
-            Conf('share', VDR.V_STRING, None, desc="""
-                Specifies the directory where share directories are created.
-                If specified the workflow share directory will be created in
-                ``<share dir>/cylc-run/<workflow-name>/share`` and a symbolic
-                link will be created from
-                ``<$HOME/cylc-run/<workflow-name>/share``. If not specified the
-                workflow share directory will be created in
-                ``$HOME/cylc-run/<workflow-name>/share``.
-            """)
-            Conf('share/cycle', VDR.V_STRING, None, desc="""
-                Specifies the directory where share/cycle directories are
-                created. If specified the workflow share/cycle directory will
-                be created in
-                ``<share/cycle dir>/cylc-run/<workflow-name>/share/cycle`` and
-                a symbolic link will be created from
-                ``$HOME/cylc-run/<workflow-name>/share/cycle``. If not
-                specified the workflow share/cycle directory will be created in
-                ``$HOME/cylc-run/<workflow-name>/share/cycle``.
-            """)
-            Conf('work', VDR.V_STRING, None, desc="""
-                Specifies the directory where work directories are created.
-                If specified the workflow work directory will be created in
-                ``<work dir>/cylc-run/<workflow-name>/work`` and a symbolic
-                link will be created from
-                ``$HOME/cylc-run/<workflow-name>/work``. If not specified the
-                workflow work directory will be created in
-                ``$HOME/cylc-run/<workflow-name>/work``.
-            """)
     # task
     with Conf('task events', desc='''
         Global site/user defaults for

cylc/flow/cfgspec/workflow.py

@@ -788,7 +788,7 @@
 
                 The top level share and work directory location can be changed
                 (e.g. to a large data area) by a global config setting (see
-                :cylc:conf:`global.cylc[symlink dirs]`).
+                :cylc:conf:`global.cylc[install][symlink dirs]`).
 
                 .. note::
 

cylc/flow/pathutil.py

@@ -19,7 +19,7 @@
 from pathlib import Path
 import re
 from shutil import rmtree
-from typing import Dict, Iterable, Set, Union
+from typing import Dict, Iterable, Set, Union, Optional, Any
 
 from cylc.flow import LOG
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
@@ -28,7 +28,6 @@
 )
 from cylc.flow.platforms import get_localhost_install_target
 
-
 # Note: do not import this elsewhere, as it might bypass unit test
 # monkeypatching:
 _CYLC_RUN_DIR = os.path.join('$HOME', 'cylc-run')
@@ -131,22 +130,29 @@ def make_workflow_run_tree(workflow):
 
 
 def make_localhost_symlinks(
-    rund: Union[Path, str], named_sub_dir: str
+    rund: Union[Path, str],
+    named_sub_dir: str,
+    symlink_conf: Optional[Dict[str, Dict[str, str]]] = None
 ) -> Dict[str, Union[Path, str]]:
     """Creates symlinks for any configured symlink dirs from glbl_cfg.
     Args:
         rund: the entire run directory path
         named_sub_dir: e.g flow_name/run1
+        symlink_conf: Symlinks dirs configuration passed from cli
 
     Returns:
         Dictionary of symlinks with sources as keys and
         destinations as values: ``{source: destination}``
 
     """
-    dirs_to_symlink = get_dirs_to_symlink(
-        get_localhost_install_target(), named_sub_dir)
     symlinks_created = {}
+    dirs_to_symlink = get_dirs_to_symlink(
+        get_localhost_install_target(),
+        named_sub_dir, symlink_conf=symlink_conf
+    )
     for key, value in dirs_to_symlink.items():
+        if value is None:
+            continue
         if key == 'run':
             symlink_path = rund
         else:
@@ -165,21 +171,34 @@ def make_localhost_symlinks(
     return symlinks_created
 
 
-def get_dirs_to_symlink(install_target: str, flow_name: str) -> Dict[str, str]:
-    """Returns dictionary of directories to symlink from glbcfg.
+def get_dirs_to_symlink(
+    install_target: str,
+    flow_name: str,
+    symlink_conf: Optional[Dict[str, Dict[str, Any]]] = None
+) -> Dict[str, Any]:
+    """Returns dictionary of directories to symlink.
 
     Note the paths should remain unexpanded, to be expanded on the remote.
-    """
-    dirs_to_symlink: Dict[str, str] = {}
-    symlink_conf = glbl_cfg().get(['symlink dirs'])
+    Args:
+        install_target: Symlinks to be created on this install target
+        flow_name: full name of the run, e.g. myflow/run1
+        symlink_conf: Symlink dirs, if sent on the cli.
+            Defaults to None, in which case global config symlink dirs will
+            be applied.
 
+    Returns:
+        dirs_to_symlink: [directory: symlink_path]
+    """
+    dirs_to_symlink: Dict[str, Any] = {}
+    if symlink_conf is None:
+        symlink_conf = glbl_cfg().get(['install', 'symlink dirs'])
     if install_target not in symlink_conf.keys():
         return dirs_to_symlink
     base_dir = symlink_conf[install_target]['run']
     if base_dir:
         dirs_to_symlink['run'] = os.path.join(base_dir, 'cylc-run', flow_name)
     for dir_ in ['log', 'share', 'share/cycle', 'work']:
-        link = symlink_conf[install_target][dir_]
+        link = symlink_conf[install_target].get(dir_, None)
         if (not link) or link == base_dir:
             continue
         dirs_to_symlink[dir_] = os.path.join(link, 'cylc-run', flow_name, dir_)
@@ -196,7 +215,9 @@ def make_symlink(path: Union[Path, str], target: Union[Path, str]) -> bool:
     path = Path(path)
     target = Path(target)
     if path.exists():
-        if path.is_symlink() and path.samefile(target):
+        # note all three checks are needed here due to case where user has set
+        # their own symlink which does not match the global config set one.
+        if path.is_symlink() and target.exists() and path.samefile(target):
             # correct symlink already exists
             return False
         # symlink name is in use by a physical file or directory
@@ -213,10 +234,11 @@ def make_symlink(path: Union[Path, str], target: Union[Path, str]) -> bool:
             raise WorkflowFilesError(
                 f"Error when symlinking. Failed to unlink bad symlink {path}.")
     target.mkdir(parents=True, exist_ok=True)
+
+    # This is needed in case share and share/cycle have the same symlink dir:
     if path.exists():
-        # Trying to link to itself; no symlink needed
-        # (e.g. path's parent is symlink to target's parent)
         return False
+
     path.parent.mkdir(parents=True, exist_ok=True)
     try:
         path.symlink_to(target)

cylc/flow/scripts/install.py

@@ -69,13 +69,13 @@
 
 """
 
-from typing import Optional, TYPE_CHECKING
+from typing import Optional, TYPE_CHECKING, Dict, Any
 
 from cylc.flow import iter_entry_points
 from cylc.flow.exceptions import PluginError
 from cylc.flow.option_parsers import CylcOptionParser as COP
 from cylc.flow.workflow_files import (
-    install_workflow, search_install_source_dirs
+    install_workflow, search_install_source_dirs, parse_cli_sym_dirs
 )
 from cylc.flow.terminal import cli_function
 
@@ -150,6 +150,18 @@ def get_option_parser():
         default=None,
         dest="source")
 
+    parser.add_option(
+        "--symlink-dirs",
+        help=(
+            "Enter a list, in the form 'log=path/to/store, share = $...'"
+            ". Use this option to override local symlinks for directories run,"
+            " log, work, share, share/cycle, as configured in global.cylc. "
+            "Enter an empty list \"\" to skip making localhost symlink dirs."
+        ),
+        action="store",
+        dest="symlink_dirs"
+    )
+
     parser.add_option(
         "--run-name",
         help="Name the run.",
@@ -165,13 +177,6 @@ def get_option_parser():
         default=False,
         dest="no_run_name")
 
-    parser.add_option(
-        "--no-symlink-dirs",
-        help="Use this option to override creating default local symlinks.",
-        action="store_true",
-        default=False,
-        dest="no_symlinks")
-
     parser = add_cylc_rose_options(parser)
 
     return parser
@@ -214,13 +219,17 @@ def install(
                 entry_point.name,
                 exc
             ) from None
-
+    cli_symdirs: Optional[Dict[str, Dict[str, Any]]] = None
+    if opts.symlink_dirs == '':
+        cli_symdirs = {}
+    elif opts.symlink_dirs:
+        cli_symdirs = parse_cli_sym_dirs(opts.symlink_dirs)
     source_dir, rundir, _flow_name = install_workflow(
         flow_name=flow_name,
         source=source,
         run_name=opts.run_name,
         no_run_name=opts.no_run_name,
-        no_symlinks=opts.no_symlinks
+        cli_symlink_dirs=cli_symdirs
     )
 
     for entry_point in iter_entry_points(