diff --git a/CHANGES.md b/CHANGES.md index 63e1baf9b46..e8238f88137 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -72,6 +72,10 @@ Add an option for displaying source workflows in `cylc scan`. - Expose runahead limiting to UIs; restore correct force-triggering of queued tasks for Cylc 8. +[#4250](https://github.com/cylc/cylc-flow/pull/4250) - +Symlink dirs localhost symlinks are now overridable with cli option +`--symlink-dirs`. + [#4103](https://github.com/cylc/cylc-flow/pull/4103) - Expose runahead limiting to UIs; restore correct force-triggering of queued tasks for Cylc 8. @@ -123,7 +127,6 @@ when initial cycle point is not valid for the cycling type. [#4228](https://github.com/cylc/cylc-flow/pull/4228) - Interacting with a workflow on the cli using `runN` is now supported. - [#4193](https://github.com/cylc/cylc-flow/pull/4193) - Standard `cylc install` now correctly installs from directories with a `.` in the name. Symlink dirs now correctly expands environment variables on the remote. Fixes minor cosmetic diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 22c45fc4641..170d206d70c 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -369,6 +369,56 @@ If workflow source directories of the same name exist in more than one of these paths, only the first one will be picked up. ''') + # Symlink Dirs + with Conf('symlink dirs', # noqa: SIM117 (keep same format) + desc=""" + Configure alternate workflow run directory locations. Symlinks from + the the standard ``$HOME/cylc-run`` locations will be created. + """): + with Conf(''): + Conf('run', VDR.V_STRING, None, desc=""" + If specified, the workflow run directory will + be created in ``/cylc-run/`` and a + symbolic link will be created from + ``$HOME/cylc-run/``. + If not specified the workflow run directory will be created + in ``$HOME/cylc-run/``. + All the workflow files and the ``.service`` directory get + installed into this directory. + """) + Conf('log', VDR.V_STRING, None, desc=""" + If specified the workflow log directory will be created in + ``/cylc-run//log`` and a symbolic + link will be created from + ``$HOME/cylc-run//log``. If not specified + the workflow log directory will be created in + ``$HOME/cylc-run//log``. + """) + Conf('share', VDR.V_STRING, None, desc=""" + If specified the workflow share directory will be + created in ``/cylc-run//share`` + and a symbolic link will be created from + ``<$HOME/cylc-run//share``. If not specified + the workflow share directory will be created in + ``$HOME/cylc-run//share``. + """) + Conf('share/cycle', VDR.V_STRING, None, desc=""" + If specified the workflow share/cycle directory + will be created in + ``/cylc-run//share/cycle`` + and a symbolic link will be created from + ``$HOME/cylc-run//share/cycle``. If not + specified the workflow share/cycle directory will be + created in ``$HOME/cylc-run//share/cycle``. + """) + Conf('work', VDR.V_STRING, None, desc=""" + If specified the workflow work directory will be created in + ``/cylc-run//work`` and a symbolic + link will be created from + ``$HOME/cylc-run//work``. If not specified + the workflow work directory will be created in + ``$HOME/cylc-run//work``. + """) with Conf('editors', desc=''' Choose your favourite text editor for editing workflow configurations. @@ -711,60 +761,6 @@ with Conf('platform groups'): # noqa: SIM117 (keep same format) with Conf(''): Conf('platforms', VDR.V_STRING_LIST) - # Symlink Dirs - with Conf('symlink dirs', # noqa: SIM117 (keep same format) - desc=""" - Define directories to be moved, symlinks from the the original - ``$HOME/cylc-run`` directories will be created. - """): - with Conf(''): - Conf('run', VDR.V_STRING, None, desc=""" - Specifies the directory where the workflow run directories are - created. If specified, the workflow run directory will be - created in ``/cylc-run/`` and a - symbolic link will be created from - ``$HOME/cylc-run/``. - If not specified the workflow run directory will be created in - ``$HOME/cylc-run/``. - All the workflow files and the ``.service`` directory get - installed into this directory. - """) - Conf('log', VDR.V_STRING, None, desc=""" - Specifies the directory where log directories are created. If - specified the workflow log directory will be created in - ``/cylc-run//log`` and a symbolic link - will be created from ``$HOME/cylc-run//log``. If - not specified the workflow log directory will be created in - ``$HOME/cylc-run//log``. - """) - Conf('share', VDR.V_STRING, None, desc=""" - Specifies the directory where share directories are created. - If specified the workflow share directory will be created in - ``/cylc-run//share`` and a symbolic - link will be created from - ``<$HOME/cylc-run//share``. If not specified the - workflow share directory will be created in - ``$HOME/cylc-run//share``. - """) - Conf('share/cycle', VDR.V_STRING, None, desc=""" - Specifies the directory where share/cycle directories are - created. If specified the workflow share/cycle directory will - be created in - ``/cylc-run//share/cycle`` and - a symbolic link will be created from - ``$HOME/cylc-run//share/cycle``. If not - specified the workflow share/cycle directory will be created in - ``$HOME/cylc-run//share/cycle``. - """) - Conf('work', VDR.V_STRING, None, desc=""" - Specifies the directory where work directories are created. - If specified the workflow work directory will be created in - ``/cylc-run//work`` and a symbolic - link will be created from - ``$HOME/cylc-run//work``. If not specified the - workflow work directory will be created in - ``$HOME/cylc-run//work``. - """) # task with Conf('task events', desc=''' Global site/user defaults for diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py index cd8827943d9..4191ccfe6e9 100644 --- a/cylc/flow/cfgspec/workflow.py +++ b/cylc/flow/cfgspec/workflow.py @@ -788,7 +788,7 @@ The top level share and work directory location can be changed (e.g. to a large data area) by a global config setting (see - :cylc:conf:`global.cylc[symlink dirs]`). + :cylc:conf:`global.cylc[install][symlink dirs]`). .. note:: diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py index 91b06d720ca..e3c3ff46c23 100644 --- a/cylc/flow/pathutil.py +++ b/cylc/flow/pathutil.py @@ -19,7 +19,7 @@ from pathlib import Path import re from shutil import rmtree -from typing import Dict, Iterable, Set, Union +from typing import Dict, Iterable, Set, Union, Optional, Any from cylc.flow import LOG from cylc.flow.cfgspec.glbl_cfg import glbl_cfg @@ -28,7 +28,6 @@ ) from cylc.flow.platforms import get_localhost_install_target - # Note: do not import this elsewhere, as it might bypass unit test # monkeypatching: _CYLC_RUN_DIR = os.path.join('$HOME', 'cylc-run') @@ -131,22 +130,29 @@ def make_workflow_run_tree(workflow): def make_localhost_symlinks( - rund: Union[Path, str], named_sub_dir: str + rund: Union[Path, str], + named_sub_dir: str, + symlink_conf: Optional[Dict[str, Dict[str, str]]] = None ) -> Dict[str, Union[Path, str]]: """Creates symlinks for any configured symlink dirs from glbl_cfg. Args: rund: the entire run directory path named_sub_dir: e.g flow_name/run1 + symlink_conf: Symlinks dirs configuration passed from cli Returns: Dictionary of symlinks with sources as keys and destinations as values: ``{source: destination}`` """ - dirs_to_symlink = get_dirs_to_symlink( - get_localhost_install_target(), named_sub_dir) symlinks_created = {} + dirs_to_symlink = get_dirs_to_symlink( + get_localhost_install_target(), + named_sub_dir, symlink_conf=symlink_conf + ) for key, value in dirs_to_symlink.items(): + if value is None: + continue if key == 'run': symlink_path = rund else: @@ -165,21 +171,34 @@ def make_localhost_symlinks( return symlinks_created -def get_dirs_to_symlink(install_target: str, flow_name: str) -> Dict[str, str]: - """Returns dictionary of directories to symlink from glbcfg. +def get_dirs_to_symlink( + install_target: str, + flow_name: str, + symlink_conf: Optional[Dict[str, Dict[str, Any]]] = None +) -> Dict[str, Any]: + """Returns dictionary of directories to symlink. Note the paths should remain unexpanded, to be expanded on the remote. - """ - dirs_to_symlink: Dict[str, str] = {} - symlink_conf = glbl_cfg().get(['symlink dirs']) + Args: + install_target: Symlinks to be created on this install target + flow_name: full name of the run, e.g. myflow/run1 + symlink_conf: Symlink dirs, if sent on the cli. + Defaults to None, in which case global config symlink dirs will + be applied. + Returns: + dirs_to_symlink: [directory: symlink_path] + """ + dirs_to_symlink: Dict[str, Any] = {} + if symlink_conf is None: + symlink_conf = glbl_cfg().get(['install', 'symlink dirs']) if install_target not in symlink_conf.keys(): return dirs_to_symlink base_dir = symlink_conf[install_target]['run'] if base_dir: dirs_to_symlink['run'] = os.path.join(base_dir, 'cylc-run', flow_name) for dir_ in ['log', 'share', 'share/cycle', 'work']: - link = symlink_conf[install_target][dir_] + link = symlink_conf[install_target].get(dir_, None) if (not link) or link == base_dir: continue dirs_to_symlink[dir_] = os.path.join(link, 'cylc-run', flow_name, dir_) @@ -196,7 +215,9 @@ def make_symlink(path: Union[Path, str], target: Union[Path, str]) -> bool: path = Path(path) target = Path(target) if path.exists(): - if path.is_symlink() and path.samefile(target): + # note all three checks are needed here due to case where user has set + # their own symlink which does not match the global config set one. + if path.is_symlink() and target.exists() and path.samefile(target): # correct symlink already exists return False # symlink name is in use by a physical file or directory @@ -213,10 +234,11 @@ def make_symlink(path: Union[Path, str], target: Union[Path, str]) -> bool: raise WorkflowFilesError( f"Error when symlinking. Failed to unlink bad symlink {path}.") target.mkdir(parents=True, exist_ok=True) + + # This is needed in case share and share/cycle have the same symlink dir: if path.exists(): - # Trying to link to itself; no symlink needed - # (e.g. path's parent is symlink to target's parent) return False + path.parent.mkdir(parents=True, exist_ok=True) try: path.symlink_to(target) diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py index c139292b795..f06f444b91f 100755 --- a/cylc/flow/scripts/install.py +++ b/cylc/flow/scripts/install.py @@ -69,13 +69,13 @@ """ -from typing import Optional, TYPE_CHECKING +from typing import Optional, TYPE_CHECKING, Dict, Any from cylc.flow import iter_entry_points from cylc.flow.exceptions import PluginError from cylc.flow.option_parsers import CylcOptionParser as COP from cylc.flow.workflow_files import ( - install_workflow, search_install_source_dirs + install_workflow, search_install_source_dirs, parse_cli_sym_dirs ) from cylc.flow.terminal import cli_function @@ -150,6 +150,18 @@ def get_option_parser(): default=None, dest="source") + parser.add_option( + "--symlink-dirs", + help=( + "Enter a list, in the form 'log=path/to/store, share = $...'" + ". Use this option to override local symlinks for directories run," + " log, work, share, share/cycle, as configured in global.cylc. " + "Enter an empty list \"\" to skip making localhost symlink dirs." + ), + action="store", + dest="symlink_dirs" + ) + parser.add_option( "--run-name", help="Name the run.", @@ -165,13 +177,6 @@ def get_option_parser(): default=False, dest="no_run_name") - parser.add_option( - "--no-symlink-dirs", - help="Use this option to override creating default local symlinks.", - action="store_true", - default=False, - dest="no_symlinks") - parser = add_cylc_rose_options(parser) return parser @@ -214,13 +219,17 @@ def install( entry_point.name, exc ) from None - + cli_symdirs: Optional[Dict[str, Dict[str, Any]]] = None + if opts.symlink_dirs == '': + cli_symdirs = {} + elif opts.symlink_dirs: + cli_symdirs = parse_cli_sym_dirs(opts.symlink_dirs) source_dir, rundir, _flow_name = install_workflow( flow_name=flow_name, source=source, run_name=opts.run_name, no_run_name=opts.no_run_name, - no_symlinks=opts.no_symlinks + cli_symlink_dirs=cli_symdirs ) for entry_point in iter_entry_points( diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py index a3078122354..901cb5373bb 100644 --- a/cylc/flow/workflow_files.py +++ b/cylc/flow/workflow_files.py @@ -43,8 +43,9 @@ PlatformLookupError, ServiceFileError, TaskRemoteMgmtError, - WorkflowFilesError, - handle_rmtree_err + handle_rmtree_err, + UserInputError, + WorkflowFilesError ) from cylc.flow.pathutil import ( expand_path, @@ -161,7 +162,7 @@ def __init__(self, key_type, key_owner, full_key_path=None, class WorkflowFiles: - """Files and directories located in the workflow directory.""" + """Names of files and directories located in the workflow directory.""" FLOW_FILE = 'flow.cylc' """The workflow configuration file.""" @@ -184,6 +185,9 @@ class WorkflowFiles: WORK_DIR = 'work' """Workflow work directory.""" + RUN_DIR = 'run' + """Workflow run directory.""" + class Service: """The directory containing Cylc system files.""" @@ -224,7 +228,7 @@ class Install: SHARE_CYCLE_DIR, SHARE_DIR, LOG_DIR, WORK_DIR, '' ]) """The paths of the symlink dirs that may be set in - global.cylc[symlink dirs], relative to the run dir + global.cylc[install][symlink dirs], relative to the run dir ('' represents the run dir).""" @@ -578,26 +582,34 @@ def register( # flow.cylc must exist so we can detect accidentally reversed args. source = os.path.abspath(source) check_flow_file(source, symlink_suiterc=True, logger=None) - symlinks_created = make_localhost_symlinks( - get_workflow_run_dir(flow_name), flow_name) - if bool(symlinks_created): - for src, dst in symlinks_created.items(): - LOG.info(f"Symlink created from {src} to {dst}") + if not is_installed(get_workflow_run_dir(flow_name)): + symlinks_created = make_localhost_symlinks( + get_workflow_run_dir(flow_name), flow_name) + if symlinks_created: + for src, dst in symlinks_created.items(): + LOG.info(f"Symlink created from {src} to {dst}") # Create service dir if necessary. srv_d = get_workflow_srv_dir(flow_name) os.makedirs(srv_d, exist_ok=True) return flow_name -def is_installed(path: Union[Path, str]) -> bool: +def is_installed(rund: Union[Path, str]) -> bool: """Check to see if the path sent contains installed flow. - Checks for valid _cylc-install directory in current folder and checks - source link exists. + Checks for valid _cylc-install directory in the two possible locations in + relation to the run directory. + + Args: + rund: run directory path to check + + Returns: + bool: True if rund belongs to an installed workflow """ - cylc_install_folder = Path(path, WorkflowFiles.Install.DIRNAME) - source = Path(cylc_install_folder, WorkflowFiles.Install.SOURCE) - return cylc_install_folder.is_dir() and source.is_symlink() + rund = Path(rund) + cylc_install_dir = Path(rund, WorkflowFiles.Install.DIRNAME) + alt_cylc_install_dir = Path(rund.parent, WorkflowFiles.Install.DIRNAME) + return cylc_install_dir.is_dir() or alt_cylc_install_dir.is_dir() def _clean_check(reg: str, run_dir: Path) -> None: @@ -1243,7 +1255,7 @@ def install_workflow( source: Optional[Union[Path, str]] = None, run_name: Optional[str] = None, no_run_name: bool = False, - no_symlinks: bool = False + cli_symlink_dirs: Optional[Dict[str, Dict[str, Any]]] = None ) -> Tuple[Path, Path, str]: """Install a workflow, or renew its installation. @@ -1259,7 +1271,7 @@ def install_workflow( rundir: for overriding the default cylc-run directory. no_run_name: Flag as True to install workflow into ~/cylc-run/ - no_symlinks: Flag as True to skip making localhost symlink dirs + cli_symlink_dirs: Symlink dirs, if entered on the cli. Return: source: The source directory. @@ -1299,10 +1311,10 @@ def install_workflow( named_run = os.path.join(named_run, run_name) elif run_num: named_run = os.path.join(named_run, f'run{run_num}') - if not no_symlinks: - symlinks_created = make_localhost_symlinks(rundir, named_run) + symlinks_created = make_localhost_symlinks( + rundir, named_run, symlink_conf=cli_symlink_dirs) install_log = _get_logger(rundir, 'cylc-install') - if not no_symlinks and bool(symlinks_created) is True: + if symlinks_created: for src, dst in symlinks_created.items(): install_log.info(f"Symlink created from {src} to {dst}") try: @@ -1491,6 +1503,54 @@ def validate_source_dir(source, flow_name): check_flow_file(source, logger=None) +def parse_cli_sym_dirs(symlink_dirs: str) -> Dict[str, Dict[str, Any]]: + """Converts command line entered symlink dirs to a dictionary. + + Args: + symlink_dirs: As entered by user on cli, + e.g. "log=$DIR, share=$DIR2". + + Raises: + WorkflowFilesError: If directory to be symlinked is not in permitted + dirs: run, log, share, work, share/cycle + + Returns: + dict: In the same form as would be returned by global config. + e.g. {'localhost': {'log': '$DIR', + 'share': '$DIR2' + } + } + """ + # Ensures the same nested dict format which is returned by the glb cfg + symdict: Dict[str, Dict[str, Any]] = {'localhost': {'run': None}} + if symlink_dirs == "": + return symdict + symlist = symlink_dirs.strip(',').split(',') + possible_symlink_dirs = set(WorkflowFiles.SYMLINK_DIRS.union( + {WorkflowFiles.RUN_DIR}) + ) + possible_symlink_dirs.remove('') + for pair in symlist: + try: + key, val = pair.split("=") + key = key.strip() + except ValueError: + raise UserInputError( + 'There is an error in --symlink-dirs option:' + f' {pair}. Try entering option in the form ' + '--symlink-dirs=\'log=$DIR, share=$DIR2, ...\'' + ) + if key not in possible_symlink_dirs: + dirs = ', '.join(possible_symlink_dirs) + raise UserInputError( + f"{key} not a valid entry for --symlink-dirs. " + f"Configurable symlink dirs are: {dirs}" + ) + symdict['localhost'][key] = val.strip() or None + + return symdict + + def unlink_runN(path: Union[Path, str]) -> bool: """Remove symlink runN if it exists. diff --git a/tests/functional/cylc-clean/00-basic.t b/tests/functional/cylc-clean/00-basic.t index 7615a9d6804..6916c9e4f91 100644 --- a/tests/functional/cylc-clean/00-basic.t +++ b/tests/functional/cylc-clean/00-basic.t @@ -28,13 +28,14 @@ SYM_NAME="$(mktemp -u)" SYM_NAME="${SYM_NAME##*tmp.}" create_test_global_config "" " -[symlink dirs] - [[localhost]] - run = ${TEST_DIR}/${SYM_NAME}/run - log = ${TEST_DIR}/${SYM_NAME}/log - share = ${TEST_DIR}/${SYM_NAME}/share - share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle - work = ${TEST_DIR}/${SYM_NAME}/work +[install] + [[symlink dirs]] + [[[localhost]]] + run = ${TEST_DIR}/${SYM_NAME}/run + log = ${TEST_DIR}/${SYM_NAME}/log + share = ${TEST_DIR}/${SYM_NAME}/share + share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle + work = ${TEST_DIR}/${SYM_NAME}/work " init_workflow "${TEST_NAME_BASE}" << '__FLOW__' [scheduler] diff --git a/tests/functional/cylc-clean/01-remote.t b/tests/functional/cylc-clean/01-remote.t index 4ac43a4d126..64a01caffcb 100644 --- a/tests/functional/cylc-clean/01-remote.t +++ b/tests/functional/cylc-clean/01-remote.t @@ -32,13 +32,14 @@ SYM_NAME="$(mktemp -u)" SYM_NAME="${SYM_NAME##*tmp.}" create_test_global_config "" " -[symlink dirs] - [[${CYLC_TEST_INSTALL_TARGET}]] - run = ${TEST_DIR}/${SYM_NAME}/run - log = ${TEST_DIR}/${SYM_NAME}/other - share = ${TEST_DIR}/${SYM_NAME}/other - share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle - work = ${TEST_DIR}/${SYM_NAME}/other +[install] + [[symlink dirs]] + [[[${CYLC_TEST_INSTALL_TARGET}]]] + run = ${TEST_DIR}/${SYM_NAME}/run + log = ${TEST_DIR}/${SYM_NAME}/other + share = ${TEST_DIR}/${SYM_NAME}/other + share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle + work = ${TEST_DIR}/${SYM_NAME}/other " init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduler] diff --git a/tests/functional/cylc-clean/02-targeted.t b/tests/functional/cylc-clean/02-targeted.t index 08a7c417b5d..d50a189fd5e 100644 --- a/tests/functional/cylc-clean/02-targeted.t +++ b/tests/functional/cylc-clean/02-targeted.t @@ -28,14 +28,15 @@ SYM_NAME="$(mktemp -u)" SYM_NAME="sym-${SYM_NAME##*tmp.}" create_test_global_config "" " -[symlink dirs] - [[localhost]] - run = ${TEST_DIR}/${SYM_NAME}/run - log = ${TEST_DIR}/${SYM_NAME}/other - share = ${TEST_DIR}/${SYM_NAME}/other - work = ${TEST_DIR}/${SYM_NAME}/other - # Need to override any symlink dirs set in global-tests.cylc: - share/cycle = +[install] + [[symlink dirs]] + [[[localhost]]] + run = ${TEST_DIR}/${SYM_NAME}/run + log = ${TEST_DIR}/${SYM_NAME}/other + share = ${TEST_DIR}/${SYM_NAME}/other + work = ${TEST_DIR}/${SYM_NAME}/other + # Need to override any symlink dirs set in global-tests.cylc: + share/cycle = " init_workflow "${TEST_NAME_BASE}" << __FLOW__ [scheduler] diff --git a/tests/functional/cylc-get-site-config/04-homeless.t b/tests/functional/cylc-get-site-config/04-homeless.t index 2dc2aacde63..0afa2abe062 100644 --- a/tests/functional/cylc-get-site-config/04-homeless.t +++ b/tests/functional/cylc-get-site-config/04-homeless.t @@ -22,14 +22,15 @@ set_test_number 3 # shellcheck disable=SC2016 create_test_global_config '' ' -[symlink dirs] - [[localhost]] - run = $HOME/dr-malcolm +[install] + [[symlink dirs]] + [[[localhost]]] + run = $HOME/dr-malcolm ' run_ok "${TEST_NAME_BASE}" \ env -u HOME \ - cylc config --item='[symlink dirs][localhost]run' + cylc config --item='[install][symlink dirs][localhost]run' cmp_ok "${TEST_NAME_BASE}.stdout" <<<"\$HOME/dr-malcolm" cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null' exit diff --git a/tests/functional/cylc-install/01-symlinks.t b/tests/functional/cylc-install/01-symlinks.t index 9a6a5cb9328..0abb6720b3d 100644 --- a/tests/functional/cylc-install/01-symlinks.t +++ b/tests/functional/cylc-install/01-symlinks.t @@ -24,11 +24,12 @@ if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"' fi -set_test_number 14 +set_test_number 30 create_test_global_config "" " -[symlink dirs] - [[localhost]] +[install] +[[symlink dirs]] + [[[localhost]]] run = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_run_dir share = \$TMPDIR/\$USER/test_cylc_symlink/ log = \$TMPDIR/\$USER/test_cylc_symlink/ @@ -43,76 +44,143 @@ run_ok "${TEST_NAME}" cylc install --flow-name="${RND_WORKFLOW_NAME}" --director contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ +WORKFLOW_RUN_DIR="$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1" +TEST_SYM="${TEST_NAME_BASE}-run-glblcfg" +run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}")" \ + = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_run_dir/cylc-run/${RND_WORKFLOW_NAME}/run1" -TEST_SYM="${TEST_NAME_BASE}-run-symlink-exists-ok" +TEST_SYM="${TEST_NAME_BASE}-share-cycle-glblcfg" +run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \ + = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle" -if [[ $(readlink "$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1") == \ - "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_run_dir/cylc-run/${RND_WORKFLOW_NAME}/run1" ]]; then - ok "$TEST_SYM" -else - fail "$TEST_SYM" -fi +for DIR in 'work' 'share' 'log'; do + TEST_SYM="${TEST_NAME_BASE}-${DIR}-glbcfg" + run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \ + = "$TMPDIR/${USER}/test_cylc_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}" +done +rm -rf "${TMPDIR}/${USER}/test_cylc_symlink/" +purge_rnd_workflow +# test cli --symlink-dirs overrides the glblcfg +SYMDIR=${TMPDIR}/${USER}/test_cylc_cli_symlink/ +TEST_NAME="${TEST_NAME_BASE}-cli-opt-install" +make_rnd_workflow +run_ok "${TEST_NAME}" cylc install --flow-name="${RND_WORKFLOW_NAME}" \ +--directory="${RND_WORKFLOW_SOURCE}" \ +--symlink-dirs="run= ${SYMDIR}cylctb_tmp_run_dir, log=${SYMDIR}, share=${SYMDIR}, \ +work = ${SYMDIR}" +contains_ok "${TEST_NAME}.stdout" <<__OUT__ +INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} +__OUT__ +WORKFLOW_RUN_DIR="$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1" + +TEST_SYM="${TEST_NAME_BASE}-run-cli" +run_ok "$TEST_SYM" test "$(readlink "${WORKFLOW_RUN_DIR}")" \ + = "$TMPDIR/${USER}/test_cylc_cli_symlink/cylctb_tmp_run_dir/cylc-run/${RND_WORKFLOW_NAME}/run1" -TEST_SYM="${TEST_NAME_BASE}-share/cycle-symlink-exists-ok" -if [[ $(readlink "$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle") == \ -"$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle" ]]; then - ok "$TEST_SYM" -else - fail "$TEST_SYM" -fi for DIR in 'work' 'share' 'log'; do - TEST_SYM="${TEST_NAME_BASE}-${DIR}-symlink-exists-ok" - if [[ $(readlink "$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}") == \ - "$TMPDIR/${USER}/test_cylc_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}" ]]; then - ok "$TEST_SYM" - else - fail "$TEST_SYM" - fi + TEST_SYM="${TEST_NAME_BASE}-${DIR}-cli" + run_ok "$TEST_SYM" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \ + = "${TMPDIR}/${USER}/test_cylc_cli_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}" done -rm -rf "${TMPDIR}/${USER}/test_cylc_symlink/" + +INSTALL_LOG="$(find "${WORKFLOW_RUN_DIR}/log/install" -type f -name '*.log')" + +for DIR in 'work' 'share' 'log'; do + grep_ok "${TMPDIR}/${USER}/test_cylc_cli_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}" "${INSTALL_LOG}" +done + +# test cylc play symlinks after cli opts (mapping to different directories) + +pushd "${WORKFLOW_RUN_DIR}" || exit 1 +cat > 'flow.cylc' << __FLOW__ +[scheduling] + [[graph]] + R1 = foo +[runtime] + [[foo]] + script = true +__FLOW__ + +popd || exit 1 + +run_ok "${TEST_NAME_BASE}-play" cylc play "${RND_WORKFLOW_NAME}/runN" --debug --no-detach + +# test ensure symlinks, not in cli install are not created from glbl cfg. +TEST_SYM="${TEST_NAME_BASE}-share-cycle-cli" +run_fail "$TEST_SYM" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \ += "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle" + +rm -rf "${TMPDIR}/${USER}/test_cylc_cli_symlink/" purge_rnd_workflow +# test no symlinks created with --symlink-dirs="" -# Test "cylc install" --no-symlink-dirs -TEST_NAME="${TEST_NAME_BASE}-no-symlinks-created" +TEST_NAME="${TEST_NAME_BASE}-no-sym-dirs-cli" make_rnd_workflow -run_ok "${TEST_NAME}" cylc install --flow-name="${RND_WORKFLOW_NAME}" --no-symlink-dirs --directory="${RND_WORKFLOW_SOURCE}" +run_ok "${TEST_NAME}" cylc install --flow-name="${RND_WORKFLOW_NAME}" \ +--directory="${RND_WORKFLOW_SOURCE}" --symlink-dirs="" contains_ok "${TEST_NAME}.stdout" <<__OUT__ INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} __OUT__ +WORKFLOW_RUN_DIR="$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1" -TEST_SYM="${TEST_NAME_BASE}-run-symlink-exists-ok" - -if [[ $(readlink "$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1") == \ - "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_run_dir/cylc-run/${RND_WORKFLOW_NAME}/run1" ]]; then - fail "$TEST_SYM" -else - ok "$TEST_SYM" -fi - +TEST_SYM="${TEST_NAME}-run" +run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}")" \ + = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_run_dir/cylc-run/${RND_WORKFLOW_NAME}/run1" - -TEST_SYM="${TEST_NAME_BASE}-share/cycle-symlink-not-exists-ok" -if [[ $(readlink "$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle") == \ -"$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/share/cycle" ]]; then - fail "$TEST_SYM" -else - ok "$TEST_SYM" -fi +TEST_SYM="${TEST_NAME}-share-cycle" +run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \ + = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle" for DIR in 'work' 'share' 'log'; do - TEST_SYM="${TEST_NAME_BASE}-${DIR}-symlink-not-exists-ok" - if [[ $(readlink "$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}") == \ - "$TMPDIR/${USER}/test_cylc_symlink/cylc-run/${RND_WORKFLOW_NAME}/${DIR}" ]]; then - fail "$TEST_SYM" - else - ok "$TEST_SYM" - fi + TEST_SYM="${TEST_NAME}-${DIR}" + run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \ + = "$TMPDIR/${USER}/test_cylc_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/${DIR}" done -rm -rf "${TMPDIR}/${USER}/test_cylc_symlink/" + +pushd "${WORKFLOW_RUN_DIR}" || exit 1 +cat > 'flow.cylc' << __FLOW__ +[scheduling] + [[graph]] + R1 = foo +[runtime] + [[foo]] + script = true +__FLOW__ + +popd || exit 1 + +run_ok "${TEST_NAME_BASE}-play" cylc play "${RND_WORKFLOW_NAME}/runN" --debug --no-detach +# test ensure localhost symlink dirs skipped for installed workflows. +TEST_SYM="${TEST_NAME_BASE}-installed-workflow-skips-symdirs" +run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}")" \ + = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_run_dir/cylc-run/${RND_WORKFLOW_NAME}/run1" +rm -rf "${TMPDIR}/${USER}/test_cylc_cli_symlink/" purge_rnd_workflow + + +# test share and share/cycle same symlinks don't error +SYMDIR=${TMPDIR}/${USER}/test_cylc_cli_symlink/ + +TEST_NAME="${TEST_NAME_BASE}-share-share-cycle-same-dirs" +make_rnd_workflow +# check install runs without failure +run_ok "${TEST_NAME}" cylc install --flow-name="${RND_WORKFLOW_NAME}" \ +--directory="${RND_WORKFLOW_SOURCE}" \ +--symlink-dirs="share/cycle=${SYMDIR}, share=${SYMDIR}" +contains_ok "${TEST_NAME}.stdout" <<__OUT__ +INSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE} +__OUT__ +WORKFLOW_RUN_DIR="$HOME/cylc-run/${RND_WORKFLOW_NAME}/run1" + +TEST_SYM="${TEST_NAME_BASE}-share-cli" +run_ok "$TEST_SYM" test "$(readlink "${WORKFLOW_RUN_DIR}/share")" \ + = "${TMPDIR}/${USER}/test_cylc_cli_symlink/cylc-run/${RND_WORKFLOW_NAME}/run1/share" + +rm -rf "${TMPDIR}/${USER}/test_cylc_cli_symlink/" +purge_rnd_workflow \ No newline at end of file diff --git a/tests/functional/remote/04-symlink-dirs.t b/tests/functional/remote/04-symlink-dirs.t index b18ede331f9..9005f5419c5 100644 --- a/tests/functional/remote/04-symlink-dirs.t +++ b/tests/functional/remote/04-symlink-dirs.t @@ -27,19 +27,20 @@ fi set_test_number 12 create_test_global_config "" " -[symlink dirs] - [[localhost]] - run = \$TMPDIR/\$USER/cylctb_tmp_run_dir - share = \$TMPDIR/\$USER - log = \$TMPDIR/\$USER - share/cycle = \$TMPDIR/\$USER/cylctb_tmp_share_dir - work = \$TMPDIR/\$USER - [[$CYLC_TEST_INSTALL_TARGET]] - run = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_run_dir - share = \$TMPDIR/\$USER/test_cylc_symlink/ - log = \$TMPDIR/\$USER/test_cylc_symlink/ - share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_share_dir - work = \$TMPDIR/\$USER/test_cylc_symlink/ +[install] + [[symlink dirs]] + [[[localhost]]] + run = \$TMPDIR/\$USER/cylctb_tmp_run_dir + share = \$TMPDIR/\$USER + log = \$TMPDIR/\$USER + share/cycle = \$TMPDIR/\$USER/cylctb_tmp_share_dir + work = \$TMPDIR/\$USER + [[[$CYLC_TEST_INSTALL_TARGET]]] + run = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_run_dir + share = \$TMPDIR/\$USER/test_cylc_symlink/ + log = \$TMPDIR/\$USER/test_cylc_symlink/ + share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_share_dir + work = \$TMPDIR/\$USER/test_cylc_symlink/ " install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index f50009a7e1d..d176e3f086e 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -63,8 +63,17 @@ def tmp_run_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): Args: reg: Workflow name. + installed: If True, make it look like the workflow was installed + using cylc install (creates _cylc-install dir). + named: If True and installed is True, the _cylc-install dir will + be created in the parent to make it look like this is a + named run. """ - def _tmp_run_dir(reg: Optional[str] = None) -> Path: + def _tmp_run_dir( + reg: Optional[str] = None, + installed: bool = False, + named: bool = False + ) -> Path: cylc_run_dir = tmp_path / 'cylc-run' cylc_run_dir.mkdir(exist_ok=True) monkeypatch.setattr('cylc.flow.pathutil._CYLC_RUN_DIR', cylc_run_dir) @@ -73,6 +82,16 @@ def _tmp_run_dir(reg: Optional[str] = None) -> Path: run_dir.mkdir(parents=True, exist_ok=True) (run_dir / WorkflowFiles.FLOW_FILE).touch(exist_ok=True) (run_dir / WorkflowFiles.Service.DIRNAME).mkdir(exist_ok=True) + if installed: + if named: + if len(Path(reg).parts) < 2: + raise ValueError("Named run requires two-level reg") + (run_dir.parent / WorkflowFiles.Install.DIRNAME).mkdir( + exist_ok=True) + else: + (run_dir / WorkflowFiles.Install.DIRNAME).mkdir( + exist_ok=True) + return run_dir return cylc_run_dir return _tmp_run_dir diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py index 3af92a833db..ea97ec9402c 100644 --- a/tests/unit/test_pathutil.py +++ b/tests/unit/test_pathutil.py @@ -174,8 +174,9 @@ def test_make_workflow_run_tree( [ pytest.param( # basic ''' - [symlink dirs] - [[the_matrix]] + [install] + [[symlink dirs]] + [[[the_matrix]]] run = $DEE work = $DAH log = $DUH @@ -193,8 +194,9 @@ def test_make_workflow_run_tree( ), pytest.param( # remove nested run symlinks ''' - [symlink dirs] - [[the_matrix]] + [install] + [[symlink dirs]] + [[[the_matrix]]] run = $DEE work = $DAH log = $DEE @@ -211,8 +213,9 @@ def test_make_workflow_run_tree( ), pytest.param( # remove only nested run symlinks ''' - [symlink dirs] - [[the_matrix]] + [install] + [[symlink dirs]] + [[[the_matrix]]] run = $DOH log = $DEE share = $DEE @@ -226,8 +229,9 @@ def test_make_workflow_run_tree( ), pytest.param( # blank entries ''' - [symlink dirs] - [[the_matrix]] + [install] + [[symlink dirs]] + [[[the_matrix]]] run = log = "" share = diff --git a/tests/unit/test_workflow_files.py b/tests/unit/test_workflow_files.py index b2817be2373..650e2ae8c09 100644 --- a/tests/unit/test_workflow_files.py +++ b/tests/unit/test_workflow_files.py @@ -29,8 +29,10 @@ CylcError, ServiceFileError, TaskRemoteMgmtError, + UserInputError, WorkflowFilesError ) +from cylc.flow.option_parsers import Options from cylc.flow.pathutil import parse_rm_dirs from cylc.flow.scripts.clean import CleanOptions from cylc.flow.workflow_files import ( @@ -39,7 +41,10 @@ _remote_clean_cmd, check_flow_file, check_nested_run_dirs, + parse_cli_sym_dirs, get_symlink_dirs, + is_installed, + parse_cli_sym_dirs, get_workflow_source_dir, glob_in_run_dir, reinstall_workflow, @@ -1441,3 +1446,61 @@ def test_check_flow_file_symlink( log_msg = f'{log_msg}. Symlink created.' assert result == tmp_path.joinpath(expected_file) assert caplog.messages == [log_msg] + + +@pytest.mark.parametrize( + 'symlink_dirs, err_msg, expected', + [ + ('log=$shortbread, share= $bourbon,share/cycle= $digestive, ', + "There is an error in --symlink-dirs option:", + None + ), + ('log=$shortbread share= $bourbon share/cycle= $digestive ', + "There is an error in --symlink-dirs option:" + " log=$shortbread share= $bourbon share/cycle= $digestive . " + "Try entering option in the form --symlink-dirs=" + "'log=$DIR, share=$DIR2, ...'", + None + ), + ('run=$NICE, log= $Garibaldi, share/cycle=$RichTea', None, + {'localhost': { + 'run': '$NICE', + 'log': '$Garibaldi', + 'share/cycle': '$RichTea' + }} + ), + ('some_other_dir=$bourbon', + 'some_other_dir not a valid entry for --symlink-dirs', + {'some_other_dir': '£bourbon'} + ), + ] +) +def test_parse_cli_sym_dirs( + symlink_dirs: str, + err_msg: str, + expected: Dict[str, Dict[str, Any]] +): + """Test parse_cli_sym_dirs returns dict or correctly raises errors on cli + symlink dir options""" + if err_msg is not None: + with pytest.raises(UserInputError) as exc: + parse_cli_sym_dirs(symlink_dirs) + assert(err_msg) in str(exc) + + else: + actual = parse_cli_sym_dirs(symlink_dirs) + + assert actual == expected + + +@pytest.mark.parametrize( + 'reg, installed, named, expected', + [('reg1/run1', True, True, True), + ('reg2', True, False, True), + ('reg3', False, False, False)] +) +def test_is_installed(tmp_run_dir: Callable, reg, installed, named, expected): + """Test is_installed correctly identifies presence of _cylc-install dir""" + cylc_run_dir: Path = tmp_run_dir(reg, installed=installed, named=named) + actual = is_installed(cylc_run_dir) + assert actual == expected