From 36cdc2963e38f69bdecb655fd7bdc44134247311 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 19 Apr 2022 11:24:54 -0700 Subject: [PATCH] Schema2 (#742) * Initial migration to schema version 2 including config rename (#678) * Implement initial migration to schema version 2. * Require project-local config to be signac.rc (not .signacrc) and make searches stricter to match. * Standardize method for getting project config at a root. * Move config to .signac/config. * Fix import order. * Address PR comments. * Remove some unnecessary code. * Address final PR coments. * Use integer schema version numbers (#688) * Change schema versioning to use integer strings. * Switch from int strings to pure ints. * Update signac/contrib/migration/__init__.py Co-authored-by: Bradley Dice Co-authored-by: Bradley Dice * Remove project name from schema (#684) * Remove project id API. * Remove project name from config as part of migration. * Fix issues with config CLI and remove project from default cfg. * Address PR comments. * Change the str of a project to the str of its root directory. * Change Project constructor to use root directory (#706) * Change project constructor to accept a root directory instead of a config file. * Change Project repr. * Address easy PR comments. * Move internal signac files into .signac subdirectory (#708) * Move shell history. * Move sp_cache file. * Address PR comments. * Move discovery to separate functions. (#711) * Move discovery to separate functions. * Address first round of PR comments. * Address PR comments. * Apply suggestions. * Remove configurable workspace directory (#714) * Remove workspace configurability. * Implement workspace_dir migration. * Apply suggestions from code review Co-authored-by: Bradley Dice * Address remaining PR comments. * Update tests/test_project.py * Remove mention of configurability from project workspace docstring * Address PR comments. Co-authored-by: Bradley Dice Co-authored-by: Corwin Kerr * Update description of schema migration. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: Bradley Dice Co-authored-by: Corwin Kerr Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- signac/__main__.py | 117 ++++++---------- signac/common/config.py | 99 ++++++-------- signac/common/validate.py | 3 +- signac/contrib/migration/__init__.py | 35 ++--- signac/contrib/migration/v1_to_v2.py | 87 ++++++++++++ signac/contrib/project.py | 165 ++++++++--------------- signac/version.py | 2 +- tests/test_job.py | 5 +- tests/test_project.py | 195 +++++++++++++-------------- tests/test_shell.py | 23 ++-- 10 files changed, 344 insertions(+), 387 deletions(-) create mode 100644 signac/contrib/migration/v1_to_v2.py diff --git a/signac/__main__.py b/signac/__main__.py index fb6848e97..492cf8846 100644 --- a/signac/__main__.py +++ b/signac/__main__.py @@ -29,7 +29,7 @@ else: READLINE = True -from . import Project, get_project, init_project +from . import get_project, init_project from .common import config from .common.configobj import Section, flatten_errors from .contrib.filterparse import _add_prefix, parse_filter_arg @@ -73,9 +73,7 @@ SHELL_BANNER = """Python {python_version} signac {signac_version} 🎨 -Project:\t{project_id}{job_banner} -Root:\t\t{root_path} -Workspace:\t{workspace_path} +Project:\t{root_path}{job_banner} Size:\t\t{size} Interact with the project interface using the "project" or "pr" variable. @@ -94,6 +92,8 @@ ) +SHELL_HISTORY_FN = os.sep.join((".signac", "shell_history")) + warnings.simplefilter("default") @@ -354,7 +354,7 @@ def main_view(args): def main_init(args): """Handle init subcommand.""" - init_project(name=args.project_id, root=os.getcwd(), workspace=args.workspace) + init_project(name=args.project_id, root=os.getcwd()) _print_err("Initialized project.") @@ -372,8 +372,9 @@ def main_schema(args): def main_sync(args): """Handle sync subcommand.""" + # TODO: This function appears to be untested. # - # Valid provided argument combinations + # Validate provided argument combinations # if args.archive: args.recursive = True @@ -421,21 +422,8 @@ def _sig(st): try: destination = get_project(root=args.destination) except LookupError: - if args.allow_workspace: - destination = Project( - config={ - "project": os.path.relpath(args.destination), - "project_dir": args.destination, - "workspace_dir": ".", - } - ) - else: - _print_err( - "WARNING: The destination appears to not be a project path. " - "Use the '-w/--allow-workspace' option if you want to " - "synchronize to a workspace directory directly." - ) - raise + _print_err("WARNING: The destination does not appear to be a project path.") + raise selection = find_with_filter_or_none(args) if args.strategy: @@ -561,10 +549,8 @@ def _main_import_interactive(project, origin, args): banner=SHELL_BANNER_INTERACTIVE_IMPORT.format( python_version=sys.version, signac_version=__version__, - project_id=project.id, job_banner="", root_path=project.root_directory(), - workspace_path=project.workspace, size=len(project), origin=args.origin, ), @@ -730,26 +716,20 @@ def main_config_show(args): if args.local and args.globalcfg: raise ValueError("You can specify either -l/--local or -g/--global, not both.") elif args.local: - for fn in config.CONFIG_FILENAMES: - if os.path.isfile(fn): - if cfg is None: - cfg = config.read_config_file(fn) - else: - cfg.merge(config.read_config_file(fn)) + if os.path.isfile(config.PROJECT_CONFIG_FN): + cfg = config.read_config_file(config.PROJECT_CONFIG_FN) elif args.globalcfg: - cfg = config.read_config_file(config.FN_CONFIG) + cfg = config.read_config_file(config.USER_CONFIG_FN) else: - cfg = config.load_config() - if cfg is None: - if args.local and args.globalcfg: - mode = " local or global " - elif args.local: - mode = " local " + cfg = config.load_config(config._locate_config_dir(os.getcwd())) + if not cfg: + if args.local: + mode = "local" elif args.globalcfg: - mode = " global " + mode = "global" else: - mode = "" - _print_err(f"Did not find a{mode}configuration file.") + mode = "local or global" + _print_err(f"Did not find a {mode} configuration file.") return for key in args.key: for kt in key.split("."): @@ -769,29 +749,25 @@ def main_config_verify(args): if args.local and args.globalcfg: raise ValueError("You can specify either -l/--local or -g/--global, not both.") elif args.local: - for fn in config.CONFIG_FILENAMES: - if os.path.isfile(fn): - if cfg is None: - cfg = config.read_config_file(fn) - else: - cfg.merge(config.read_config_file(fn)) + if os.path.isfile(config.PROJECT_CONFIG_FN): + cfg = config.read_config_file(config.PROJECT_CONFIG_FN) elif args.globalcfg: - cfg = config.read_config_file(config.FN_CONFIG) + cfg = config.read_config_file(config.USER_CONFIG_FN) else: - cfg = config.load_config() - if cfg is None: - if args.local and args.globalcfg: - mode = " local or global " - elif args.local: - mode = " local " + cfg = config.load_config(config._locate_config_dir(os.getcwd())) + if not cfg: + if args.local: + mode = "local" elif args.globalcfg: - mode = " global " + mode = "global" else: - mode = "" - raise RuntimeWarning(f"Did not find a{mode}configuration file.") - if cfg.filename is not None: - _print_err(f"Verification of config file '{cfg.filename}'.") - verify_config(cfg) + mode = "local or global" + _print_err(f"Did not find a {mode} configuration file.") + return + else: + if cfg.filename is not None: + _print_err(f"Verification of config file '{cfg.filename}'.") + verify_config(cfg) def main_config_set(args): @@ -802,11 +778,10 @@ def main_config_set(args): if args.local and args.globalcfg: raise ValueError("You can specify either -l/--local or -g/--global, not both.") elif args.local: - for fn_config in config.CONFIG_FILENAMES: - if os.path.isfile(fn_config): - break + if os.path.isfile(config.PROJECT_CONFIG_FN): + fn_config = config.PROJECT_CONFIG_FN elif args.globalcfg: - fn_config = config.FN_CONFIG + fn_config = config.USER_CONFIG_FN else: raise ValueError( "You need to specify either -l/--local or -g/--global " @@ -886,7 +861,7 @@ def jobs(): else: # interactive if READLINE: if "PyPy" not in platform.python_implementation(): - fn_hist = project.fn(".signac_shell_history") + fn_hist = project.fn(SHELL_HISTORY_FN) try: readline.read_history_file(fn_hist) readline.set_history_length(1000) @@ -915,10 +890,8 @@ def write_history_file(): banner=SHELL_BANNER.format( python_version=sys.version, signac_version=__version__, - project_id=project.id, job_banner=f"\nJob:\t\t{job.id}" if job is not None else "", root_path=project.root_directory(), - workspace_path=project.workspace, size=len(project), ), ) @@ -947,13 +920,6 @@ def main(): parser_init = subparsers.add_parser("init") parser_init.add_argument("project_id", nargs="?", help=argparse.SUPPRESS) - parser_init.add_argument( - "-w", - "--workspace", - type=str, - default="workspace", - help="The path to the workspace directory.", - ) parser_init.set_defaults(func=main_init) parser_project = subparsers.add_parser("project") @@ -1499,13 +1465,6 @@ def main(): "--no-keys", action="store_true", help="Never overwrite any conflicting keys." ) - parser_sync.add_argument( - "-w", - "--allow-workspace", - action="store_true", - help="Allow the specification of a workspace (instead of a project) directory " - "as the destination path.", - ) parser_sync.add_argument( "--force", action="store_true", help="Ignore all warnings, just synchronize." ) diff --git a/signac/common/config.py b/signac/common/config.py index 614e6506e..4918f412d 100644 --- a/signac/common/config.py +++ b/signac/common/config.py @@ -12,57 +12,51 @@ logger = logging.getLogger(__name__) -DEFAULT_FILENAME = ".signacrc" -CONFIG_FILENAMES = [DEFAULT_FILENAME, "signac.rc"] -HOME = os.path.expanduser("~") -CONFIG_PATH = [HOME] -FN_CONFIG = os.path.expanduser("~/.signacrc") +PROJECT_CONFIG_FN = os.path.join(".signac", "config") +USER_CONFIG_FN = os.path.expanduser(os.path.join("~", ".signacrc")) +# TODO: Consider making this entire module internal and removing all its +# functions from the public API. -def _search_local(root): - for fn in CONFIG_FILENAMES: - fn_ = os.path.abspath(os.path.join(root, fn)) - if os.path.isfile(fn_): - yield fn_ +def _get_project_config_fn(root): + return os.path.abspath(os.path.join(root, PROJECT_CONFIG_FN)) -def _search_tree(root=None): - """Locates signac configuration files in a directory hierarchy. + +def _locate_config_dir(search_path): + """Locates root directory containing a signac configuration file in a directory hierarchy. Parameters ---------- root : str - Path to search. Uses ``os.getcwd()`` if None (Default value = None). + Starting path to search. + Returns + -------- + str or None + The root directory containing the configuration file if one is found, otherwise None. """ - if root is None: - root = os.getcwd() + root = os.path.abspath(search_path) while True: - yield from _search_local(root) - up = os.path.abspath(os.path.join(root, "..")) + if os.path.isfile(_get_project_config_fn(root)): + return root + # TODO: Could use the walrus operator here when we completely drop + # Python 3.7 support if we like the operator. + up = os.path.dirname(root) if up == root: - msg = "Reached filesystem root." - logger.debug(msg) - return + logger.debug("Reached filesystem root, no config found.") + return None else: root = up -def _search_standard_dirs(): - """Locates signac configuration files in standard directories.""" - for path in CONFIG_PATH: - yield from _search_local(path) - - -def read_config_file(filename, configspec=None, *args, **kwargs): +def read_config_file(filename): """Read a configuration file. Parameters ---------- filename : str The path to the file to read. - configspec : List[str], optional - The key-value pairs supported in the config. Returns -------- @@ -70,15 +64,15 @@ def read_config_file(filename, configspec=None, *args, **kwargs): The config contained in the file. """ logger.debug(f"Reading config file '{filename}'.") - if configspec is None: - configspec = cfg.split("\n") + configspec = cfg.split("\n") try: - config = Config(filename, configspec=configspec, *args, **kwargs) + config = Config(filename, configspec=configspec) except (OSError, ConfigObjError) as error: - msg = "Failed to read configuration file '{}':\n{}" - raise ConfigError(msg.format(filename, error)) + raise ConfigError(f"Failed to read configuration file '{filename}':\n{error}") verification = config.verify() if verification is not True: + # TODO: In the future this should raise an error, not just a + # debug-level logging notice. logger.debug( "Config file '{}' may contain invalid values.".format( os.path.abspath(filename) @@ -87,42 +81,33 @@ def read_config_file(filename, configspec=None, *args, **kwargs): return config -def load_config(root=None, local=False): - """Load configuration, searching upward from a root path if desired. +def load_config(root=None): + """Load configuration from a project directory. Parameters ---------- root : str - The path from which to begin searching for config files. - local : bool, optional - If ``True``, only search in the provided directory and do not traverse - upwards through the filesystem (Default value: False). + The project path to pull project-local configuration data from. Returns -------- :class:`Config` - The composite configuration including both local and global config data - if requested. + The composite configuration including both project-local and global + config data if requested. Note that because this config is a composite, + modifications to the returned value will not be reflected in the files. """ if root is None: root = os.getcwd() config = Config(configspec=cfg.split("\n")) - if local: - for fn in _search_local(root): - tmp = read_config_file(fn) - config.merge(tmp) - if "project" in tmp: - config["project_dir"] = os.path.dirname(fn) - break - else: - for fn in _search_standard_dirs(): + + # Add in any global or user config files. For now this only finds user-specific + # files, but it could be updated in the future to support e.g. system-wide config files. + for fn in (USER_CONFIG_FN,): + if os.path.isfile(fn): config.merge(read_config_file(fn)) - for fn in _search_tree(root): - tmp = read_config_file(fn) - config.merge(tmp) - if "project" in tmp: - config["project_dir"] = os.path.dirname(fn) - break + + if os.path.isfile(_get_project_config_fn(root)): + config.merge(read_config_file(_get_project_config_fn(root))) return config diff --git a/signac/common/validate.py b/signac/common/validate.py index c7a65fbae..c5a1c9f5f 100644 --- a/signac/common/validate.py +++ b/signac/common/validate.py @@ -14,8 +14,7 @@ def get_validator(): # noqa: D103 return Validator() +# TODO: Rename to something internal and uppercase e.g. _CFG. cfg = """ -project = string() -workspace_dir = string(default='workspace') schema_version = string(default='1') """ diff --git a/signac/contrib/migration/__init__.py b/signac/contrib/migration/__init__.py index d0e7f9aee..ca5303327 100644 --- a/signac/contrib/migration/__init__.py +++ b/signac/contrib/migration/__init__.py @@ -7,10 +7,10 @@ import sys from filelock import FileLock -from packaging import version from ...version import SCHEMA_VERSION, __version__ from .v0_to_v1 import _load_config_v1, _migrate_v0_to_v1 +from .v1_to_v2 import _load_config_v2, _migrate_v1_to_v2 FN_MIGRATION_LOCKFILE = ".SIGNAC_PROJECT_MIGRATION_LOCK" @@ -24,18 +24,17 @@ # writeable, i.e. it must be possible to persist in-memory changes from these # objects to the underlying config files. _CONFIG_LOADERS = { - "1": _load_config_v1, + 1: _load_config_v1, + 2: _load_config_v2, } _MIGRATIONS = { - ("0", "1"): _migrate_v0_to_v1, + (0, 1): _migrate_v0_to_v1, + (1, 2): _migrate_v1_to_v2, } -_PARSED_SCHEMA_VERSION = version.parse(SCHEMA_VERSION) - - -_VERSION_LIST = list(reversed(sorted(version.parse(v) for v in _CONFIG_LOADERS.keys()))) +_VERSION_LIST = list(reversed(sorted(_CONFIG_LOADERS.keys()))) def _get_config_schema_version(root_directory, version_guess): @@ -49,7 +48,7 @@ def _get_config_schema_version(root_directory, version_guess): # Note: We could consider using a different component as the key # for _CONFIG_LOADERS, but since this is an internal detail it's # not terribly consequential. - config = _CONFIG_LOADERS[guess.public](root_directory) + config = _CONFIG_LOADERS[guess](root_directory) break except Exception: # The load failed, go to the next @@ -57,18 +56,16 @@ def _get_config_schema_version(root_directory, version_guess): else: raise RuntimeError("Unable to load config file.") try: - return version.parse(config["schema_version"]) + return int(config["schema_version"]) except KeyError: # The default schema version is version 0. - return version.parse("0") + return 0 def _collect_migrations(root_directory): - schema_version = _PARSED_SCHEMA_VERSION + schema_version = int(SCHEMA_VERSION) - current_schema_version = _get_config_schema_version( - root_directory, _PARSED_SCHEMA_VERSION - ) + current_schema_version = _get_config_schema_version(root_directory, schema_version) if current_schema_version > schema_version: # Project config schema version is newer and therefore not supported. raise RuntimeError( @@ -81,11 +78,9 @@ def _collect_migrations(root_directory): guess = current_schema_version while _get_config_schema_version(root_directory, guess) < schema_version: for (origin, destination), migration in _MIGRATIONS.items(): - if version.parse(origin) == _get_config_schema_version( - root_directory, guess - ): + if origin == _get_config_schema_version(root_directory, guess): yield (origin, destination), migration - guess = version.parse(destination) + guess = destination break else: raise RuntimeError( @@ -126,9 +121,7 @@ def apply_migrations(root_directory): f"Failed to apply migration {destination}." ) from e else: - config = _CONFIG_LOADERS[version.parse(destination).public]( - root_directory - ) + config = _CONFIG_LOADERS[destination](root_directory) config["schema_version"] = destination config.write() diff --git a/signac/contrib/migration/v1_to_v2.py b/signac/contrib/migration/v1_to_v2.py new file mode 100644 index 000000000..73c6c0e64 --- /dev/null +++ b/signac/contrib/migration/v1_to_v2.py @@ -0,0 +1,87 @@ +# Copyright (c) 2022 The Regents of the University of Michigan +# All rights reserved. +# This software is licensed under the BSD 3-Clause License. +"""Migrate from schema version 1 to version 2. + +This migration involves the following changes: + - Moving the signac.rc config file to .signac/config + - Moving .signac_shell_history to .signac/shell_history + - Moving .signac_sp_cache.json.gz to .signac/statepoint_cache.json.gz + - Removing the project name from the config. Projects are now identified + solely by their directories. + - Removing the workspace_dir key from the config. The workspace directory + is no longer configurable. +""" + +import os + +from signac.common import configobj +from signac.common.config import _get_project_config_fn +from signac.contrib.project import Project +from signac.synced_collections.backends.collection_json import BufferedJSONAttrDict + +from .v0_to_v1 import _load_config_v1 + +# A minimal v2 config. +_cfg = """ +schema_version = string(default='0') +""" + + +def _load_config_v2(root_directory): + cfg = configobj.ConfigObj( + os.path.join(root_directory, ".signac", "config"), configspec=_cfg.split("\n") + ) + validator = configobj.validate.Validator() + if cfg.validate(validator) is not True: + raise RuntimeError( + "This project's config file is not compatible with signac's v2 schema." + ) + return cfg + + +def _migrate_v1_to_v2(root_directory): + """Migrate from schema version 1 to version 2.""" + # Load the v1 config. + cfg = _load_config_v1(root_directory) + fn_doc = os.path.join(root_directory, Project.FN_DOCUMENT) + doc = BufferedJSONAttrDict(filename=fn_doc, write_concern=True) + + # Try to migrate a custom workspace directory if one exists. + current_workspace_name = cfg.get("workspace_dir") + if current_workspace_name is not None: + if current_workspace_name != "workspace": + current_workspace = os.path.join(root_directory, current_workspace_name) + new_workspace = os.path.join(root_directory, "workspace") + if os.path.exists(new_workspace): + raise RuntimeError( + "Workspace directories are no longer configurable in schema version 2, and " + f"must be 'workspace', but {new_workspace} already exists. Please remove or " + f"move it so that the currently configured workspace directory " + f"{current_workspace} can be moved to {new_workspace}." + ) + os.replace(current_workspace, new_workspace) + del cfg["workspace_dir"] + + # Delete project name from config and store in project doc. + doc["signac_project_name"] = cfg["project"] + del cfg["project"] + cfg.write() + + # Move signac.rc to .signac/config + v1_fn = os.path.join(root_directory, "signac.rc") + v2_fn = _get_project_config_fn(root_directory) + os.mkdir(os.path.dirname(v2_fn)) + os.replace(v1_fn, v2_fn) + + # Now move all other files. + files_to_move = { + ".signac_shell_history": os.sep.join((".signac", "shell_history")), + ".signac_sp_cache.json.gz": os.sep.join( + (".signac", "statepoint_cache.json.gz") + ), + } + for src, dst in files_to_move.items(): + os.replace( + os.sep.join((root_directory, src)), os.sep.join((root_directory, dst)) + ) diff --git a/signac/contrib/project.py b/signac/contrib/project.py index 9dee6cb33..700bb9cf6 100644 --- a/signac/contrib/project.py +++ b/signac/contrib/project.py @@ -21,7 +21,13 @@ from packaging import version -from ..common.config import Config, load_config, read_config_file +from ..common.config import ( + Config, + _get_project_config_fn, + _locate_config_dir, + load_config, + read_config_file, +) from ..common.deprecation import deprecated from ..core.h5store import H5StoreManager from ..sync import sync_projects @@ -101,16 +107,17 @@ def __setitem__(self, key, value): class Project: """The handle on a signac project. - Application developers should usually not need to - directly instantiate this class, but use - :meth:`~signac.get_project` instead. + A :class:`Project` may only be constructed in a directory that is already a + signac project, i.e. a directory in which :func:`~signac.init_project` has + already been run. If project discovery (searching upwards in the folder + hierarchy until a project root is discovered) is desired, users should + instead invoke :func:`~signac.get_project` or :meth:`Project.get_project`. Parameters ---------- - config : - The project configuration to use. By default, it loads the first signac - project configuration found while searching upward from the current - working directory (Default value = None). + root : str, optional + The project root directory. By default, the current working directory + (Default value = None). """ @@ -120,28 +127,25 @@ class Project: KEY_DATA = "signac_data" "The project's datastore key." - FN_CACHE = ".signac_sp_cache.json.gz" + FN_CACHE = os.sep.join((".signac", "statepoint_cache.json.gz")) "The default filename for the state point cache file." _use_pandas_for_html_repr = True # toggle use of pandas for html repr - def __init__(self, config=None): - if config is None: - config = load_config() - self._config = _ProjectConfig(config) - self._lock = RLock() - - # Ensure that the project id is configured. - try: - self._id = str(self.config["project"]) - except KeyError: + def __init__(self, root=None): + if root is None: + root = os.getcwd() + if not os.path.isfile(_get_project_config_fn(root)): raise LookupError( - "Unable to determine project id. " - "Please verify that '{}' is a signac project path.".format( - os.path.abspath(self.config.get("project_dir", os.getcwd())) - ) + f"Unable to find project at path '{os.path.abspath(root)}'." ) + # Project constructor does not search upward, so the provided root must + # be a project directory. + config = load_config(root) + self._config = _ProjectConfig(config) + self._lock = RLock() + # Ensure that the project's data schema is supported. self._check_schema_compatibility() @@ -153,13 +157,10 @@ def __init__(self, config=None): # Prepare root directory and workspace paths. # os.path is used instead of pathlib.Path for performance. - self._root_directory = self.config["project_dir"] - ws = os.path.expandvars( - self.config.get("workspace_dir", "workspace") + self._root_directory = os.path.abspath(root) + self._workspace = _CallableString( + os.path.join(self._root_directory, "workspace") ) - if not os.path.isabs(ws): - ws = os.path.join(self._root_directory, ws) - self._workspace = _CallableString(ws) # Prepare workspace directory. if not os.path.isdir(self.workspace): @@ -167,8 +168,8 @@ def __init__(self, config=None): _mkdir_p(self.workspace) except OSError: logger.error( - "Error occurred while trying to create " - "workspace directory for project {}.".format(self.id) + "Error occurred while trying to create workspace directory for project at root " + f"{self.root_directory()}." ) raise @@ -186,13 +187,10 @@ def __init__(self, config=None): ) def __str__(self): - """Return the project's id.""" - return str(self.id) + return str(self.root_directory()) def __repr__(self): - return "{type}.get_project({root})".format( - type=self.__class__.__name__, root=repr(self.root_directory()) - ) + return f"{self.__class__.__name__}({repr(self.root_directory())})" def _repr_html_(self): """Project details in HTML format for use in IPython environment. @@ -205,8 +203,7 @@ def _repr_html_(self): """ return ( "

" - + f"Project: {self.id}
" - + f"Root: {self.root_directory()}
" + + f"Project: {self.path}
" + f"Workspace: {self.workspace}
" + f"Size: {len(self)}" + "

" @@ -251,32 +248,9 @@ def path(self): @property def workspace(self): - """str: The project's workspace directory. - - The workspace defaults to `project_root/workspace`. Configure this - directory with the ``'workspace_dir'`` configuration option. A relative - path is assumed to be relative to the project's root directory. - - .. note:: - The configuration will respect environment variables, - such as ``$HOME``. - - See :ref:`signac project -w ` for the command line equivalent. - """ + """str: The project's workspace directory.""" return self._workspace - @property - def id(self): - """Get the project identifier. - - Returns - ------- - str - The project id. - - """ - return self._id - def _check_schema_compatibility(self): """Check whether this project's data schema is compatible with this version. @@ -286,8 +260,8 @@ def _check_schema_compatibility(self): If the schema version is incompatible. """ - schema_version = version.parse(SCHEMA_VERSION) - config_schema_version = version.parse(self.config["schema_version"]) + schema_version = SCHEMA_VERSION + config_schema_version = int(self.config["schema_version"]) if config_schema_version > schema_version: # Project config schema version is newer and therefore not supported. raise IncompatibleSchemaVersion( @@ -1360,9 +1334,7 @@ def _build_index(self, include_job_document=False): # workspace, job id, and document file name to be # well-formed, so just use str.join with os.sep instead of # os.path.join for speed. - fn_document = os.sep.join( - (self.workspace, job_id, Job.FN_DOCUMENT) - ) + fn_document = os.sep.join((self.workspace, job_id, Job.FN_DOCUMENT)) with open(fn_document, "rb") as file: doc["doc"] = json.loads(file.read().decode()) except OSError as error: @@ -1496,7 +1468,7 @@ def temporary_project(self, dir=None): yield tmp_project @classmethod - def init_project(cls, *args, root=None, workspace=None, make_dir=True, **kwargs): + def init_project(cls, *args, root=None, **kwargs): """Initialize a project in the provided root directory. It is safe to call this function multiple times with the same @@ -1511,12 +1483,6 @@ def init_project(cls, *args, root=None, workspace=None, make_dir=True, **kwargs) root : str, optional The root directory for the project. Defaults to the current working directory. - workspace : str, optional - The workspace directory for the project. - Defaults to a subdirectory ``workspace`` in the project root. - make_dir : bool, optional - Create the project root directory if it does not exist yet - (Default value = True). Returns ------- @@ -1586,29 +1552,15 @@ def init_project(cls, *args, root=None, workspace=None, make_dir=True, **kwargs) f"project document in which {name_key}={existing_name}." ) except LookupError: - fn_config = os.path.join(root, "signac.rc") - if make_dir: - _mkdir_p(os.path.dirname(fn_config)) + fn_config = _get_project_config_fn(root) + _mkdir_p(os.path.dirname(fn_config)) config = read_config_file(fn_config) - config["project"] = name - if workspace is not None: - config["workspace_dir"] = workspace config["schema_version"] = SCHEMA_VERSION config.write() project = cls.get_project(root=root) if name is not None: project.doc[name_key] = name - assert project.id == str(name) - return project - else: - if workspace is not None and os.path.realpath( - workspace - ) != os.path.realpath(project.workspace): - raise RuntimeError( - f"Failed to initialize project. Path '{os.path.abspath(root)}' already " - "contains a conflicting project configuration." - ) - return project + return project @classmethod def get_project(cls, root=None, search=True, **kwargs): @@ -1641,18 +1593,19 @@ def get_project(cls, root=None, search=True, **kwargs): """ if root is None: root = os.getcwd() - config = load_config(root=root, local=False) - if "project" not in config or ( - not search - and os.path.realpath(config["project_dir"]) != os.path.realpath(root) - ): + + old_root = root + if not search and not os.path.isfile(_get_project_config_fn(root)): + root = None + else: + root = _locate_config_dir(root) + + if not root: raise LookupError( - "Unable to determine project id for path '{}'.".format( - os.path.abspath(root) - ) + f"Unable to find project at path '{os.path.abspath(old_root)}'." ) - return cls(config=config, **kwargs) + return cls(root=root, **kwargs) @classmethod def get_job(cls, root=None): @@ -2170,7 +2123,7 @@ def _repr_html_(self): return repr(self) + self._repr_html_jobs() -def init_project(*args, root=None, workspace=None, make_dir=True, **kwargs): +def init_project(*args, root=None, **kwargs): """Initialize a project. It is safe to call this function multiple times with the same arguments. @@ -2182,12 +2135,6 @@ def init_project(*args, root=None, workspace=None, make_dir=True, **kwargs): root : str, optional The root directory for the project. Defaults to the current working directory. - workspace : str, optional - The workspace directory for the project. - Defaults to a subdirectory ``workspace`` in the project root. - make_dir : bool, optional - Create the project root directory, if it does not exist yet (Default - value = True). Returns ------- @@ -2201,9 +2148,7 @@ def init_project(*args, root=None, workspace=None, make_dir=True, **kwargs): configuration. """ - return Project.init_project( - *args, root=root, workspace=workspace, make_dir=make_dir, **kwargs - ) + return Project.init_project(*args, root=root, **kwargs) def get_project(root=None, search=True, **kwargs): diff --git a/signac/version.py b/signac/version.py index 1b614ea50..f4aa889ec 100644 --- a/signac/version.py +++ b/signac/version.py @@ -6,7 +6,7 @@ __version__ = "1.7.0" -SCHEMA_VERSION = "1" +SCHEMA_VERSION = 2 __all__ = [ diff --git a/tests/test_job.py b/tests/test_job.py index bc97290b6..53967f26a 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -70,12 +70,9 @@ def setUp(self, request): self._tmp_dir = TemporaryDirectory(prefix="signac_") request.addfinalizer(self._tmp_dir.cleanup) self._tmp_pr = os.path.join(self._tmp_dir.name, "pr") - self._tmp_wd = os.path.join(self._tmp_dir.name, "wd") os.mkdir(self._tmp_pr) self.config = signac.common.config.load_config() - self.project = self.project_class.init_project( - root=self._tmp_pr, workspace=self._tmp_wd - ) + self.project = self.project_class.init_project(root=self._tmp_pr) def tearDown(self): pass diff --git a/tests/test_project.py b/tests/test_project.py index eb41ede3a..99f0ac4f8 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -1,6 +1,7 @@ # Copyright (c) 2017 The Regents of the University of Michigan # All rights reserved. # This software is licensed under the BSD 3-Clause License. +import gzip import io import itertools import json @@ -9,6 +10,7 @@ import pickle import re import sys +import textwrap import uuid from contextlib import contextmanager, redirect_stderr from tarfile import TarFile @@ -22,13 +24,19 @@ from test_job import TestJobBase import signac -from signac.common.config import load_config, read_config_file +from signac.common.config import ( + PROJECT_CONFIG_FN, + _get_project_config_fn, + load_config, + read_config_file, +) from signac.contrib.errors import ( IncompatibleSchemaVersion, JobsCorruptedError, StatepointParsingError, WorkspaceError, ) +from signac.contrib.hashing import calc_id from signac.contrib.linked_view import _find_all_links from signac.contrib.project import JobsCursor, Project # noqa: F401 from signac.contrib.schema import ProjectSchema @@ -63,6 +71,12 @@ test_token = {"test_token": str(uuid.uuid4())} +# Use the major version to fail tests expected to fail in 3.0. +_CURRENT_VERSION = version.parse(signac.__version__) +_VERSION_3 = version.parse("3.0.0") +_VERSION_2 = version.parse("2.0.0") + + S_FORMAT1 = """{ 'a': 'int([0, 1, 2, ..., 8, 9], 10)', 'b.b2': 'int([0, 1, 2, ..., 8, 9], 10)', @@ -82,45 +96,29 @@ class TestProjectBase(TestJobBase): class TestProject(TestProjectBase): - def test_get(self): - pass - def test_repr(self): - repr(self.project) p = eval(repr(self.project)) assert repr(p) == repr(self.project) assert p == self.project def test_str(self): - str(self.project) == self.project.id + assert str(self.project) == self.project.root_directory() def test_root_directory(self): assert self._tmp_pr == self.project.root_directory() def test_workspace_directory(self): - assert self._tmp_wd == self.project.workspace + assert os.path.join(self._tmp_pr, "workspace") == self.project.workspace def test_config_modification(self): # In-memory modification of the project configuration is # deprecated as of 1.3, and will be removed in version 2.0. # This unit test should reflect that change beginning 2.0, # and check that the project configuration is immutable. + assert _CURRENT_VERSION < _VERSION_2 with pytest.raises(ValueError): self.project.config["foo"] = "bar" - def test_workspace_directory_with_env_variable(self): - try: - with TemporaryDirectory() as tmp_dir: - os.environ["SIGNAC_ENV_DIR_TEST"] = os.path.join(tmp_dir, "work_here") - project = self.project_class.init_project( - root=tmp_dir, - workspace="${SIGNAC_ENV_DIR_TEST}", - ) - assert project.workspace == os.environ["SIGNAC_ENV_DIR_TEST"] - finally: - if "SIGNAC_ENV_DIR_TEST" in os.environ: - del os.environ["SIGNAC_ENV_DIR_TEST"] - def test_workspace_directory_exists(self): assert os.path.exists(self.project.workspace) @@ -209,24 +207,6 @@ def test_data(self): self.project.data = {"a": {"b": 45}} assert self.project.data == {"a": {"b": 45}} - def test_workspace_path_normalization(self): - def norm_path(p): - return os.path.abspath(os.path.expandvars(p)) - - assert self.project.workspace == norm_path(self._tmp_wd) - - with TemporaryDirectory() as tmp_dir: - abs_path = os.path.join(tmp_dir, "path", "to", "workspace") - project = self.project_class.init_project(root=tmp_dir, workspace=abs_path) - assert project.workspace == norm_path(abs_path) - - with TemporaryDirectory() as tmp_dir: - rel_path = norm_path(os.path.join("path", "to", "workspace")) - project = self.project_class.init_project(root=tmp_dir, workspace=rel_path) - assert project.workspace == norm_path( - os.path.join(project.root_directory(), rel_path) - ) - def test_no_workspace_warn_on_find(self, caplog): if os.path.exists(self.project.workspace): os.rmdir(self.project.workspace) @@ -241,13 +221,11 @@ def test_no_workspace_warn_on_find(self, caplog): @pytest.mark.skipif(WINDOWS, reason="Symbolic links are unsupported on Windows.") def test_workspace_broken_link_error_on_find(self): with TemporaryDirectory() as tmp_dir: - project = self.project_class.init_project( - root=tmp_dir, workspace="workspace-link" - ) - os.rmdir(os.path.join(tmp_dir, "workspace-link")) + project = self.project_class.init_project(root=tmp_dir) + os.rmdir(project.workspace) os.symlink( os.path.join(tmp_dir, "workspace~"), - os.path.join(tmp_dir, "workspace-link"), + project.workspace, ) with pytest.raises(WorkspaceError): list(project.find_jobs()) @@ -272,7 +250,6 @@ def test_workspace_read_only_path(self): finally: logging.disable(logging.NOTSET) - assert not os.path.isdir(self._tmp_wd) assert not os.path.isdir(self.project.workspace) def test_find_jobs(self): @@ -1029,11 +1006,6 @@ def test_temp_project(self): assert not os.path.isdir(tmp_root_dir) -# Use the major version to fail tests expected to fail in 3.0. -_MAJOR_VERSION = version.parse(signac.__version__) -_VERSION_3 = version.parse("3.0.0") - - class TestProjectNameDeprecations: warning_context = pytest.warns( FutureWarning, match="Project names were removed in signac 2.0" @@ -1044,7 +1016,7 @@ def setUp(self, request): self._tmp_dir = TemporaryDirectory() def test_name_only_positional(self): - assert _MAJOR_VERSION < _VERSION_3 + assert _CURRENT_VERSION < _VERSION_3 with self.warning_context: project = signac.init_project("name", root=self._tmp_dir.name) @@ -1058,14 +1030,14 @@ def test_name_only_positional(self): signac.init_project("new_name", root=self._tmp_dir.name) def test_name_with_other_args_positional(self): - assert _MAJOR_VERSION < _VERSION_3 + assert _CURRENT_VERSION < _VERSION_3 with pytest.raises( TypeError, match="takes 0 positional arguments but 2 were given" ): signac.init_project("project", self._tmp_dir.name) def test_name_only_keyword(self): - assert _MAJOR_VERSION < _VERSION_3 + assert _CURRENT_VERSION < _VERSION_3 os.chdir(self._tmp_dir.name) with self.warning_context: project = signac.init_project(name="name") @@ -1080,7 +1052,7 @@ def test_name_only_keyword(self): signac.init_project(name="new_name") def test_name_with_other_args_keyword(self): - assert _MAJOR_VERSION < _VERSION_3 + assert _CURRENT_VERSION < _VERSION_3 with pytest.raises(TypeError, match="got an unexpected keyword argument 'foo'"): signac.init_project(name="project", foo="bar") @@ -2272,14 +2244,6 @@ def test_get_project(self): assert project.workspace == os.path.join(root, "workspace") assert project.root_directory() == root - def test_project_no_id(self): - root = self._tmp_dir.name - signac.init_project(root=root) - config = load_config(root) - del config["project"] - with pytest.raises(LookupError): - Project(config=config) - def test_get_project_non_local(self): root = self._tmp_dir.name subdir = os.path.join(root, "subdir") @@ -2313,9 +2277,6 @@ def test_init(self): project = signac.Project.get_project(root=root) assert project.workspace == os.path.join(root, "workspace") assert project.root_directory() == root - # Deviating initialization parameters should result in errors. - with pytest.raises(RuntimeError): - signac.init_project(root=root, workspace="workspace2") def test_nested_project(self): def check_root(root=None): @@ -2444,7 +2405,7 @@ def test_project_schema_versions(self): assert version.parse(self.project.config["schema_version"]) < version.parse( impossibly_high_schema_version ) - config = read_config_file(self.project.fn("signac.rc")) + config = read_config_file(_get_project_config_fn(self.project.root_directory())) config["schema_version"] = impossibly_high_schema_version config.write() with pytest.raises(IncompatibleSchemaVersion): @@ -2454,36 +2415,6 @@ def test_project_schema_versions(self): with pytest.raises(RuntimeError): apply_migrations(self.project.root_directory()) - # Ensure that migration fails on an invalid version. - invalid_schema_version = "0.5" - config = read_config_file(self.project.fn("signac.rc")) - config["schema_version"] = invalid_schema_version - config.write() - with pytest.raises(RuntimeError): - apply_migrations(self.project.root_directory()) - - @pytest.mark.parametrize("implicit_version", [True, False]) - def test_project_schema_version_migration(self, implicit_version): - from signac.contrib.migration import apply_migrations - - config = read_config_file(self.project.fn("signac.rc")) - if implicit_version: - del config["schema_version"] - assert "schema_version" not in config - else: - config["schema_version"] = "0" - assert config["schema_version"] == "0" - config.write() - err = io.StringIO() - with redirect_stderr(err): - apply_migrations(self.project.root_directory()) - config = read_config_file(self.project.fn("signac.rc")) - assert config["schema_version"] == "1" - project = signac.get_project(root=self.project.root_directory()) - assert project.config["schema_version"] == "1" - assert "OK" in err.getvalue() - assert "0 to 1" in err.getvalue() - def test_no_migration(self): # This unit test should fail as long as there are no schema migrations # implemented within the signac.contrib.migration package. @@ -2498,6 +2429,75 @@ def test_no_migration(self): assert len(migrations) == 0 +class TestSchemaMigration: + @pytest.mark.parametrize("implicit_version", [True, False]) + @pytest.mark.parametrize("workspace_exists", [True, False]) + def test_project_schema_version_migration(self, implicit_version, workspace_exists): + from signac.contrib.migration import apply_migrations + + with TemporaryDirectory() as dirname: + # Create v1 config file. + cfg_fn = os.path.join(dirname, "signac.rc") + workspace_dir = "workspace_dir" + with open(cfg_fn, "w") as f: + f.write( + textwrap.dedent( + f"""\ + project = project + workspace_dir = {workspace_dir} + schema_version = 0""" + ) + ) + + # Create a custom workspace + os.makedirs(os.path.join(dirname, workspace_dir)) + if workspace_exists: + os.makedirs(os.path.join(dirname, "workspace")) + + # Create a shell history file. + history_fn = os.path.join(dirname, ".signac_shell_history") + with open(history_fn, "w") as f: + f.write("print(project)") + + # Create a statepoint cache. Note that this cache does not + # correspond to actual statepoints since we don't currently have + # any in this project, but that's fine for migration testing. + history_fn = os.path.join(dirname, ".signac_sp_cache.json.gz") + sp = {"a": 1} + with gzip.open(history_fn, "wb") as f: + f.write(json.dumps({calc_id(sp): sp}).encode()) + + # If no schema version is present in the config it is equivalent to + # version 0, so we test both explicit and implicit versions. + config = read_config_file(cfg_fn) + if implicit_version: + del config["schema_version"] + assert "schema_version" not in config + else: + assert config["schema_version"] == "0" + config.write() + + # If the 'workspace' directory already exists the migration should fail. + if workspace_exists: + with pytest.raises(RuntimeError): + apply_migrations(dirname) + return + + err = io.StringIO() + with redirect_stderr(err): + apply_migrations(dirname) + config = load_config(dirname) + assert config["schema_version"] == "2" + project = signac.get_project(root=dirname) + assert project.config["schema_version"] == "2" + assert "OK" in err.getvalue() + assert "0 to 1" in err.getvalue() + assert "1 to 2" in err.getvalue() + assert os.path.isfile(project.fn(PROJECT_CONFIG_FN)) + assert os.path.isfile(project.fn(os.sep.join((".signac", "shell_history")))) + assert os.path.isfile(project.fn(Project.FN_CACHE)) + + class TestProjectPickling(TestProjectBase): def test_pickle_project_empty(self): blob = pickle.dumps(self.project) @@ -2544,12 +2544,9 @@ def setUp_base_h5Store(self, request): self._tmp_dir = TemporaryDirectory(prefix="signac_") request.addfinalizer(self._tmp_dir.cleanup) self._tmp_pr = os.path.join(self._tmp_dir.name, "pr") - self._tmp_wd = os.path.join(self._tmp_dir.name, "wd") os.mkdir(self._tmp_pr) self.config = load_config() - self.project = self.project_class.init_project( - root=self._tmp_pr, workspace=self._tmp_wd - ) + self.project = self.project_class.init_project(root=self._tmp_pr) self._fn_store = os.path.join(self._tmp_dir.name, "signac_data.h5") self._fn_store_other = os.path.join(self._tmp_dir.name, "other.h5") diff --git a/tests/test_shell.py b/tests/test_shell.py index 9a7210d67..5635cedb6 100644 --- a/tests/test_shell.py +++ b/tests/test_shell.py @@ -744,7 +744,7 @@ def test_config_show(self): self.call("python -m signac init".split()) out = self.call("python -m signac config --local show".split()).strip() - cfg = config.read_config_file("signac.rc") + cfg = config.read_config_file(".signac/config") expected = config.Config(cfg).write() assert out.split(os.linesep) == expected @@ -754,7 +754,7 @@ def test_config_show(self): assert out.split(os.linesep) == expected out = self.call("python -m signac config --global show".split()).strip() - cfg = config.read_config_file(config.FN_CONFIG) + cfg = config.read_config_file(config.USER_CONFIG_FN) expected = config.Config(cfg).write() assert out.split(os.linesep) == expected @@ -770,12 +770,12 @@ def test_config_set(self): assert "[x]" in cfg assert "y = z" in cfg - backup_config = os.path.exists(config.FN_CONFIG) - global_config_path_backup = config.FN_CONFIG + ".tmp" + backup_config = os.path.exists(config.USER_CONFIG_FN) + global_config_path_backup = config.USER_CONFIG_FN + ".tmp" try: # Make a backup of the global config if it exists if backup_config: - shutil.copy2(config.FN_CONFIG, global_config_path_backup) + shutil.copy2(config.USER_CONFIG_FN, global_config_path_backup) # Test the global config CLI self.call("python -m signac config --global set b c".split()) @@ -786,20 +786,15 @@ def test_config_set(self): # Revert the global config to its previous state (or remove it if # it did not exist) if backup_config: - shutil.move(global_config_path_backup, config.FN_CONFIG) + shutil.move(global_config_path_backup, config.USER_CONFIG_FN) else: - os.remove(config.FN_CONFIG) + os.remove(config.USER_CONFIG_FN) def test_config_verify(self): # no config file - with pytest.raises(ExitCodeError): - self.call("python -m signac config --local verify".split(), error=True) - err = self.call( - "python -m signac config --local verify".split(), - error=True, - raise_error=False, - ) + err = self.call("python -m signac config --local verify".split(), error=True) assert "Did not find a local configuration file" in err + self.call("python -m signac init my_project".split()) err = self.call("python -m signac config --local verify".split(), error=True) assert "Passed" in err