Skip to content

Commit

Permalink
Schema2 (#742)
Browse files Browse the repository at this point in the history 
* Initial migration to schema version 2 including config rename (#678)

* Implement initial migration to schema version 2.

* Require project-local config to be signac.rc (not .signacrc) and make searches stricter to match.

* Standardize method for getting project config at a root.

* Move config to .signac/config.

* Fix import order.

* Address PR comments.

* Remove some unnecessary code.

* Address final PR coments.

* Use integer schema version numbers (#688)

* Change schema versioning to use integer strings.

* Switch from int strings to pure ints.

* Update signac/contrib/migration/__init__.py

Co-authored-by: Bradley Dice <[email protected]>

Co-authored-by: Bradley Dice <[email protected]>

* Remove project name from schema (#684)

* Remove project id API.

* Remove project name from config as part of migration.

* Fix issues with config CLI and remove project from default cfg.

* Address PR comments.

* Change the str of a project to the str of its root directory.

* Change Project constructor to use root directory (#706)

* Change project constructor to accept a root directory instead of a config file.

* Change Project repr.

* Address easy PR comments.

* Move internal signac files into .signac subdirectory (#708)

* Move shell history.

* Move sp_cache file.

* Address PR comments.

* Move discovery to separate functions. (#711)

* Move discovery to separate functions.

* Address first round of PR comments.

* Address PR comments.

* Apply suggestions.

* Remove configurable workspace directory  (#714)

* Remove workspace configurability.

* Implement workspace_dir migration.

* Apply suggestions from code review

Co-authored-by: Bradley Dice <[email protected]>

* Address remaining PR comments.

* Update tests/test_project.py

* Remove mention of configurability from project workspace docstring

* Address PR comments.

Co-authored-by: Bradley Dice <[email protected]>
Co-authored-by: Corwin Kerr <[email protected]>

* Update description of schema migration.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: Bradley Dice <[email protected]>
Co-authored-by: Corwin Kerr <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
@vyasr @bdice
@cbkerr @pre-commit-ci
4 people authored Apr 19, 2022
1 parent e04c2ed commit 36cdc29
Show file tree
Hide file tree
Showing 10 changed files with 344 additions and 387 deletions.
117 changes: 38 additions & 79 deletions signac/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
else:
READLINE = True

from . import Project, get_project, init_project
from . import get_project, init_project
from .common import config
from .common.configobj import Section, flatten_errors
from .contrib.filterparse import _add_prefix, parse_filter_arg
Expand Down Expand Up @@ -73,9 +73,7 @@
SHELL_BANNER = """Python {python_version}
signac {signac_version} 🎨
Project:\t{project_id}{job_banner}
Root:\t\t{root_path}
Workspace:\t{workspace_path}
Project:\t{root_path}{job_banner}
Size:\t\t{size}
Interact with the project interface using the "project" or "pr" variable.
Expand All @@ -94,6 +92,8 @@
)


SHELL_HISTORY_FN = os.sep.join((".signac", "shell_history"))

warnings.simplefilter("default")


Expand Down Expand Up @@ -354,7 +354,7 @@ def main_view(args):

def main_init(args):
"""Handle init subcommand."""
init_project(name=args.project_id, root=os.getcwd(), workspace=args.workspace)
init_project(name=args.project_id, root=os.getcwd())
_print_err("Initialized project.")


Expand All @@ -372,8 +372,9 @@ def main_schema(args):

def main_sync(args):
"""Handle sync subcommand."""
# TODO: This function appears to be untested.
#
# Valid provided argument combinations
# Validate provided argument combinations
#
if args.archive:
args.recursive = True
Expand Down Expand Up @@ -421,21 +422,8 @@ def _sig(st):
try:
destination = get_project(root=args.destination)
except LookupError:
if args.allow_workspace:
destination = Project(
config={
"project": os.path.relpath(args.destination),
"project_dir": args.destination,
"workspace_dir": ".",
}
)
else:
_print_err(
"WARNING: The destination appears to not be a project path. "
"Use the '-w/--allow-workspace' option if you want to "
"synchronize to a workspace directory directly."
)
raise
_print_err("WARNING: The destination does not appear to be a project path.")
raise
selection = find_with_filter_or_none(args)

if args.strategy:
Expand Down Expand Up @@ -561,10 +549,8 @@ def _main_import_interactive(project, origin, args):
banner=SHELL_BANNER_INTERACTIVE_IMPORT.format(
python_version=sys.version,
signac_version=__version__,
project_id=project.id,
job_banner="",
root_path=project.root_directory(),
workspace_path=project.workspace,
size=len(project),
origin=args.origin,
),
Expand Down Expand Up @@ -730,26 +716,20 @@ def main_config_show(args):
if args.local and args.globalcfg:
raise ValueError("You can specify either -l/--local or -g/--global, not both.")
elif args.local:
for fn in config.CONFIG_FILENAMES:
if os.path.isfile(fn):
if cfg is None:
cfg = config.read_config_file(fn)
else:
cfg.merge(config.read_config_file(fn))
if os.path.isfile(config.PROJECT_CONFIG_FN):
cfg = config.read_config_file(config.PROJECT_CONFIG_FN)
elif args.globalcfg:
cfg = config.read_config_file(config.FN_CONFIG)
cfg = config.read_config_file(config.USER_CONFIG_FN)
else:
cfg = config.load_config()
if cfg is None:
if args.local and args.globalcfg:
mode = " local or global "
elif args.local:
mode = " local "
cfg = config.load_config(config._locate_config_dir(os.getcwd()))
if not cfg:
if args.local:
mode = "local"
elif args.globalcfg:
mode = " global "
mode = "global"
else:
mode = ""
_print_err(f"Did not find a{mode}configuration file.")
mode = "local or global"
_print_err(f"Did not find a {mode} configuration file.")
return
for key in args.key:
for kt in key.split("."):
Expand All @@ -769,29 +749,25 @@ def main_config_verify(args):
if args.local and args.globalcfg:
raise ValueError("You can specify either -l/--local or -g/--global, not both.")
elif args.local:
for fn in config.CONFIG_FILENAMES:
if os.path.isfile(fn):
if cfg is None:
cfg = config.read_config_file(fn)
else:
cfg.merge(config.read_config_file(fn))
if os.path.isfile(config.PROJECT_CONFIG_FN):
cfg = config.read_config_file(config.PROJECT_CONFIG_FN)
elif args.globalcfg:
cfg = config.read_config_file(config.FN_CONFIG)
cfg = config.read_config_file(config.USER_CONFIG_FN)
else:
cfg = config.load_config()
if cfg is None:
if args.local and args.globalcfg:
mode = " local or global "
elif args.local:
mode = " local "
cfg = config.load_config(config._locate_config_dir(os.getcwd()))
if not cfg:
if args.local:
mode = "local"
elif args.globalcfg:
mode = " global "
mode = "global"
else:
mode = ""
raise RuntimeWarning(f"Did not find a{mode}configuration file.")
if cfg.filename is not None:
_print_err(f"Verification of config file '{cfg.filename}'.")
verify_config(cfg)
mode = "local or global"
_print_err(f"Did not find a {mode} configuration file.")
return
else:
if cfg.filename is not None:
_print_err(f"Verification of config file '{cfg.filename}'.")
verify_config(cfg)


def main_config_set(args):
Expand All @@ -802,11 +778,10 @@ def main_config_set(args):
if args.local and args.globalcfg:
raise ValueError("You can specify either -l/--local or -g/--global, not both.")
elif args.local:
for fn_config in config.CONFIG_FILENAMES:
if os.path.isfile(fn_config):
break
if os.path.isfile(config.PROJECT_CONFIG_FN):
fn_config = config.PROJECT_CONFIG_FN
elif args.globalcfg:
fn_config = config.FN_CONFIG
fn_config = config.USER_CONFIG_FN
else:
raise ValueError(
"You need to specify either -l/--local or -g/--global "
Expand Down Expand Up @@ -886,7 +861,7 @@ def jobs():
else: # interactive
if READLINE:
if "PyPy" not in platform.python_implementation():
fn_hist = project.fn(".signac_shell_history")
fn_hist = project.fn(SHELL_HISTORY_FN)
try:
readline.read_history_file(fn_hist)
readline.set_history_length(1000)
Expand Down Expand Up @@ -915,10 +890,8 @@ def write_history_file():
banner=SHELL_BANNER.format(
python_version=sys.version,
signac_version=__version__,
project_id=project.id,
job_banner=f"\nJob:\t\t{job.id}" if job is not None else "",
root_path=project.root_directory(),
workspace_path=project.workspace,
size=len(project),
),
)
Expand Down Expand Up @@ -947,13 +920,6 @@ def main():

parser_init = subparsers.add_parser("init")
parser_init.add_argument("project_id", nargs="?", help=argparse.SUPPRESS)
parser_init.add_argument(
"-w",
"--workspace",
type=str,
default="workspace",
help="The path to the workspace directory.",
)
parser_init.set_defaults(func=main_init)

parser_project = subparsers.add_parser("project")
Expand Down Expand Up @@ -1499,13 +1465,6 @@ def main():
"--no-keys", action="store_true", help="Never overwrite any conflicting keys."
)

parser_sync.add_argument(
"-w",
"--allow-workspace",
action="store_true",
help="Allow the specification of a workspace (instead of a project) directory "
"as the destination path.",
)
parser_sync.add_argument(
"--force", action="store_true", help="Ignore all warnings, just synchronize."
)
Expand Down
99 changes: 42 additions & 57 deletions signac/common/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,73 +12,67 @@

logger = logging.getLogger(__name__)

DEFAULT_FILENAME = ".signacrc"
CONFIG_FILENAMES = [DEFAULT_FILENAME, "signac.rc"]
HOME = os.path.expanduser("~")
CONFIG_PATH = [HOME]
FN_CONFIG = os.path.expanduser("~/.signacrc")
PROJECT_CONFIG_FN = os.path.join(".signac", "config")
USER_CONFIG_FN = os.path.expanduser(os.path.join("~", ".signacrc"))

# TODO: Consider making this entire module internal and removing all its
# functions from the public API.

def _search_local(root):
for fn in CONFIG_FILENAMES:
fn_ = os.path.abspath(os.path.join(root, fn))
if os.path.isfile(fn_):
yield fn_

def _get_project_config_fn(root):
return os.path.abspath(os.path.join(root, PROJECT_CONFIG_FN))

def _search_tree(root=None):
"""Locates signac configuration files in a directory hierarchy.

def _locate_config_dir(search_path):
"""Locates root directory containing a signac configuration file in a directory hierarchy.
Parameters
----------
root : str
Path to search. Uses ``os.getcwd()`` if None (Default value = None).
Starting path to search.
Returns
--------
str or None
The root directory containing the configuration file if one is found, otherwise None.
"""
if root is None:
root = os.getcwd()
root = os.path.abspath(search_path)
while True:
yield from _search_local(root)
up = os.path.abspath(os.path.join(root, ".."))
if os.path.isfile(_get_project_config_fn(root)):
return root
# TODO: Could use the walrus operator here when we completely drop
# Python 3.7 support if we like the operator.
up = os.path.dirname(root)
if up == root:
msg = "Reached filesystem root."
logger.debug(msg)
return
logger.debug("Reached filesystem root, no config found.")
return None
else:
root = up


def _search_standard_dirs():
"""Locates signac configuration files in standard directories."""
for path in CONFIG_PATH:
yield from _search_local(path)


def read_config_file(filename, configspec=None, *args, **kwargs):
def read_config_file(filename):
"""Read a configuration file.
Parameters
----------
filename : str
The path to the file to read.
configspec : List[str], optional
The key-value pairs supported in the config.
Returns
--------
:class:`Config`
The config contained in the file.
"""
logger.debug(f"Reading config file '{filename}'.")
if configspec is None:
configspec = cfg.split("\n")
configspec = cfg.split("\n")
try:
config = Config(filename, configspec=configspec, *args, **kwargs)
config = Config(filename, configspec=configspec)
except (OSError, ConfigObjError) as error:
msg = "Failed to read configuration file '{}':\n{}"
raise ConfigError(msg.format(filename, error))
raise ConfigError(f"Failed to read configuration file '{filename}':\n{error}")
verification = config.verify()
if verification is not True:
# TODO: In the future this should raise an error, not just a
# debug-level logging notice.
logger.debug(
"Config file '{}' may contain invalid values.".format(
os.path.abspath(filename)
Expand All @@ -87,42 +81,33 @@ def read_config_file(filename, configspec=None, *args, **kwargs):
return config


def load_config(root=None, local=False):
"""Load configuration, searching upward from a root path if desired.
def load_config(root=None):
"""Load configuration from a project directory.
Parameters
----------
root : str
The path from which to begin searching for config files.
local : bool, optional
If ``True``, only search in the provided directory and do not traverse
upwards through the filesystem (Default value: False).
The project path to pull project-local configuration data from.
Returns
--------
:class:`Config`
The composite configuration including both local and global config data
if requested.
The composite configuration including both project-local and global
config data if requested. Note that because this config is a composite,
modifications to the returned value will not be reflected in the files.
"""
if root is None:
root = os.getcwd()
config = Config(configspec=cfg.split("\n"))
if local:
for fn in _search_local(root):
tmp = read_config_file(fn)
config.merge(tmp)
if "project" in tmp:
config["project_dir"] = os.path.dirname(fn)
break
else:
for fn in _search_standard_dirs():

# Add in any global or user config files. For now this only finds user-specific
# files, but it could be updated in the future to support e.g. system-wide config files.
for fn in (USER_CONFIG_FN,):
if os.path.isfile(fn):
config.merge(read_config_file(fn))
for fn in _search_tree(root):
tmp = read_config_file(fn)
config.merge(tmp)
if "project" in tmp:
config["project_dir"] = os.path.dirname(fn)
break

if os.path.isfile(_get_project_config_fn(root)):
config.merge(read_config_file(_get_project_config_fn(root)))
return config


Expand Down
3 changes: 1 addition & 2 deletions signac/common/validate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@ def get_validator(): # noqa: D103
return Validator()


# TODO: Rename to something internal and uppercase e.g. _CFG.
cfg = """
project = string()
workspace_dir = string(default='workspace')
schema_version = string(default='1')
"""
Loading

0 comments on commit 36cdc29

Please sign in to comment.