From 62682f69864931e7d51174d0d24d3975de2c900b Mon Sep 17 00:00:00 2001 From: Matt Shin Date: Mon, 19 Mar 2018 11:46:28 +0000 Subject: [PATCH] Lazy loading of `global.rc` Load expensive module only if required. Allow reload. --- bin/cylc-cat-log | 24 +++++------ bin/cylc-cat-state | 6 +-- bin/cylc-documentation | 14 +++---- bin/cylc-edit | 6 +-- bin/cylc-get-site-config | 10 ++--- bin/cylc-gscan | 4 +- bin/cylc-jobscript | 6 +-- bin/cylc-ls-checkpoints | 4 +- bin/cylc-monitor | 4 +- bin/cylc-ping | 6 +-- bin/cylc-report-timings | 4 +- bin/cylc-scan | 12 +++--- bin/cylc-submit | 4 +- bin/cylc-suite-state | 4 +- bin/cylc-trigger | 6 +-- bin/cylc-upgrade-run-dir | 4 +- bin/cylc-view | 8 ++-- bin/gcapture | 4 +- lib/cylc/cfgspec/glbl_cfg.py | 24 +++++++++++ lib/cylc/cfgspec/globalcfg.py | 73 ++++++++++++++++----------------- lib/cylc/cfgspec/suite.py | 6 +-- lib/cylc/gui/app_gcylc.py | 14 +++---- lib/cylc/gui/tailer.py | 8 ++-- lib/cylc/gui/updater_graph.py | 4 +- lib/cylc/hostuserutil.py | 5 ++- lib/cylc/job_file.py | 10 ++--- lib/cylc/mp_pool.py | 4 +- lib/cylc/network/httpclient.py | 4 +- lib/cylc/network/httpserver.py | 10 ++--- lib/cylc/network/port_scan.py | 12 +++--- lib/cylc/prompt.py | 4 +- lib/cylc/remote.py | 8 ++-- lib/cylc/scheduler.py | 22 +++++----- lib/cylc/suite_events.py | 4 +- lib/cylc/suite_logging.py | 17 +++----- lib/cylc/suite_srv_files_mgr.py | 17 ++++---- lib/cylc/task_events_mgr.py | 16 ++++---- lib/cylc/task_job_mgr.py | 8 ++-- lib/cylc/task_remote_mgr.py | 10 ++--- 39 files changed, 213 insertions(+), 197 deletions(-) create mode 100644 lib/cylc/cfgspec/glbl_cfg.py diff --git a/bin/cylc-cat-log b/bin/cylc-cat-log index fbeed136963..05782908fa2 100755 --- a/bin/cylc-cat-log +++ b/bin/cylc-cat-log @@ -42,7 +42,7 @@ from cylc.option_parsers import CylcOptionParser as COP from cylc.rundb import CylcSuiteDAO from cylc.hostuserutil import is_remote from cylc.suite_logging import get_logs -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.task_id import TaskID @@ -148,7 +148,7 @@ def get_option_parser(): def get_suite_log_path(options, suite): """Return file name of a suite log, given the options.""" - log_dir = GLOBAL_CFG.get_derived_host_item(suite, "suite log directory") + log_dir = glbl_cfg().get_derived_host_item(suite, "suite log directory") if options.list_mode: basename = "." else: @@ -186,7 +186,7 @@ def get_task_job_log_path( if submit_num != "NN": submit_num = "%02d" % submit_num return os.path.normpath(os.path.join( - GLOBAL_CFG.get_derived_host_item( + glbl_cfg().get_derived_host_item( suite, "suite job log directory", host, owner), point, task, submit_num, basename)) @@ -204,7 +204,7 @@ def get_task_job_attrs(options, suite, point, task, submit_num): return (None, None) suite_dao = CylcSuiteDAO( os.path.join( - GLOBAL_CFG.get_derived_host_item(suite, "suite run directory"), + glbl_cfg().get_derived_host_item(suite, "suite run directory"), "log", CylcSuiteDAO.DB_FILE_BASE_NAME), is_public=True) task_job_data = suite_dao.select_task_job(None, point, task, submit_num) @@ -242,7 +242,7 @@ def get_task_job_attrs(options, suite, point, task, submit_num): key = "out viewer" else: key = "err viewer" - conf = GLOBAL_CFG.get_host_item("batch systems", host, owner) + conf = glbl_cfg().get_host_item("batch systems", host, owner) command0_tmpl = conf[str(task_job_data["batch_sys_name"])][key] except (KeyError, TypeError): return (user_at_host, None) @@ -291,7 +291,7 @@ def main(): else: owner, host = (None, user_at_host) - cylc_tmpdir = GLOBAL_CFG.get_tmpdir() + cylc_tmpdir = glbl_cfg().get_tmpdir() # Construct the shell command commands = [] @@ -311,12 +311,12 @@ def main(): elif options.tail: if user_at_host: # Replace 'cat' with the remote tail command. - cmd_tmpl = str(GLOBAL_CFG.get_host_item( + cmd_tmpl = str(glbl_cfg().get_host_item( "remote tail command template", host, owner)) commands.append(shlex.split(cmd_tmpl % {"filename": filename})) else: # Replace 'cat' with the local tail command. - cmd_tmpl = str(GLOBAL_CFG.get_host_item( + cmd_tmpl = str(glbl_cfg().get_host_item( "local tail command template")) commands.append(shlex.split(cmd_tmpl % {"filename": filename})) elif options.geditor or options.editor: @@ -324,7 +324,7 @@ def main(): viewfile = mkstemp(dir=cylc_tmpdir)[1] if user_at_host: cmd = shlex.split( - GLOBAL_CFG.get_host_item('scp command', host, owner)) + [ + glbl_cfg().get_host_item('scp command', host, owner)) + [ '%s:%s' % (user_at_host, filename), viewfile] else: cmd = ['cp', filename, viewfile] @@ -336,9 +336,9 @@ def main(): sys.stderr.write(err) sys.exit(ret_code) if options.geditor: - editor = GLOBAL_CFG.get(['editors', 'gui']) + editor = glbl_cfg().get(['editors', 'gui']) elif options.editor: - editor = GLOBAL_CFG.get(['editors', 'terminal']) + editor = glbl_cfg().get(['editors', 'terminal']) command_list = shlex.split(editor) command_list.append(viewfile) @@ -351,7 +351,7 @@ def main(): # Deal with [user@]host. if user_at_host and editor is None: - ssh = str(GLOBAL_CFG.get_host_item("ssh command", host, owner)) + ssh = str(glbl_cfg().get_host_item("ssh command", host, owner)) for i, command in enumerate(commands): commands[i] = shlex.split(ssh) + ["-n", user_at_host] + command diff --git a/bin/cylc-cat-state b/bin/cylc-cat-state index 578fb4edefd..c8d54c48f47 100755 --- a/bin/cylc-cat-state +++ b/bin/cylc-cat-state @@ -32,7 +32,7 @@ import re import sqlite3 import traceback -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.dump import dump_to_stdout, get_stop_state_summary import cylc.flags from cylc.option_parsers import CylcOptionParser as COP @@ -78,7 +78,7 @@ def _get_state_lines(suite): """Get state lines from suite runtime DB.""" dao = CylcSuiteDAO( os.path.join( - GLOBAL_CFG.get_derived_host_item(suite, 'suite run directory'), + glbl_cfg().get_derived_host_item(suite, 'suite run directory'), 'log', CylcSuiteDAO.DB_FILE_BASE_NAME), is_public=True) data = { @@ -149,7 +149,7 @@ def _callback_task_pool(data, row): def _get_state_lines_compat(suite): """Read old state file from normal location, for backward compat.""" state_file_path = os.path.join( - GLOBAL_CFG.get_derived_host_item(suite, 'suite run directory'), + glbl_cfg().get_derived_host_item(suite, 'suite run directory'), "state", "state") lines = [] for line in open(state_file_path, 'rb').readlines(): diff --git a/bin/cylc-documentation b/bin/cylc-documentation index 7828c6abc0b..3b5fcfcc019 100755 --- a/bin/cylc-documentation +++ b/bin/cylc-documentation @@ -45,7 +45,7 @@ from subprocess import call from optparse import OptionParser import cylc.flags -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.run_get_stdout import run_get_stdout @@ -93,13 +93,13 @@ def main(): (options, args) = parser.parse_args() cylc.flags.debug = options.debug - intranet_url = GLOBAL_CFG.get(['documentation', 'urls', 'local index']) - internet_url = GLOBAL_CFG.get(['documentation', 'urls', + intranet_url = glbl_cfg().get(['documentation', 'urls', 'local index']) + internet_url = glbl_cfg().get(['documentation', 'urls', 'internet homepage']) - html_file = GLOBAL_CFG.get(['documentation', 'files', 'html index']) - html_viewer = GLOBAL_CFG.get(['document viewers', 'html']) - pdf_file = GLOBAL_CFG.get(['documentation', 'files', 'pdf user guide']) - pdf_viewer = GLOBAL_CFG.get(['document viewers', 'pdf']) + html_file = glbl_cfg().get(['documentation', 'files', 'html index']) + html_viewer = glbl_cfg().get(['document viewers', 'html']) + pdf_file = glbl_cfg().get(['documentation', 'files', 'pdf user guide']) + pdf_viewer = glbl_cfg().get(['document viewers', 'pdf']) if len(args) == 0: # Cylc documentation. if options.pdf: diff --git a/bin/cylc-edit b/bin/cylc-edit index 19e2407a020..5570255cee2 100755 --- a/bin/cylc-edit +++ b/bin/cylc-edit @@ -67,7 +67,7 @@ from subprocess import call from shutil import copy import cylc.flags -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.option_parsers import CylcOptionParser as COP from cylc.suite_srv_files_mgr import SuiteSrvFilesManager from parsec.include import inline, \ @@ -97,9 +97,9 @@ def main(): suiterc = SuiteSrvFilesManager().parse_suite_arg(options, args[0])[1] if options.geditor: - editor = GLOBAL_CFG.get(['editors', 'gui']) + editor = glbl_cfg().get(['editors', 'gui']) else: - editor = GLOBAL_CFG.get(['editors', 'terminal']) + editor = glbl_cfg().get(['editors', 'terminal']) suitedir = os.path.dirname(suiterc) diff --git a/bin/cylc-get-site-config b/bin/cylc-get-site-config index 81c18ad5e0d..d72e8eac41f 100755 --- a/bin/cylc-get-site-config +++ b/bin/cylc-get-site-config @@ -74,14 +74,14 @@ def main(): if len(args) != 0: parser.error("ERROR: wrong number of arguments") - # import GLOBAL_CFG here to avoid aborting before command help is printed - from cylc.cfgspec.globalcfg import GLOBAL_CFG + # import glbl_cfg here to avoid aborting before command help is printed + from cylc.cfgspec.glbl_cfg import glbl_cfg if options.run_dir: - print GLOBAL_CFG.get_host_item('run directory') + print glbl_cfg().get_host_item('run directory') elif options.site_dir: - print GLOBAL_CFG.SITE_CONF_DIR + print glbl_cfg().SITE_CONF_DIR else: - GLOBAL_CFG.idump( + glbl_cfg().idump( options.item, sparse=options.sparse, pnative=options.pnative) diff --git a/bin/cylc-gscan b/bin/cylc-gscan index c58d926f037..5f221ca68f7 100755 --- a/bin/cylc-gscan +++ b/bin/cylc-gscan @@ -40,7 +40,7 @@ gtk.settings_get_default().set_long_property( gtk.settings_get_default().set_long_property( "gtk-menu-images", True, "main") -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.gui.gscan import ScanApp from cylc.option_parsers import CylcOptionParser as COP @@ -81,7 +81,7 @@ def main(): options, args = parser.parse_args() if options.all_ports: - args.extend(GLOBAL_CFG.get(["suite host scanning", "hosts"])) + args.extend(glbl_cfg().get(["suite host scanning", "hosts"])) scan_app = ScanApp( hosts=args, patterns_name=options.patterns_name, diff --git a/bin/cylc-jobscript b/bin/cylc-jobscript index 4bd28db3b1b..8966db304e1 100755 --- a/bin/cylc-jobscript +++ b/bin/cylc-jobscript @@ -49,11 +49,11 @@ __END__ editor () { python -c " import sys -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg if sys.argv[1] in ['-g', '--gedit']: - print GLOBAL_CFG.get(['editors', 'gui']) + print glbl_cfg().get(['editors', 'gui']) elif sys.argv[1] in ['-e', '--edit']: - print GLOBAL_CFG.get(['editors', 'terminal']) + print glbl_cfg().get(['editors', 'terminal']) " $1 } diff --git a/bin/cylc-ls-checkpoints b/bin/cylc-ls-checkpoints index 321b8e55c86..1292971ad46 100755 --- a/bin/cylc-ls-checkpoints +++ b/bin/cylc-ls-checkpoints @@ -30,7 +30,7 @@ if remrun(): import os -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.option_parsers import CylcOptionParser as COP from cylc.rundb import CylcSuiteDAO @@ -97,7 +97,7 @@ def list_checkpoints(suite, callback): def _get_dao(suite): """Return the DAO (public) for suite.""" - suite_log_dir = GLOBAL_CFG.get_derived_host_item( + suite_log_dir = glbl_cfg().get_derived_host_item( suite, 'suite log directory') pub_db_path = os.path.join(os.path.dirname(suite_log_dir), CylcSuiteDAO.DB_FILE_BASE_NAME) diff --git a/bin/cylc-monitor b/bin/cylc-monitor index 20abf6a862d..70ea3e8be47 100755 --- a/bin/cylc-monitor +++ b/bin/cylc-monitor @@ -36,7 +36,7 @@ from parsec.OrderedDict import OrderedDict from cylc.option_parsers import CylcOptionParser as COP from cylc.network.httpclient import SuiteRuntimeServiceClient, ClientError from cylc.wallclock import get_time_string_from_unix_time -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.task_state import ( TASK_STATUS_RUNAHEAD, TASK_STATUSES_ORDERED, TASK_STATUSES_RESTRICTED) @@ -73,7 +73,7 @@ The USER_AT_HOST argument allows suite selection by 'cylc scan' output: "The state summary line still reflects all task proxies.", action="store_true", default=False, dest="restricted") - def_sort_order = GLOBAL_CFG.get(["monitor", "sort order"]) + def_sort_order = glbl_cfg().get(["monitor", "sort order"]) self.parser.add_option( "-s", "--sort", metavar="ORDER", diff --git a/bin/cylc-ping b/bin/cylc-ping index b39f6d63177..1b375c5a5dd 100755 --- a/bin/cylc-ping +++ b/bin/cylc-ping @@ -32,7 +32,7 @@ import cylc.flags from cylc.option_parsers import CylcOptionParser as COP from cylc.task_id import TaskID from cylc.network.httpclient import SuiteRuntimeServiceClient -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg def main(): @@ -48,8 +48,8 @@ def main(): (options, args) = parser.parse_args() if options.print_ports: - base = GLOBAL_CFG.get(['comms', 'base port']) - range = GLOBAL_CFG.get(['comms', 'maximum number of ports']) + base = glbl_cfg().get(['comms', 'base port']) + range = glbl_cfg().get(['comms', 'maximum number of ports']) print base, '<= port <=', base + range sys.exit(0) diff --git a/bin/cylc-report-timings b/bin/cylc-report-timings index 4aaa01421f0..5ee8e86a91f 100755 --- a/bin/cylc-report-timings +++ b/bin/cylc-report-timings @@ -58,7 +58,7 @@ import contextlib import os import cylc.flags -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.option_parsers import CylcOptionParser as COP from cylc.rundb import CylcSuiteDAO @@ -158,7 +158,7 @@ def format_rows(header, rows): def _get_dao(suite): """Return the DAO (public) for suite.""" - suite_log_dir = GLOBAL_CFG.get_derived_host_item( + suite_log_dir = glbl_cfg().get_derived_host_item( suite, 'suite log directory' ) pub_db_path = os.path.join( diff --git a/bin/cylc-scan b/bin/cylc-scan index 0cbba9c2541..e9db8ad5f8b 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -49,7 +49,7 @@ if "--use-ssh" in sys.argv[1:]: import re import json -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.hostuserutil import get_user from cylc.network.port_scan import scan_many, get_scan_items_from_fs from cylc.option_parsers import CylcOptionParser as COP @@ -160,8 +160,8 @@ def main(): options, args = parser.parse_args() if options.print_ports: - base = GLOBAL_CFG.get(["communication", "base port"]) - max_num_ports = GLOBAL_CFG.get( + base = glbl_cfg().get(["communication", "base port"]) + max_num_ports = glbl_cfg().get( ["communication", "maximum number of ports"]) print base, "<= port <=", base + max_num_ports sys.exit(0) @@ -220,7 +220,7 @@ def main(): parser.error( '--%s=%s: bad regular expression' % (key, pattern)) if options.all_ports: - args.extend(GLOBAL_CFG.get(["suite host scanning", "hosts"])) + args.extend(glbl_cfg().get(["suite host scanning", "hosts"])) if not args: args = get_scan_items_from_fs(cres['suite-owner']) if not args: @@ -297,8 +297,8 @@ def main(): elif not meta_items["title"]: print indent + bold("Title:\n") + indent * 2 + "(no title)" else: - print (indent + bold("Title:\n") + indent * 2 + - '"%s"' % meta_items["title"]) + print(indent + bold("Title:\n") + indent * 2 + + '"%s"' % meta_items["title"]) for metaitem, metavalue in meta_items.items(): if metaitem != "title": if metaitem == "description" or metaitem == "group": diff --git a/bin/cylc-submit b/bin/cylc-submit index 1f6fc28af70..d259a535f3e 100755 --- a/bin/cylc-submit +++ b/bin/cylc-submit @@ -37,7 +37,7 @@ if remrun(): import os from time import sleep -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.config import SuiteConfig from cylc.cycling.loader import get_point import cylc.flags @@ -100,7 +100,7 @@ def main(): taskdef, get_point(point_str).standardise(), is_startup=True)) # Initialise job submit environment - GLOBAL_CFG.create_cylc_run_tree(suite) + glbl_cfg().create_cylc_run_tree(suite) task_job_mgr = TaskJobManager( suite, SuiteProcPool(), SuiteDatabaseManager(), suite_srv_mgr) task_job_mgr.task_remote_mgr.single_task_mode = True diff --git a/bin/cylc-suite-state b/bin/cylc-suite-state index 4b9eb699c27..655068e2a0a 100755 --- a/bin/cylc-suite-state +++ b/bin/cylc-suite-state @@ -61,7 +61,7 @@ if remrun(): import cylc.flags from cylc.option_parsers import CylcOptionParser as COP from cylc.dbstatecheck import CylcSuiteDBChecker -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.command_polling import Poller from cylc.task_state import TASK_STATUSES_ORDERED @@ -239,7 +239,7 @@ def main(): # re-invocation). run_dir = os.path.expandvars( os.path.expanduser( - options.run_dir or GLOBAL_CFG.get_host_item('run directory'))) + options.run_dir or glbl_cfg().get_host_item('run directory'))) pollargs = {'suite': suite, 'run_dir': run_dir, diff --git a/bin/cylc-trigger b/bin/cylc-trigger index 3f6030837f3..0659af71b06 100755 --- a/bin/cylc-trigger +++ b/bin/cylc-trigger @@ -52,7 +52,7 @@ import cylc.flags from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP from cylc.network.httpclient import SuiteRuntimeServiceClient -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg def main(): @@ -138,9 +138,9 @@ def main(): # Edit the new job file. if options.geditor: - editor = GLOBAL_CFG.get(['editors', 'gui']) + editor = glbl_cfg().get(['editors', 'gui']) else: - editor = GLOBAL_CFG.get(['editors', 'terminal']) + editor = glbl_cfg().get(['editors', 'terminal']) # The editor command may have options, e.g. 'emacs -nw'. command_list = re.split(' ', editor) command_list.append(jobfile_path) diff --git a/bin/cylc-upgrade-run-dir b/bin/cylc-upgrade-run-dir index 6ca8a3155e6..c224609e1b2 100755 --- a/bin/cylc-upgrade-run-dir +++ b/bin/cylc-upgrade-run-dir @@ -30,7 +30,7 @@ import shutil from optparse import OptionParser from collections import defaultdict -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.mkdir_p import mkdir_p import cylc.flags @@ -138,7 +138,7 @@ def main(): if os.path.isdir(arg0): rundir = arg0 else: - rundir = GLOBAL_CFG.get_derived_host_item(arg0, "suite run directory") + rundir = glbl_cfg().get_derived_host_item(arg0, "suite run directory") if not os.path.isdir(rundir): sys.exit("ERROR: Directory not found: %s" % rundir) diff --git a/bin/cylc-view b/bin/cylc-view index 55c952a1514..342b0f1476f 100755 --- a/bin/cylc-view +++ b/bin/cylc-view @@ -42,7 +42,7 @@ import shlex from subprocess import call import cylc.flags -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.option_parsers import CylcOptionParser as COP from cylc.suite_srv_files_mgr import SuiteSrvFilesManager from cylc.templatevars import load_template_vars @@ -109,11 +109,11 @@ def main(): options, args = parser.parse_args() suite, suiterc = SuiteSrvFilesManager().parse_suite_arg(options, args[0]) - cylc_tmpdir = GLOBAL_CFG.get_tmpdir() + cylc_tmpdir = glbl_cfg().get_tmpdir() if options.geditor: - editor = GLOBAL_CFG.get(['editors', 'gui']) + editor = glbl_cfg().get(['editors', 'gui']) else: - editor = GLOBAL_CFG.get(['editors', 'terminal']) + editor = glbl_cfg().get(['editors', 'terminal']) # read in the suite.rc file viewcfg = {'mark': options.mark, diff --git a/bin/gcapture b/bin/gcapture index 950d2fc374e..93047ff497c 100755 --- a/bin/gcapture +++ b/bin/gcapture @@ -31,7 +31,7 @@ from optparse import OptionParser sys.path.append( os.path.dirname(os.path.realpath(os.path.abspath(__file__))) + '/../lib') -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.gui.gcapture import gcapture, gcapture_tmpfile # This is a unit test for $CYLC_DIR/lib/cylc/gui/gcapture.py @@ -78,7 +78,7 @@ if options.other: else: ignore_command = False -cylc_tmpdir = GLOBAL_CFG.get_tmpdir() +cylc_tmpdir = glbl_cfg().get_tmpdir() gobject.threads_init() if options.filep: diff --git a/lib/cylc/cfgspec/glbl_cfg.py b/lib/cylc/cfgspec/glbl_cfg.py new file mode 100644 index 00000000000..22f6824ea90 --- /dev/null +++ b/lib/cylc/cfgspec/glbl_cfg.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2018 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Allow lazy loading of `cylc.cfgspec.globalcfg`.""" + + +def glbl_cfg(): + """Load and return the global configuration singleton instance.""" + from cylc.cfgspec.globalcfg import GlobalConfig + return GlobalConfig.get_inst() diff --git a/lib/cylc/cfgspec/globalcfg.py b/lib/cylc/cfgspec/globalcfg.py index adb7b5363db..18407407728 100644 --- a/lib/cylc/cfgspec/globalcfg.py +++ b/lib/cylc/cfgspec/globalcfg.py @@ -409,43 +409,45 @@ class GlobalConfig(config): def get_inst(cls): """Return the singleton instance.""" if not cls._DEFAULT: - if cylc.flags.verbose: - print "Loading site/user config files" cls._DEFAULT = cls(SPEC, upg) - conf_path_str = os.getenv("CYLC_CONF_PATH") - count = 0 - if conf_path_str is None: - # CYLC_CONF_PATH not defined, use default locations - for old_base, conf_dir in [ - [cls.OLD_SITE_CONF_BASE, cls.SITE_CONF_DIR], - [cls.OLD_USER_CONF_BASE, cls.USER_CONF_DIR]]: - for base in [cls.CONF_BASE, old_base]: - file_name = os.path.join(conf_dir, base) - if os.access(file_name, os.F_OK | os.R_OK): - try: - cls._DEFAULT.loadcfg( - file_name, "global config") - except ParsecError as exc: - if count == 0: - sys.stderr.write( - "WARNING: ignoring bad site config %s:" - "\n%s\n" % (file_name, str(exc))) - else: - sys.stderr.write( - "ERROR: bad user config %s:\n" % ( - file_name)) - raise - count += 1 - break - elif conf_path_str: - # CYLC_CONF_PATH defined with a value - for path in conf_path_str.split(os.pathsep): - file_name = os.path.join(path, cls.CONF_BASE) - if os.access(file_name, os.F_OK | os.R_OK): - cls._DEFAULT.loadcfg(file_name, "global config") - cls._DEFAULT.transform() + cls._DEFAULT.load() return cls._DEFAULT + def load(self): + """Load or reload configuration from files.""" + self.sparse.clear() + self.dense.clear() + if cylc.flags.verbose: + print "Loading site/user config files" + conf_path_str = os.getenv("CYLC_CONF_PATH") + if conf_path_str is None: + # CYLC_CONF_PATH not defined, use default locations + for old_base, conf_dir, is_site in [ + [self.OLD_SITE_CONF_BASE, self.SITE_CONF_DIR, True], + [self.OLD_USER_CONF_BASE, self.USER_CONF_DIR, False]]: + for base in [self.CONF_BASE, old_base]: + fname = os.path.join(conf_dir, base) + if os.access(fname, os.F_OK | os.R_OK): + try: + self.loadcfg(fname, "global config") + except ParsecError as exc: + if is_site: + sys.stderr.write( + "WARNING: ignoring bad site config %s:" + "\n%s\n" % (fname, str(exc))) + else: + sys.stderr.write( + "ERROR: bad user config %s:\n" % (fname)) + raise + break + elif conf_path_str: + # CYLC_CONF_PATH defined with a value + for path in conf_path_str.split(os.pathsep): + fname = os.path.join(path, self.CONF_BASE) + if os.access(fname, os.F_OK | os.R_OK): + self.loadcfg(fname, "global config") + self.transform() + def get_derived_host_item( self, suite, item, host=None, owner=None, replace_home=False): """Compute hardwired paths relative to the configurable top dirs.""" @@ -648,6 +650,3 @@ def transform(self): for key, val in cfg['hosts']['localhost'].items(): if val and 'directory' in key: cfg['hosts']['localhost'][key] = expandvars(val) - - -GLOBAL_CFG = GlobalConfig.get_inst() diff --git a/lib/cylc/cfgspec/suite.py b/lib/cylc/cfgspec/suite.py index 900e75f6d6a..983e4b23b5f 100644 --- a/lib/cylc/cfgspec/suite.py +++ b/lib/cylc/cfgspec/suite.py @@ -30,7 +30,7 @@ from cylc.cfgspec.utils import ( coerce_interval, coerce_interval_list, DurationFloat) -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.network import PRIVILEGE_LEVELS, PRIV_SHUTDOWN from cylc.task_id import TaskID @@ -201,7 +201,7 @@ def _coerce_parameter_list(value, keys, _): }, 'cylc': { 'UTC mode': vdr( - vtype='boolean', default=GLOBAL_CFG.get(['cylc', 'UTC mode'])), + vtype='boolean', default=glbl_cfg().get(['cylc', 'UTC mode'])), 'cycle point format': vdr( vtype='cycletime_format', default=None), 'cycle point num expanded year digits': vdr( @@ -286,7 +286,7 @@ def _coerce_parameter_list(value, keys, _): vtype='string', options=PRIVILEGE_LEVELS[ :PRIVILEGE_LEVELS.index(PRIV_SHUTDOWN) + 1], - default=GLOBAL_CFG.get(['authentication', 'public'])) + default=glbl_cfg().get(['authentication', 'public'])) }, }, 'scheduling': { diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index ea064b84054..7688dd267df 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -64,7 +64,7 @@ from cylc.gui.gcapture import gcapture_tmpfile from cylc.suite_srv_files_mgr import SuiteSrvFilesManager from cylc.suite_logging import SuiteLog -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.cfgspec.gcylc import gcfg from cylc.wallclock import get_current_time_string from cylc.task_state import ( @@ -151,8 +151,8 @@ def __init__(self, suite, owner, host, port, self.ungrouped_views = ungrouped_views self.use_defn_order = use_defn_order - self.cylc_tmpdir = GLOBAL_CFG.get_tmpdir() - self.no_prompt = GLOBAL_CFG.get( + self.cylc_tmpdir = glbl_cfg().get_tmpdir() + self.no_prompt = glbl_cfg().get( ['disable interactive command prompts'] ) self.imagedir = get_image_dir() @@ -2291,7 +2291,7 @@ def _popup_logview(self, task_id, task_state_summary, choice=None): filenames = [] name, point_str = TaskID.split(task_id) itask_log_dir = os.path.join( - GLOBAL_CFG.get_derived_host_item( + glbl_cfg().get_derived_host_item( self.cfg.suite, "suite job log directory", ), point_str, @@ -2311,7 +2311,7 @@ def _popup_logview(self, task_id, task_state_summary, choice=None): job_user, job_host = (None, job_user_at_host) if is_remote(job_host, job_user): job_log_dir = job_user_at_host + ':' + os.path.join( - GLOBAL_CFG.get_derived_host_item( + glbl_cfg().get_derived_host_item( self.cfg.suite, 'suite job log directory', job_host, job_user, ), @@ -2381,7 +2381,7 @@ def _get_logview_cmd_tmpls_map(self, task_id, filenames): else: owner, host = (None, user_at_host) try: - conf = GLOBAL_CFG.get_host_item( + conf = glbl_cfg().get_host_item( "batch systems", host, owner) cmd_tmpl = conf[summary["batch_sys_name"]][key + " tailer"] ret[filename] = cmd_tmpl % { @@ -2852,7 +2852,7 @@ def create_main_menu(self): doc_menu.append(gtk.SeparatorMenuItem()) - if GLOBAL_CFG.get(['documentation', 'urls', 'local index']): + if glbl_cfg().get(['documentation', 'urls', 'local index']): cug_www_item = gtk.ImageMenuItem('(http://) Local Document Index') img = gtk.image_new_from_stock(gtk.STOCK_JUMP_TO, gtk.ICON_SIZE_MENU) diff --git a/lib/cylc/gui/tailer.py b/lib/cylc/gui/tailer.py index 267c8c1b5d7..261cde53b7c 100644 --- a/lib/cylc/gui/tailer.py +++ b/lib/cylc/gui/tailer.py @@ -28,7 +28,7 @@ import threading from time import sleep -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.gui.warning_dialog import warning_dialog @@ -85,16 +85,16 @@ def run(self): owner, host = user_at_host.split("@", 1) else: owner, host = (None, user_at_host) - ssh = str(GLOBAL_CFG.get_host_item("ssh command", host, owner)) + ssh = str(glbl_cfg().get_host_item("ssh command", host, owner)) command = shlex.split(ssh) + ["-n", user_at_host] if not cmd_tmpl: - cmd_tmpl = str(GLOBAL_CFG.get_host_item( + cmd_tmpl = str(glbl_cfg().get_host_item( "remote tail command template", host, owner)) command.append(cmd_tmpl % {"filename": filename}) else: filename = self.filename if not cmd_tmpl: - cmd_tmpl = str(GLOBAL_CFG.get_host_item( + cmd_tmpl = str(glbl_cfg().get_host_item( "local tail command template")) command += shlex.split(cmd_tmpl % {"filename": filename}) diff --git a/lib/cylc/gui/updater_graph.py b/lib/cylc/gui/updater_graph.py index 548f8b2d6a2..438cfd652ed 100644 --- a/lib/cylc/gui/updater_graph.py +++ b/lib/cylc/gui/updater_graph.py @@ -24,7 +24,7 @@ from time import sleep import traceback -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg import cylc.flags from cylc.graphing import CGraphPlain from cylc.gui.warning_dialog import warning_dialog @@ -120,7 +120,7 @@ def __init__(self, cfg, updater, theme, info_bar, xdot): self.group_all = True self.graph_frame_count = 0 - self.suite_share_dir = GLOBAL_CFG.get_derived_host_item( + self.suite_share_dir = glbl_cfg().get_derived_host_item( self.cfg.suite, 'suite share directory') def toggle_write_dot_frames(self): diff --git a/lib/cylc/hostuserutil.py b/lib/cylc/hostuserutil.py index 0020eee329a..476a7cc711c 100644 --- a/lib/cylc/hostuserutil.py +++ b/lib/cylc/hostuserutil.py @@ -51,6 +51,8 @@ import socket from time import time +from cylc.cfgspec.glbl_cfg import glbl_cfg + class HostUtil(object): """host and user ID utility.""" @@ -122,8 +124,7 @@ def _get_host_info(self, target=None): @staticmethod def _get_identification_cfg(key): """Return the [suite host self-identification]key global conf.""" - from cylc.cfgspec.globalcfg import GLOBAL_CFG - return GLOBAL_CFG.get(['suite host self-identification', key]) + return glbl_cfg().get(['suite host self-identification', key]) def get_host(self): """Return the preferred identifier for the suite (or current) host. diff --git a/lib/cylc/job_file.py b/lib/cylc/job_file.py index f120c743806..c73f641b185 100644 --- a/lib/cylc/job_file.py +++ b/lib/cylc/job_file.py @@ -23,7 +23,7 @@ from subprocess import Popen, PIPE from cylc.batch_sys_manager import BatchSysManager -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg import cylc.flags @@ -114,14 +114,14 @@ def _check_script_value(value): @staticmethod def _get_derived_host_item(job_conf, key): - """Return derived host item from GLOBAL_CFG.""" - return GLOBAL_CFG.get_derived_host_item( + """Return derived host item from glbl_cfg().""" + return glbl_cfg().get_derived_host_item( job_conf['suite_name'], key, job_conf["host"], job_conf["owner"]) @staticmethod def _get_host_item(job_conf, key): - """Return host item from GLOBAL_CFG.""" - return GLOBAL_CFG.get_host_item( + """Return host item from glbl_cfg().""" + return glbl_cfg().get_host_item( key, job_conf["host"], job_conf["owner"]) @staticmethod diff --git a/lib/cylc/mp_pool.py b/lib/cylc/mp_pool.py index fb268805b01..667ef6c4dd4 100644 --- a/lib/cylc/mp_pool.py +++ b/lib/cylc/mp_pool.py @@ -39,7 +39,7 @@ import time import traceback -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg import cylc.flags from cylc.suite_logging import LOG from cylc.wallclock import get_current_time_string @@ -152,7 +152,7 @@ class SuiteProcPool(object): def __init__(self, pool_size=None): self.pool_size = ( pool_size or - GLOBAL_CFG.get(["process pool size"]) or + glbl_cfg().get(["process pool size"]) or multiprocessing.cpu_count()) # (The Pool class defaults to cpu_count anyway, but does not # expose the result via its public interface). diff --git a/lib/cylc/network/httpclient.py b/lib/cylc/network/httpclient.py index 667e86f5122..227854b5838 100644 --- a/lib/cylc/network/httpclient.py +++ b/lib/cylc/network/httpclient.py @@ -26,6 +26,7 @@ from uuid import uuid4 import warnings +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.exceptions import CylcError import cylc.flags from cylc.network import NO_PASSPHRASE @@ -277,8 +278,7 @@ def _call_server_get_url(self, function, **kwargs): comms_protocol = self.comms_protocol if comms_protocol is None: # Use standard setting from global configuration - from cylc.cfgspec.globalcfg import GLOBAL_CFG - comms_protocol = GLOBAL_CFG.get(['communication', 'method']) + comms_protocol = glbl_cfg().get(['communication', 'method']) url = '%s://%s:%s/%s' % ( comms_protocol, self.host, self.port, function) # If there are any parameters left in the dict after popping, diff --git a/lib/cylc/network/httpserver.py b/lib/cylc/network/httpserver.py index 3895640cc0d..385ae5f774a 100644 --- a/lib/cylc/network/httpserver.py +++ b/lib/cylc/network/httpserver.py @@ -30,7 +30,7 @@ from uuid import uuid4 import cherrypy -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.exceptions import CylcError import cylc.flags from cylc.network import ( @@ -57,13 +57,13 @@ def __init__(self, suite): self.port = None # Figure out the ports we are allowed to use. - base_port = GLOBAL_CFG.get(['communication', 'base port']) - max_ports = GLOBAL_CFG.get( + base_port = glbl_cfg().get(['communication', 'base port']) + max_ports = glbl_cfg().get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) - comms_options = GLOBAL_CFG.get(['communication', 'options']) + comms_options = glbl_cfg().get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be @@ -74,7 +74,7 @@ def __init__(self, suite): self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() - self.comms_method = GLOBAL_CFG.get(['communication', 'method']) + self.comms_method = glbl_cfg().get(['communication', 'method']) self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( diff --git a/lib/cylc/network/port_scan.py b/lib/cylc/network/port_scan.py index 83c3018d486..3bf66053e7a 100644 --- a/lib/cylc/network/port_scan.py +++ b/lib/cylc/network/port_scan.py @@ -25,7 +25,7 @@ import traceback from uuid import uuid4 -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg import cylc.flags from cylc.hostuserutil import is_remote_host, get_host_ip_by_name from cylc.network.httpclient import ( @@ -143,15 +143,15 @@ def scan_many(items, timeout=None, updater=None): else: # Full port range for a host if base_port is None or max_ports is None: - base_port = GLOBAL_CFG.get(['communication', 'base port']) - max_ports = GLOBAL_CFG.get( + base_port = glbl_cfg().get(['communication', 'base port']) + max_ports = glbl_cfg().get( ['communication', 'maximum number of ports']) for port in range(base_port, base_port + max_ports): todo_set.add((item, port)) proc_items = [] results = [] # Number of child processes - max_procs = GLOBAL_CFG.get(["process pool size"]) + max_procs = glbl_cfg().get(["process pool size"]) if max_procs is None: max_procs = cpu_count() try: @@ -243,7 +243,7 @@ def get_scan_items_from_fs(owner_pattern=None, updater=None): srv_files_mgr = SuiteSrvFilesManager() if owner_pattern is None: # Run directory of current user only - run_dirs = [(GLOBAL_CFG.get_host_item('run directory'), None)] + run_dirs = [(glbl_cfg().get_host_item('run directory'), None)] else: # Run directory of all users matching "owner_pattern". # But skip those with /nologin or /false shells @@ -254,7 +254,7 @@ def get_scan_items_from_fs(owner_pattern=None, updater=None): continue if owner_pattern.match(pwent.pw_name): run_dirs.append(( - GLOBAL_CFG.get_host_item( + glbl_cfg().get_host_item( 'run directory', owner=pwent.pw_name, owner_home=pwent.pw_dir), diff --git a/lib/cylc/prompt.py b/lib/cylc/prompt.py index 217daa87580..76f121c1df8 100644 --- a/lib/cylc/prompt.py +++ b/lib/cylc/prompt.py @@ -18,7 +18,7 @@ import sys -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg def prompt(question, force=False, gui=False, no_force=False, no_abort=False, @@ -29,7 +29,7 @@ def prompt(question, force=False, gui=False, no_force=False, no_abort=False, If force is True don't prompt, just return immediately. """ - if (force or GLOBAL_CFG.get(['disable interactive command prompts'])) and ( + if (force or glbl_cfg().get(['disable interactive command prompts'])) and ( not no_force): return True if gui: diff --git a/lib/cylc/remote.py b/lib/cylc/remote.py index 46bf4b68ee0..ae854e9f3ce 100644 --- a/lib/cylc/remote.py +++ b/lib/cylc/remote.py @@ -25,6 +25,7 @@ import sys from textwrap import TextWrapper +from cylc.cfgspec.glbl_cfg import glbl_cfg import cylc.flags @@ -85,13 +86,12 @@ def execute(self, env=None, path=None, dry_run=False, forward_x11=False): if not self.is_remote: return False - from cylc.cfgspec.globalcfg import GLOBAL_CFG from cylc.version import CYLC_VERSION name = os.path.basename(self.argv[0])[5:] # /path/to/cylc-foo => foo # Build the remote command - command = shlex.split(GLOBAL_CFG.get_host_item( + command = shlex.split(glbl_cfg().get_host_item( "ssh command", self.host, self.owner)) if forward_x11: command.append("-Y") @@ -108,7 +108,7 @@ def execute(self, env=None, path=None, dry_run=False, forward_x11=False): # Use bash -l? ssh_login_shell = self.ssh_login_shell if ssh_login_shell is None: - ssh_login_shell = GLOBAL_CFG.get_host_item( + ssh_login_shell = glbl_cfg().get_host_item( "use login shell", self.host, self.owner) # Pass cylc version through. @@ -124,7 +124,7 @@ def execute(self, env=None, path=None, dry_run=False, forward_x11=False): if path: command.append(os.sep.join(path + ["cylc"])) else: - command.append(GLOBAL_CFG.get_host_item( + command.append(glbl_cfg().get_host_item( "cylc executable", self.host, self.owner)) command.append(name) diff --git a/lib/cylc/scheduler.py b/lib/cylc/scheduler.py index c3ff474030c..933a8f52901 100644 --- a/lib/cylc/scheduler.py +++ b/lib/cylc/scheduler.py @@ -31,7 +31,7 @@ import isodatetime.parsers from parsec.util import printcfg -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.config import SuiteConfig from cylc.cycling import PointParsingError from cylc.cycling.loader import get_point, standardise_point_string @@ -126,7 +126,7 @@ def __init__(self, is_restart, options, args): self.suiterc_update_time = None # For user-defined batch system handlers sys.path.append(os.path.join(self.suite_dir, 'python')) - self.suite_run_dir = GLOBAL_CFG.get_derived_host_item( + self.suite_run_dir = glbl_cfg().get_derived_host_item( self.suite, 'suite run directory') self.config = None @@ -210,7 +210,7 @@ def start(self): """Start the server.""" self._start_print_blurb() - GLOBAL_CFG.create_cylc_run_tree(self.suite) + glbl_cfg().create_cylc_run_tree(self.suite) if self.is_restart: self.suite_db_mgr.restart_upgrade() @@ -950,14 +950,14 @@ def configure_comms_daemon(self): mgr.KEY_PORT: str(self.port), mgr.KEY_OWNER: self.owner, mgr.KEY_SUITE_RUN_DIR_ON_SUITE_HOST: self.suite_run_dir, - mgr.KEY_TASK_MSG_MAX_TRIES: str(GLOBAL_CFG.get( + mgr.KEY_TASK_MSG_MAX_TRIES: str(glbl_cfg().get( ['task messaging', 'maximum number of tries'])), - mgr.KEY_TASK_MSG_RETRY_INTVL: str(float(GLOBAL_CFG.get( + mgr.KEY_TASK_MSG_RETRY_INTVL: str(float(glbl_cfg().get( ['task messaging', 'retry interval']))), - mgr.KEY_TASK_MSG_TIMEOUT: str(float(GLOBAL_CFG.get( + mgr.KEY_TASK_MSG_TIMEOUT: str(float(glbl_cfg().get( ['task messaging', 'connection timeout']))), mgr.KEY_VERSION: CYLC_VERSION, - mgr.KEY_COMMS_PROTOCOL: GLOBAL_CFG.get( + mgr.KEY_COMMS_PROTOCOL: glbl_cfg().get( ['communication', 'method'])} try: mgr.dump_contact_file(self.suite, contact_data) @@ -984,7 +984,7 @@ def load_suiterc(self, is_reload=False): ) self.suiterc_update_time = time() # Dump the loaded suiterc for future reference. - cfg_logdir = GLOBAL_CFG.get_derived_host_item( + cfg_logdir = glbl_cfg().get_derived_host_item( self.suite, 'suite config log directory') time_str = get_current_time_string( override_use_utc=True, use_basic_format=True, @@ -1071,9 +1071,9 @@ def configure_suite_environment(self): for var, val in [ ('CYLC_SUITE_RUN_DIR', self.suite_run_dir), ('CYLC_SUITE_LOG_DIR', self.suite_log.get_dir()), - ('CYLC_SUITE_WORK_DIR', GLOBAL_CFG.get_derived_host_item( + ('CYLC_SUITE_WORK_DIR', glbl_cfg().get_derived_host_item( self.suite, 'suite work directory')), - ('CYLC_SUITE_SHARE_DIR', GLOBAL_CFG.get_derived_host_item( + ('CYLC_SUITE_SHARE_DIR', glbl_cfg().get_derived_host_item( self.suite, 'suite share directory')), ('CYLC_SUITE_DEF_PATH', self.suite_dir)]: os.environ[var] = val @@ -1714,7 +1714,7 @@ def _update_cpu_usage(self): def _get_cylc_conf(self, key, default=None): """Return a named setting under [cylc] from suite.rc or global.rc.""" - for getter in [self.config.cfg['cylc'], GLOBAL_CFG.get(['cylc'])]: + for getter in [self.config.cfg['cylc'], glbl_cfg().get(['cylc'])]: try: value = getter[key] except KeyError: diff --git a/lib/cylc/suite_events.py b/lib/cylc/suite_events.py index 6ca713f4660..093a2d05a2d 100644 --- a/lib/cylc/suite_events.py +++ b/lib/cylc/suite_events.py @@ -21,7 +21,7 @@ import os from pipes import quote -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.mp_pool import SuiteProcContext from cylc.hostuserutil import get_host, get_user from cylc.suite_logging import LOG @@ -57,7 +57,7 @@ def get_events_conf(config, key, default=None): """Return a named [cylc][[events]] configuration.""" for getter in [ config.cfg['cylc']['events'], - GLOBAL_CFG.get(['cylc', 'events'])]: + glbl_cfg().get(['cylc', 'events'])]: try: value = getter[key] except KeyError: diff --git a/lib/cylc/suite_logging.py b/lib/cylc/suite_logging.py index cb29c662043..213c76c0c52 100644 --- a/lib/cylc/suite_logging.py +++ b/lib/cylc/suite_logging.py @@ -28,12 +28,7 @@ from time import time -try: - # Permits use of module by standalone utilities that aren't part of a - # suite. - from cylc.cfgspec.globalcfg import GLOBAL_CFG -except ImportError: - pass +from cylc.cfgspec.glbl_cfg import glbl_cfg import cylc.flags from cylc.wallclock import (get_time_string_from_unix_time, get_current_time_string) @@ -254,11 +249,11 @@ def __init__(self, suite, test_params=None): self._group = None if not test_params: self.is_test = False - self.max_bytes = GLOBAL_CFG.get( + self.max_bytes = glbl_cfg().get( ['suite logging', 'maximum size in bytes']) - self.roll_at_startup = GLOBAL_CFG.get( + self.roll_at_startup = glbl_cfg().get( ['suite logging', 'roll over at start-up']) - self.archive_length = GLOBAL_CFG.get( + self.archive_length = glbl_cfg().get( ['suite logging', 'rolling archive length']) else: self.is_test = True @@ -270,7 +265,7 @@ def __init__(self, suite, test_params=None): if test_params: self.ldir = test_params['ldir'] else: - self.ldir = GLOBAL_CFG.get_derived_host_item( + self.ldir = glbl_cfg().get_derived_host_item( suite, 'suite log directory') self.log_paths = {} self.log_paths[self.LOG] = os.path.join(self.ldir, self.LOG) @@ -303,7 +298,7 @@ def get_dir(self): def get_dir_for_suite(suite): """Returns the logging directory for a given suite without setting up suite logging.""" - return GLOBAL_CFG.get_derived_host_item(suite, 'suite log directory') + return glbl_cfg().get_derived_host_item(suite, 'suite log directory') def get_lines(self, log, prev_size, max_lines=10): """Read content from log file up to max_lines from prev_size.""" diff --git a/lib/cylc/suite_srv_files_mgr.py b/lib/cylc/suite_srv_files_mgr.py index 04b5f29adde..b705cf7bdb0 100644 --- a/lib/cylc/suite_srv_files_mgr.py +++ b/lib/cylc/suite_srv_files_mgr.py @@ -24,6 +24,7 @@ from string import ascii_letters, digits import sys +from cylc.cfgspec.glbl_cfg import glbl_cfg import cylc.flags from cylc.mkdir_p import mkdir_p from cylc.hostuserutil import ( @@ -138,8 +139,7 @@ def detect_old_contact_file(self, reg, check_host_port=None): cmd = ["timeout", "10", "ps", self.PS_OPTS, str(old_pid_str)] if is_remote_host(old_host): import shlex - from cylc.cfgspec.globalcfg import GLOBAL_CFG - ssh_str = str(GLOBAL_CFG.get_host_item("ssh command", old_host)) + ssh_str = str(glbl_cfg().get_host_item("ssh command", old_host)) cmd = shlex.split(ssh_str) + ["-n", old_host] + cmd from subprocess import Popen, PIPE from time import sleep, time @@ -329,8 +329,7 @@ def get_suite_srv_dir(self, reg, suite_owner=None): run_d = os.getenv("CYLC_SUITE_RUN_DIR") if (not run_d or os.getenv("CYLC_SUITE_NAME") != reg or os.getenv("CYLC_SUITE_OWNER") != suite_owner): - from cylc.cfgspec.globalcfg import GLOBAL_CFG - run_d = GLOBAL_CFG.get_derived_host_item( + run_d = glbl_cfg().get_derived_host_item( reg, 'suite run directory') return os.path.join(run_d, self.DIR_BASE_SRV) @@ -342,8 +341,7 @@ def list_suites(self, regfilter=None): rec_regfilter = re.compile(regfilter) except re.error as exc: raise ValueError("%s: %s" % (regfilter, exc)) - from cylc.cfgspec.globalcfg import GLOBAL_CFG - run_d = GLOBAL_CFG.get_host_item('run directory') + run_d = glbl_cfg().get_host_item('run directory') results = [] for dirpath, dnames, fnames in os.walk(run_d, followlinks=True): # Always descend for top directory, but @@ -623,11 +621,10 @@ def _load_remote_item(self, item, reg, owner, host): host = 'localhost' if owner is None: owner = get_user() - from cylc.cfgspec.globalcfg import GLOBAL_CFG if item == 'contact' and not is_remote_host(host): # Attempt to read suite contact file via the local filesystem. path = r'%(run_d)s/%(srv_base)s' % { - 'run_d': GLOBAL_CFG.get_derived_host_item( + 'run_d': glbl_cfg().get_derived_host_item( reg, 'suite run directory', 'localhost', owner, replace_home=False), 'srv_base': self.DIR_BASE_SRV, @@ -644,14 +641,14 @@ def _load_remote_item(self, item, reg, owner, host): r'''cat "%(run_d)s/%(srv_base)s/%(item)s"''' ) % { 'prefix': prefix, - 'run_d': GLOBAL_CFG.get_derived_host_item( + 'run_d': glbl_cfg().get_derived_host_item( reg, 'suite run directory', host, owner), 'srv_base': self.DIR_BASE_SRV, 'item': item } import shlex command = shlex.split( - GLOBAL_CFG.get_host_item('ssh command', host, owner)) + glbl_cfg().get_host_item('ssh command', host, owner)) command += ['-n', owner + '@' + host, script] from subprocess import Popen, PIPE try: diff --git a/lib/cylc/task_events_mgr.py b/lib/cylc/task_events_mgr.py index f57948045fd..8d7a33b7f6b 100644 --- a/lib/cylc/task_events_mgr.py +++ b/lib/cylc/task_events_mgr.py @@ -39,7 +39,7 @@ from parsec.config import ItemNotFoundError from cylc.broadcast_mgr import BroadcastMgr -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg import cylc.flags from cylc.mp_pool import SuiteProcContext from cylc.suite_logging import ERR, LOG @@ -134,7 +134,7 @@ def get_host_conf(self, itask, key, default=None, skey="remote"): return itask.tdef.rtconfig[skey][key] else: try: - return GLOBAL_CFG.get_host_item( + return glbl_cfg().get_host_item( key, itask.task_host, itask.task_owner) except (KeyError, ItemNotFoundError): pass @@ -150,7 +150,7 @@ def get_task_job_log( self, suite, point, name, submit_num=None, tail=None): """Return the job log path.""" args = [ - GLOBAL_CFG.get_derived_host_item(suite, "suite job log directory"), + glbl_cfg().get_derived_host_item(suite, "suite job log directory"), self.get_task_job_id(point, name, submit_num)] if tail: args.append(tail) @@ -513,7 +513,7 @@ def _get_events_conf(self, itask, key, default=None): for getter in [ self.broadcast_mgr.get_broadcast(itask.identity).get("events"), itask.tdef.rtconfig["events"], - GLOBAL_CFG.get()["task events"]]: + glbl_cfg().get()["task events"]]: try: value = getter.get(key) except (AttributeError, ItemNotFoundError, KeyError): @@ -529,8 +529,8 @@ def _process_job_logs_retrieval(self, schd_ctx, ctx, id_keys): s_user, s_host = ctx.user_at_host.split("@", 1) else: s_user, s_host = (None, ctx.user_at_host) - ssh_str = str(GLOBAL_CFG.get_host_item("ssh command", s_host, s_user)) - rsync_str = str(GLOBAL_CFG.get_host_item( + ssh_str = str(glbl_cfg().get_host_item("ssh command", s_host, s_user)) + rsync_str = str(glbl_cfg().get_host_item( "retrieve job logs command", s_host, s_user)) cmd = shlex.split(rsync_str) + ["--rsh=" + ssh_str] @@ -549,10 +549,10 @@ def _process_job_logs_retrieval(self, schd_ctx, ctx, id_keys): cmd += ["--include=%s" % (include) for include in sorted(includes)] cmd.append("--exclude=/**") # exclude everything else # Remote source - cmd.append(ctx.user_at_host + ":" + GLOBAL_CFG.get_derived_host_item( + cmd.append(ctx.user_at_host + ":" + glbl_cfg().get_derived_host_item( schd_ctx.suite, "suite job log directory", s_host, s_user) + "/") # Local target - cmd.append(GLOBAL_CFG.get_derived_host_item( + cmd.append(glbl_cfg().get_derived_host_item( schd_ctx.suite, "suite job log directory") + "/") self.proc_pool.put_command( SuiteProcContext(ctx, cmd, env=dict(os.environ), id_keys=id_keys), diff --git a/lib/cylc/task_job_mgr.py b/lib/cylc/task_job_mgr.py index bf227b9667f..5158ab0fa4a 100644 --- a/lib/cylc/task_job_mgr.py +++ b/lib/cylc/task_job_mgr.py @@ -35,7 +35,7 @@ from parsec.util import pdeepcopy, poverride from cylc.batch_sys_manager import BatchSysManager -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.envvar import expandvars import cylc.flags from cylc.hostuserutil import is_remote_host, is_remote_user @@ -261,7 +261,7 @@ def submit_task_jobs(self, suite, itasks, is_simulation=False): if remote_mode: cmd.append('--remote-mode') cmd.append('--') - cmd.append(GLOBAL_CFG.get_derived_host_item( + cmd.append(glbl_cfg().get_derived_host_item( suite, 'suite job log directory', host, owner)) stdin_file_paths = [] job_log_dirs = [] @@ -638,7 +638,7 @@ def _run_job_cmd(self, cmd_key, suite, itasks, callback): if is_remote_user(owner): cmd.append("--user=%s" % (owner)) cmd.append("--") - cmd.append(GLOBAL_CFG.get_derived_host_item( + cmd.append(glbl_cfg().get_derived_host_item( suite, "suite job log directory", host, owner)) job_log_dirs = [] for itask in sorted(itasks, key=lambda itask: itask.identity): @@ -859,7 +859,7 @@ def _prep_submit_task_job_impl(self, suite, itask, rtconfig): job_d = self.task_events_mgr.get_task_job_id( itask.point, itask.tdef.name, itask.submit_num) job_file_path = os.path.join( - GLOBAL_CFG.get_derived_host_item( + glbl_cfg().get_derived_host_item( suite, "suite job log directory", itask.task_host, itask.task_owner), job_d, self.JOB_FILE_BASE) diff --git a/lib/cylc/task_remote_mgr.py b/lib/cylc/task_remote_mgr.py index 56af0665c88..bea7d99b0d0 100644 --- a/lib/cylc/task_remote_mgr.py +++ b/lib/cylc/task_remote_mgr.py @@ -33,7 +33,7 @@ from time import time from uuid import uuid4 -from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.cfgspec.glbl_cfg import glbl_cfg import cylc.flags from cylc.hostuserutil import is_remote, is_remote_host, is_remote_user from cylc.mp_pool import SuiteProcContext @@ -117,7 +117,7 @@ def remote_host_select(self, host_str): host_str = value # command succeeded else: # Command not launched (or already reset) - timeout = GLOBAL_CFG.get(['task host select command timeout']) + timeout = glbl_cfg().get(['task host select command timeout']) if timeout: cmd = ['timeout', str(int(timeout)), 'bash', '-c', cmd_str] else: @@ -209,7 +209,7 @@ def remote_init(self, host, owner): if cylc.flags.debug: cmd.append('--debug') cmd.append(str(self.uuid)) - cmd.append(GLOBAL_CFG.get_derived_host_item( + cmd.append(glbl_cfg().get_derived_host_item( self.suite, 'suite run directory', host, owner)) self.proc_pool.put_command( SuiteProcContext( @@ -248,7 +248,7 @@ def remote_tidy(self): cmd.append('--user=%s' % owner) if cylc.flags.debug: cmd.append('--debug') - cmd.append(os.path.join(GLOBAL_CFG.get_derived_host_item( + cmd.append(os.path.join(glbl_cfg().get_derived_host_item( self.suite, 'suite run directory', host, owner))) procs[(host, owner)] = ( cmd, @@ -319,7 +319,7 @@ def _remote_init_items(self, host, owner): where name is relative path under suite run directory. """ items = [] - comm_meth = GLOBAL_CFG.get_host_item( + comm_meth = glbl_cfg().get_host_item( 'task communication method', host, owner) LOG.debug('comm_meth=%s' % comm_meth) if comm_meth in ['ssh', 'http', 'https']: