add support for alternate easyconfig parameters/templates/constants

easybuild/framework/easyconfig/easyconfig.py

@@ -59,9 +59,10 @@
 from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
 from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION, retrieve_blocks_in_spec
 from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
-from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS
+from easybuild.framework.easyconfig.parser import ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS, REPLACED_PARAMETERS
 from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig
-from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict
+from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATES, DEPRECATED_TEMPLATES, TEMPLATE_CONSTANTS
+from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_DYNAMIC, template_constant_dict
 from easybuild.tools import LooseVersion
 from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg
 from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG
@@ -118,11 +119,13 @@ def handle_deprecated_or_replaced_easyconfig_parameters(ec_method):
     def new_ec_method(self, key, *args, **kwargs):
         """Check whether any replace easyconfig parameters are still used"""
         # map deprecated parameters to their replacements, issue deprecation warning(/error)
-        if key in DEPRECATED_PARAMETERS:
+        if key in ALTERNATE_PARAMETERS:
+            key = ALTERNATE_PARAMETERS[key]
+        elif key in DEPRECATED_PARAMETERS:
             depr_key = key
             key, ver = DEPRECATED_PARAMETERS[depr_key]
             _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver)
-        if key in REPLACED_PARAMETERS:
+        elif key in REPLACED_PARAMETERS:
             _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')
         return ec_method(self, key, *args, **kwargs)
 
@@ -179,7 +182,7 @@ def triage_easyconfig_params(variables, ec):
 
     for key in variables:
         # validations are skipped, just set in the config
-        if key in ec or key in DEPRECATED_PARAMETERS.keys():
+        if any(key in d for d in (ec, DEPRECATED_PARAMETERS.keys(), ALTERNATE_PARAMETERS.keys())):
             ec_params[key] = variables[key]
             _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key]))
         elif key in REPLACED_PARAMETERS:
@@ -658,7 +661,7 @@ def set_keys(self, params):
         with self.disable_templating():
             for key in sorted(params.keys()):
                 # validations are skipped, just set in the config
-                if key in self._config.keys() or key in DEPRECATED_PARAMETERS.keys():
+                if any(key in x.keys() for x in (self._config, ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS)):
                     self[key] = params[key]
                     self.log.info("setting easyconfig parameter %s: value %s (type: %s)",
                                   key, self[key], type(self[key]))
@@ -1994,7 +1997,34 @@ def resolve_template(value, tmpl_dict):
             try:
                 value = value % tmpl_dict
             except KeyError:
-                _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict)
+                # check if any alternate and/or deprecated templates resolve
+                try:
+                    orig_value = value
+                    # map old templates to new values for alternate and deprecated templates
+                    alt_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, new_tmpl) in
+                               ALTERNATE_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()}
+                    alt_map2 = {new_tmpl: tmpl_dict[old_tmpl] for (old_tmpl, new_tmpl) in
+                                ALTERNATE_TEMPLATES.items() if old_tmpl in tmpl_dict.keys()}
+                    depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in
+                                DEPRECATED_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()}
+
+                    # try templating with alternate and deprecated templates included
+                    value = value % {**tmpl_dict, **alt_map, **alt_map2, **depr_map}
+
+                    for old_tmpl, val in depr_map.items():
+                        # check which deprecated templates were replaced, and issue deprecation warnings
+                        if old_tmpl in orig_value and val in value:
+                            new_tmpl, ver = DEPRECATED_TEMPLATES[old_tmpl]
+                            _log.deprecated(f"Easyconfig template '{old_tmpl}' is deprecated, use '{new_tmpl}' instead",
+                                            ver)
+                except KeyError:
+                    _log.warning(f"Unable to resolve template value {value} with dict {tmpl_dict}")
+
+                for key in tmpl_dict:
+                    if key in DEPRECATED_TEMPLATES:
+                        new_key, ver = DEPRECATED_TEMPLATES[key]
+                        _log.deprecated(f"Easyconfig template '{key}' is deprecated, use '{new_key}' instead", ver)
+
     else:
         # this block deals with references to objects and returns other references
         # for reading this is ok, but for self['x'] = {}

easybuild/framework/easyconfig/format/pyheaderconfigobj.py

@@ -39,6 +39,7 @@
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
 from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat
 from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
+from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATE_CONSTANTS, DEPRECATED_TEMPLATE_CONSTANTS
 from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.configobj import ConfigObj
@@ -86,6 +87,58 @@ def build_easyconfig_variables_dict():
     return vars_dict
 
 
+def handle_deprecated_constants(method):
+    """Decorator to handle deprecated easyconfig template constants"""
+    def wrapper(self, key, *args, **kwargs):
+        """Check whether any deprecated constants are used"""
+        alternate = ALTERNATE_TEMPLATE_CONSTANTS
+        deprecated = DEPRECATED_TEMPLATE_CONSTANTS
+        if key in alternate:
+            key = alternate[key]
+        elif key in deprecated:
+            depr_key = key
+            key, ver = deprecated[depr_key]
+            _log.deprecated(f"Easyconfig template constant '{depr_key}' is deprecated, use '{key}' instead", ver)
+        return method(self, key, *args, **kwargs)
+    return wrapper
+
+
+class DeprecatedDict(dict):
+    """Custom dictionary that handles deprecated easyconfig template constants gracefully"""
+
+    def __init__(self, *args, **kwargs):
+        self.clear()
+        self.update(*args, **kwargs)
+
+    @handle_deprecated_constants
+    def __contains__(self, key):
+        return super().__contains__(key)
+
+    @handle_deprecated_constants
+    def __delitem__(self, key):
+        return super().__delitem__(key)
+
+    @handle_deprecated_constants
+    def __getitem__(self, key):
+        return super().__getitem__(key)
+
+    @handle_deprecated_constants
+    def __setitem__(self, key, value):
+        return super().__setitem__(key, value)
+
+    def update(self, *args, **kwargs):
+        if args:
+            if isinstance(args[0], dict):
+                for key, value in args[0].items():
+                    self.__setitem__(key, value)
+            else:
+                for key, value in args[0]:
+                    self.__setitem__(key, value)
+
+        for key, value in kwargs.items():
+            self.__setitem__(key, value)
+
+
 class EasyConfigFormatConfigObj(EasyConfigFormat):
     """
     Extended EasyConfig format, with support for a header and sections that are actually parsed (as opposed to exec'ed).
@@ -176,7 +229,7 @@ def parse_header(self, header):
 
     def parse_pyheader(self, pyheader):
         """Parse the python header, assign to docstring and cfg"""
-        global_vars = self.pyheader_env()
+        global_vars = DeprecatedDict(self.pyheader_env())
         self.log.debug("pyheader initial global_vars %s", global_vars)
         self.log.debug("pyheader text being exec'ed: %s", pyheader)
 

easybuild/framework/easyconfig/parser.py

@@ -42,6 +42,11 @@
 from easybuild.tools.filetools import read_file, write_file
 
 
+# alternate easyconfig parameters, and their non-deprecated equivalents
+ALTERNATE_PARAMETERS = {
+    # <new_param>: <equivalent_param>,
+}
+
 # deprecated easyconfig parameters, and their replacements
 DEPRECATED_PARAMETERS = {
     # <old_param>: (<new_param>, <deprecation_version>),

easybuild/framework/easyconfig/templates.py

@@ -161,6 +161,26 @@
     ('SHLIB_EXT', get_shared_lib_ext(), 'extension for shared libraries'),
 ]
 
+# alternate templates, and their equivalents
+ALTERNATE_TEMPLATES = {
+    # <new>: <equivalent_template>,
+}
+
+# deprecated templates, and their replacements
+DEPRECATED_TEMPLATES = {
+    # <old_template>: (<new_template>, <deprecation_version>),
+}
+
+# alternate template constants, and their equivalents
+ALTERNATE_TEMPLATE_CONSTANTS = {
+    # <new_template_constant>: <equivalent_template_constant>,
+}
+
+# deprecated template constants, and their replacements
+DEPRECATED_TEMPLATE_CONSTANTS = {
+    # <old_template_constant>: (<new_template_constant>, <deprecation_version>),
+}
+
 extensions = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z']
 for ext in extensions:
     suffix = ext.replace('.', '_').upper()

test/framework/easyconfig.py

@@ -40,7 +40,6 @@
 import textwrap
 from collections import OrderedDict
 from easybuild.tools import LooseVersion
-from importlib import reload
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
 from unittest import TextTestRunner
 
@@ -120,6 +119,11 @@ def setUp(self):
         github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT)
         self.skip_github_tests = github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None
 
+        self.orig_easyconfig_DEPRECATED_PARAMETERS = easyconfig.easyconfig.DEPRECATED_PARAMETERS
+        self.orig_easyconfig_DEPRECATED_TEMPLATES = easyconfig.easyconfig.DEPRECATED_TEMPLATES
+        self.orig_easyconfig_ALTERNATE_PARAMETERS = easyconfig.easyconfig.ALTERNATE_PARAMETERS
+        self.orig_easyconfig_ALTERNATE_TEMPLATES = easyconfig.easyconfig.ALTERNATE_TEMPLATES
+
     def prep(self):
         """Prepare for test."""
         # (re)cleanup last test file
@@ -133,10 +137,17 @@ def prep(self):
     def tearDown(self):
         """ make sure to remove the temporary file """
         st.get_cpu_architecture = self.orig_get_cpu_architecture
+
         super(EasyConfigTest, self).tearDown()
         if os.path.exists(self.eb_file):
             os.remove(self.eb_file)
 
+        # restore orignal values of DEPRECATED_TEMPLATES & co in easyconfig.templates
+        easyconfig.easyconfig.DEPRECATED_PARAMETERS = self.orig_easyconfig_DEPRECATED_PARAMETERS
+        easyconfig.easyconfig.DEPRECATED_TEMPLATES = self.orig_easyconfig_DEPRECATED_TEMPLATES
+        easyconfig.easyconfig.ALTERNATE_PARAMETERS = self.orig_easyconfig_ALTERNATE_PARAMETERS
+        easyconfig.easyconfig.ALTERNATE_TEMPLATES = self.orig_easyconfig_ALTERNATE_TEMPLATES
+
     def test_empty(self):
         """ empty files should not parse! """
         self.contents = "# empty string"
@@ -1451,6 +1462,47 @@ def test_sysroot_template(self):
         self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix)
         self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix)
 
+    def test_template_deprecation_and_alternate(self):
+        """Test deprecation of (and alternate) templates"""
+
+        self.prep()
+
+        template_test_deprecations = {
+            'builddir': ('depr_build_dir', '1000000000'),
+            'cudaver': ('depr_cuda_ver', '1000000000'),
+            'start_dir': ('depr_start_dir', '1000000000'),
+        }
+        easyconfig.easyconfig.DEPRECATED_TEMPLATES = template_test_deprecations
+
+        template_test_alternates = {
+            'installdir': 'alt_install_dir',
+            'version_major_minor': 'alt_ver_maj_min',
+        }
+        easyconfig.easyconfig.ALTERNATE_TEMPLATES = template_test_alternates
+
+        tmpl_str = ("cd %(start_dir)s && make %(namelower)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'"
+                    " && echo %(alt_install_dir)s %(version_major_minor)s")
+        tmpl_dict = {
+            'depr_build_dir': '/example/build_dir',
+            'depr_cuda_ver': '12.1.1',
+            'installdir': '/example/installdir',
+            'start_dir': '/example/build_dir/start_dir',
+            'alt_ver_maj_min': '1.2',
+            'namelower': 'foo',
+        }
+
+        with self.mocked_stdout_stderr() as (_, stderr):
+            res = resolve_template(tmpl_str, tmpl_dict)
+        stderr = stderr.getvalue()
+
+        for tmpl in [*template_test_deprecations.keys(), *template_test_alternates.keys()]:
+            self.assertNotIn("%(" + tmpl + ")s", res)
+
+        for old, (new, ver) in template_test_deprecations.items():
+            depr_str = (f"WARNING: Deprecated functionality, will no longer work in EasyBuild v{ver}: "
+                        f"Easyconfig template '{old}' is deprecated, use '{new}' instead")
+            self.assertIn(depr_str, stderr)
+
     def test_constant_doc(self):
         """test constant documentation"""
         doc = avail_easyconfig_constants()
@@ -1766,6 +1818,52 @@ def foo(key):
 
             self.assertErrorRegex(EasyBuildError, error_regex, foo, key)
 
+    def test_alternate_easyconfig_parameters(self):
+        """Test handling of alternate easyconfig parameters."""
+
+        test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')
+
+        test_ec_txt = read_file(toy_ec)
+        test_ec_txt = test_ec_txt.replace('postinstallcmds', 'post_install_cmds')
+        test_ec_txt = test_ec_txt.replace('moduleclass', 'env_mod_class')
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        write_file(test_ec, test_ec_txt)
+
+        # post_install_cmds is not accepted unless it's registered as an alternative easyconfig parameter
+        easyconfig.easyconfig.ALTERNATE_PARAMETERS = {}
+        self.assertErrorRegex(EasyBuildError, "post_install_cmds -> postinstallcmds", EasyConfig, test_ec)
+
+        easyconfig.easyconfig.ALTERNATE_PARAMETERS = {
+            'env_mod_class': 'moduleclass',
+            'post_install_cmds': 'postinstallcmds',
+        }
+        ec = EasyConfig(test_ec)
+
+        expected = 'tools'
+        self.assertEqual(ec['moduleclass'], expected)
+        self.assertEqual(ec['env_mod_class'], expected)
+
+        expected = ['echo TOY > %(installdir)s/README']
+        self.assertEqual(ec['postinstallcmds'], expected)
+        self.assertEqual(ec['post_install_cmds'], expected)
+
+        # test setting of easyconfig parameter with original & alternate name
+        ec['moduleclass'] = 'test1'
+        self.assertEqual(ec['moduleclass'], 'test1')
+        self.assertEqual(ec['env_mod_class'], 'test1')
+        ec.update('moduleclass', 'test2')
+        self.assertEqual(ec['moduleclass'], 'test1 test2 ')
+        self.assertEqual(ec['env_mod_class'], 'test1 test2 ')
+
+        ec['env_mod_class'] = 'test3'
+        self.assertEqual(ec['moduleclass'], 'test3')
+        self.assertEqual(ec['env_mod_class'], 'test3')
+        ec.update('env_mod_class', 'test4')
+        self.assertEqual(ec['moduleclass'], 'test3 test4 ')
+        self.assertEqual(ec['env_mod_class'], 'test3 test4 ')
+
     def test_deprecated_easyconfig_parameters(self):
         """Test handling of deprecated easyconfig parameters."""
         os.environ.pop('EASYBUILD_DEPRECATED')
@@ -1775,21 +1873,15 @@ def test_deprecated_easyconfig_parameters(self):
         test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
         ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb'))
 
-        orig_deprecated_parameters = copy.deepcopy(easyconfig.parser.DEPRECATED_PARAMETERS)
-        easyconfig.parser.DEPRECATED_PARAMETERS.update({
+        easyconfig.easyconfig.DEPRECATED_PARAMETERS = {
             'foobar': ('barfoo', '0.0'),  # deprecated since forever
             # won't be actually deprecated for a while;
             # note that we should map foobarbarfoo to a valid easyconfig parameter here,
             # or we'll hit errors when parsing an easyconfig file that uses it
             'foobarbarfoo': ('required_linked_shared_libs', '1000000000'),
-        })
-
-        # copy classes before reloading, so we can restore them (otherwise isinstance checks fail)
-        orig_EasyConfig = copy.deepcopy(easyconfig.easyconfig.EasyConfig)
-        orig_ActiveMNS = copy.deepcopy(easyconfig.easyconfig.ActiveMNS)
-        reload(easyconfig.parser)
+        }
 
-        for key, (newkey, depr_ver) in easyconfig.parser.DEPRECATED_PARAMETERS.items():
+        for key, (newkey, depr_ver) in easyconfig.easyconfig.DEPRECATED_PARAMETERS.items():
             if LooseVersion(depr_ver) <= easybuild.tools.build_log.CURRENT_VERSION:
                 # deprecation error
                 error_regex = "DEPRECATED.*since v%s.*'%s' is deprecated.*use '%s' instead" % (depr_ver, key, newkey)
@@ -1802,12 +1894,13 @@ def foo(key):
                 self.assertErrorRegex(EasyBuildError, error_regex, foo, key)
             else:
                 # only deprecation warning, but key is replaced when getting/setting
-                ec[key] = 'test123'
-                self.assertEqual(ec[newkey], 'test123')
-                self.assertEqual(ec[key], 'test123')
-                ec[newkey] = '123test'
-                self.assertEqual(ec[newkey], '123test')
-                self.assertEqual(ec[key], '123test')
+                with self.mocked_stdout_stderr():
+                    ec[key] = 'test123'
+                    self.assertEqual(ec[newkey], 'test123')
+                    self.assertEqual(ec[key], 'test123')
+                    ec[newkey] = '123test'
+                    self.assertEqual(ec[newkey], '123test')
+                    self.assertEqual(ec[key], '123test')
 
         variables = {
             'name': 'example',
@@ -1838,12 +1931,6 @@ def foo(key):
             ec = EasyConfig(test_ec)
         self.assertEqual(ec['required_linked_shared_libs'], 'foobarbarfoo')
 
-        easyconfig.parser.DEPRECATED_PARAMETERS = orig_deprecated_parameters
-        reload(easyconfig.parser)
-        reload(easyconfig.easyconfig)
-        easyconfig.easyconfig.EasyConfig = orig_EasyConfig
-        easyconfig.easyconfig.ActiveMNS = orig_ActiveMNS
-
     def test_unknown_easyconfig_parameter(self):
         """Check behaviour when unknown easyconfig parameters are used."""
         self.contents = '\n'.join([