create symlink from lib64 to lib subdir in installation directories to avoid that GCC prefers /lib64 system directories

easybuild/framework/easyblock.py

@@ -76,7 +76,7 @@
 from easybuild.tools.filetools import diff_files, dir_contains_files, download_file, encode_class_name, extract_file
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
-from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file
+from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
 from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
@@ -2348,6 +2348,15 @@ def post_install_step(self):
                 run_cmd(cmd, simple=True, log_ok=True, log_all=True)
 
         self.fix_shebang()
+        # GCC linker searches system /lib64 path before the $LIBRARY_PATH paths.
+        # However for each <dir> in $LIBRARY_PATH (where <dir> is often <prefix>/lib) it searches <dir>/../lib64 first.
+        # So we create <prefix>/lib64 as a symlink to <prefix>/lib to make it prefer EB installed libraries.
+        # See https://github.com/easybuilders/easybuild-easyconfigs/issues/5776
+        if build_option('lib64_lib_symlink'):
+            lib_dir = os.path.join(self.installdir, 'lib')
+            lib64_dir = os.path.join(self.installdir, 'lib64')
+            if os.path.exists(lib_dir) and not os.path.exists(lib64_dir):
+                symlink(lib_dir, lib64_dir)
 
     def sanity_check_step(self, *args, **kwargs):
         """

easybuild/tools/config.py

@@ -267,6 +267,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'extended_dry_run_ignore_errors',
         'fixed_installdir_naming_scheme',
         'lib64_fallback_sanity_check',
+        'lib64_lib_symlink',
         'mpi_tests',
         'map_toolchains',
         'modules_tool_version_check',

easybuild/tools/options.py

@@ -400,6 +400,8 @@ def override_options(self):
             'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False),
             'lib64-fallback-sanity-check': ("Fallback in sanity check to lib64/ equivalent for missing libraries",
                                             None, 'store_true', True),
+            'lib64-lib-symlink': ("Automatically create symlinks for lib64/ pointing to lib/ if the former is missing",
+                                  None, 'store_true', True),
             'max-fail-ratio-adjust-permissions': ("Maximum ratio for failures to allow when adjusting permissions",
                                                   'float', 'store', DEFAULT_MAX_FAIL_RATIO_PERMS),
             'minimal-toolchains': ("Use minimal toolchain when resolving dependencies", None, 'store_true', False),

test/framework/toy_build.py

@@ -1969,7 +1969,7 @@ def test_sanity_check_paths_lib64(self):
         ectxt = read_file(ec_file)
 
         # modify test easyconfig: move lib/libtoy.a to lib64/libtoy.a
-        ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy', ('lib/libtoy.a', 'lib/libfoo.a')],", ectxt)
+        ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', ('lib/libtoy.a', 'lib/libfoo.a')],", ectxt)
         postinstallcmd = "mkdir %(installdir)s/lib64 && mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a"
         ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt)
 
@@ -1979,54 +1979,58 @@ def test_sanity_check_paths_lib64(self):
         # sanity check fails if lib64 fallback in sanity check is disabled
         error_pattern = r"Sanity check failed: no file found at 'lib/libtoy.a' or 'lib/libfoo.a' in "
         self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
-                              extra_args=['--disable-lib64-fallback-sanity-check'], raise_error=True, verbose=False)
+                              extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
+                              raise_error=True, verbose=False)
 
         # all is fine is lib64 fallback check is enabled (which it is by default)
         self.test_toy_build(ec_file=test_ec, raise_error=True)
 
         # also check with 'lib' in sanity check dirs (special case)
-        ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy'],", ectxt)
-        ectxt = re.sub("\s*'dirs'.*", "'dirs': ['lib'],", ectxt)
+        ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy'],", ectxt)
+        ectxt = re.sub(r"\s*'dirs'.*", "'dirs': ['lib'],", ectxt)
         write_file(test_ec, ectxt)
 
         error_pattern = r"Sanity check failed: no \(non-empty\) directory found at 'lib' in "
         self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
-                              extra_args=['--disable-lib64-fallback-sanity-check'], raise_error=True, verbose=False)
+                              extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
+                              raise_error=True, verbose=False)
 
-        self.test_toy_build(ec_file=test_ec, raise_error=True)
+        self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
 
         # also check other way around (lib64 -> lib)
         ectxt = read_file(ec_file)
-        ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy', 'lib64/libtoy.a'],", ectxt)
+        ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', 'lib64/libtoy.a'],", ectxt)
         write_file(test_ec, ectxt)
 
         # sanity check fails if lib64 fallback in sanity check is disabled, since lib64/libtoy.a is not there
         error_pattern = r"Sanity check failed: no file found at 'lib64/libtoy.a' in "
         self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
-                              extra_args=['--disable-lib64-fallback-sanity-check'], raise_error=True, verbose=False)
+                              extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
+                              raise_error=True, verbose=False)
 
         # sanity check passes when lib64 fallback is enabled (by default), since lib/libtoy.a is also considered
-        self.test_toy_build(ec_file=test_ec, raise_error=True)
+        self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
 
         # also check with 'lib64' in sanity check dirs (special case)
-        ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy'],", ectxt)
-        ectxt = re.sub("\s*'dirs'.*", "'dirs': ['lib64'],", ectxt)
+        ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy'],", ectxt)
+        ectxt = re.sub(r"\s*'dirs'.*", "'dirs': ['lib64'],", ectxt)
         write_file(test_ec, ectxt)
 
         error_pattern = r"Sanity check failed: no \(non-empty\) directory found at 'lib64' in "
         self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
-                              extra_args=['--disable-lib64-fallback-sanity-check'], raise_error=True, verbose=False)
+                              extra_args=['--disable-lib64-fallback-sanity-check', '--disable-lib64-lib-symlink'],
+                              raise_error=True, verbose=False)
 
-        self.test_toy_build(ec_file=test_ec, raise_error=True)
+        self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
 
         # check whether fallback works for files that's more than 1 subdir deep
         ectxt = read_file(ec_file)
-        ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy', 'lib/test/libtoy.a'],", ectxt)
+        ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', 'lib/test/libtoy.a'],", ectxt)
         postinstallcmd = "mkdir -p %(installdir)s/lib64/test && "
         postinstallcmd += "mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/test/libtoy.a"
         ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt)
         write_file(test_ec, ectxt)
-        self.test_toy_build(ec_file=test_ec, raise_error=True)
+        self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True)
 
     def test_toy_build_enhanced_sanity_check(self):
         """Test enhancing of sanity check."""
@@ -3051,6 +3055,35 @@ def test_toy_build_unicode_description(self):
 
         self.test_toy_build(ec_file=test_ec, raise_error=True)
 
+    def test_test_toy_build_lib64_symlink(self):
+        """Check whether lib64 symlink to lib subdirectory is created."""
+        # this is done to ensure that <installdir>/lib64 is considered before /lib64 by GCC linker,
+        # see https://github.com/easybuilders/easybuild-easyconfigs/issues/5776
+
+        # by default, lib64 symlink is created
+        self.test_toy_build()
+
+        toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
+        lib_path = os.path.join(toy_installdir, 'lib')
+        lib64_path = os.path.join(toy_installdir, 'lib64')
+
+        self.assertTrue(os.path.exists(lib_path))
+        self.assertTrue(os.path.exists(lib64_path))
+        self.assertTrue(os.path.isdir(lib_path))
+        self.assertFalse(os.path.islink(lib_path))
+        self.assertTrue(os.path.islink(lib64_path))
+        self.assertTrue(os.path.samefile(lib_path, lib64_path))
+
+        # cleanup and try again with --disable-lib64-lib-symlink
+        remove_dir(self.test_installpath)
+        self.test_toy_build(extra_args=['--disable-lib64-lib-symlink'])
+
+        self.assertTrue(os.path.exists(lib_path))
+        self.assertFalse(os.path.exists(lib64_path))
+        self.assertFalse('lib64' in os.listdir(toy_installdir))
+        self.assertTrue(os.path.isdir(lib_path))
+        self.assertFalse(os.path.islink(lib_path))
+
 
 def suite():
     """ return all the tests in this file """