also pick up on toolchain components in Toolchain._add_dependency_variables

easybuild/tools/toolchain/toolchain.py

@@ -219,7 +219,7 @@ def __init__(self, name=None, version=None, mns=None, class_constants=None, tcde
 
         self._init_class_constants(class_constants)
 
-        self.tcdeps = tcdeps
+        self.tcdeps = tcdeps if tcdeps else []
 
         # toolchain instances are created before initiating build options sometimes, e.g. for --list-toolchains
         self.dry_run = build_option('extended_dry_run', default=False)
@@ -628,13 +628,12 @@ def _load_toolchain_module(self, silent=False):
                 dry_run_msg("module load %s" % tc_mod, silent=silent)
             else:
                 # first simulate loads for toolchain dependencies, if required information is available
-                if self.tcdeps is not None:
-                    for tcdep in self.tcdeps:
-                        modname = tcdep['short_mod_name']
-                        dry_run_msg("module load %s [SIMULATED]" % modname, silent=silent)
-                        # 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
-                        deproot = '$%s' % get_software_root_env_var_name(tcdep['name'])
-                        self._simulated_load_dependency_module(tcdep['name'], tcdep['version'], {'prefix': deproot})
+                for tcdep in self.tcdeps:
+                    modname = tcdep['short_mod_name']
+                    dry_run_msg("module load %s [SIMULATED]" % modname, silent=silent)
+                    # 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
+                    deproot = '$%s' % get_software_root_env_var_name(tcdep['name'])
+                    self._simulated_load_dependency_module(tcdep['name'], tcdep['version'], {'prefix': deproot})
 
                 dry_run_msg("module load %s [SIMULATED]" % tc_mod, silent=silent)
                 # use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output)
@@ -1140,21 +1139,21 @@ def _add_dependency_variables(self, names=None, cpp=None, ld=None):
         :names: list of strings containing the name of the dependency
         """
         # collect dependencies
-        dependencies = self.dependencies if names is None else [{"name": name} for name in names if name]
+        deps = self.dependencies if names is None else [{'name': name} for name in names if name]
 
-        # collect software install prefixes for dependencies
-        dependency_roots = []
-        for dep in dependencies:
-            if dep.get("external_module", False):
+        # collect software install prefixes for toolchain components + dependencies
+        dep_roots = []
+        for dep in deps + self.tcdeps:
+            if dep.get('external_module', False):
                 # for software names provided via external modules, install prefix may be unknown
-                names = dep["external_module_metadata"].get("name", [])
-                dependency_roots.extend([root for root in self.get_software_root(names) if root is not None])
+                names = dep['external_module_metadata'].get('name', [])
+                dep_roots.extend([x for x in self.get_software_root(names) if x is not None])
             else:
-                dependency_roots.extend(self.get_software_root(dep["name"]))
+                dep_roots.extend(self.get_software_root(dep['name']))
 
-        for root in dependency_roots:
-            self._add_dependency_cpp_headers(root, extra_dirs=cpp)
-            self._add_dependency_linker_paths(root, extra_dirs=ld)
+        for dep_root in dep_roots:
+            self._add_dependency_cpp_headers(dep_root, extra_dirs=cpp)
+            self._add_dependency_linker_paths(dep_root, extra_dirs=ld)
 
     def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None):
         """

test/framework/easyblock.py

@@ -3720,7 +3720,7 @@ def test_exts_deps_build_env(self):
         test_ec = os.path.join(self.test_prefix, 'test.eb')
         test_ec_txt = read_file(toy_ec)
         test_ec_txt += textwrap.dedent("""
-            toolchain = {'name': 'GCCcore', 'version': '12.3.0'}
+            toolchain = {'name': 'gompi', 'version': '2023a'}
 
             dependencies = [
                 ('zlib', '1.2.13'),
@@ -3739,23 +3739,34 @@ def test_exts_deps_build_env(self):
         """)
         write_file(test_ec, test_ec_txt)
 
-        # put dummy zlib module in place where we can control $EBROOTZLIB value
-        zlib_mod_file = os.path.join(testdir, 'modules', 'zlib', '1.2.13-GCCcore-12.3.0')
-        zlib_fn = os.path.basename(zlib_mod_file)
+        # put dummy modules in place where we can control $EBROOT value
+        openmpi_fn = '4.1.5-GCC-12.3.0'
+        zlib_fn = '1.2.13-GCCcore-12.3.0'
+        mod_files = [
+            ('OpenMPI', openmpi_fn),
+            ('zlib', zlib_fn),
+        ]
+
+        test_mods = os.path.join(self.test_prefix, 'modules')
 
-        zlib_root = os.path.join(self.test_prefix, 'software', 'zlib', zlib_fn)
-        write_file(os.path.join(zlib_root, 'include', 'zlib.h'), '')
-        write_file(os.path.join(zlib_root, 'include', 'zlib', 'common.h'), '')
+        for name, mod_fn in mod_files:
+            mod_fp = os.path.join(testdir, 'modules', name, mod_fn)
 
-        zlib_mod_txt = read_file(zlib_mod_file)
-        zlib_mod_txt = re.sub("set root.*", f"set root {zlib_root}", zlib_mod_txt)
-        # add statement to inject extra subdirectory to $CPATH,
-        # which is supposed to be retained in build environment
-        zlib_mod_txt += '\nprepend-path\tCPATH\t$root/include/zlib'
+            header_fn = 'zlib.h' if name == 'zlib' else 'mpi.h'
+
+            dep_root = os.path.join(self.test_prefix, 'software', name, mod_fn)
+            write_file(os.path.join(dep_root, 'include', header_fn), '')
+            write_file(os.path.join(dep_root, 'include', name, 'common.h'), '')
+
+            mod_txt = read_file(mod_fp)
+            mod_txt = re.sub("set root.*", f"set root {dep_root}", mod_txt)
+            # add statement to inject extra subdirectory to $CPATH,
+            # which is supposed to be retained in build environment
+            mod_txt += f'\nprepend-path\tCPATH\t$root/include/{name}'
+
+            test_mod_file = os.path.join(test_mods, name, mod_fn)
+            write_file(test_mod_file, mod_txt)
 
-        test_mods = os.path.join(self.test_prefix, 'modules')
-        test_zlib_mod_file = os.path.join(test_mods, 'zlib', zlib_fn)
-        write_file(test_zlib_mod_file, zlib_mod_txt)
         self.modtool.use(test_mods)
 
         env_vars = {
@@ -3771,7 +3782,6 @@ def test_exts_deps_build_env(self):
                 f'--search-path-cpp-headers={search_path_cpp_headers}',
                 '--debug',
             ]
-            os.environ['C_INCLUDE_PATH'] = '/usr/local/include'
             with self.mocked_stdout_stderr():
                 with self.log_to_testlogfile():
                     self.eb_main(args, raise_error=True, do_build=True, verbose=True)
@@ -3785,7 +3795,12 @@ def test_exts_deps_build_env(self):
             # check whether $C_INCLUDE_PATH is correctly set in build environment of 'bar' extension
             for env_var in env_vars[search_path_cpp_headers]:
                 # both 'include' and 'include/zlib' subdirectories should be retained
-                paths = [f'software/zlib/{zlib_fn}/include/zlib', f'software/zlib/{zlib_fn}/include']
+                paths = [
+                    f'software/OpenMPI/{openmpi_fn}/include/OpenMPI',
+                    f'software/OpenMPI/{openmpi_fn}/include',
+                    f'software/zlib/{zlib_fn}/include/zlib',
+                    f'software/zlib/{zlib_fn}/include',
+                ]
                 if env_var.endswith('PATH'):
                     regex = re.compile(f'^{env_var}=' + ':'.join('[^ ]+/' + p for p in paths) + '$', re.M)
                 elif env_var == 'CPPFLAGS':

test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.40-GCCcore-12.3.0.eb

@@ -0,0 +1,14 @@
+# minimal easyconfig, only really neededd so toolchain hierarchy for gompi/2023a can be determined
+
+# should be software specific, but OK for testing purposes
+easyblock = 'EB_toy'
+
+name = 'binutils'
+version = '2.26'
+
+homepage = 'http://directory.fsf.org/project/binutils/'
+description = "binutils: GNU binary utilities"
+
+toolchain = {'name': 'GCCcore', 'version': '12.3.0'}
+
+moduleclass = 'tools'

test/framework/easyconfigs/test_ecs/g/GCC/GCC-12.3.0.eb

@@ -0,0 +1,21 @@
+# minimal easyconfig, only really neededd so toolchain hierarchy for gompi/2023a can be determined
+
+# should be EB_GCC, but OK for testing purposes
+easyblock = 'EB_toy'
+
+name = 'GCC'
+version = '12.3.0'
+
+homepage = 'http://gcc.gnu.org/'
+description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran,
+ Java, and Ada, as well as libraries for these languages (libstdc++, libgcj,...)."""
+
+toolchain = SYSTEM
+
+dependencies = [
+    ('GCCcore', version),
+    # binutils built on top of GCCcore, which was built on top of (system-built) binutils
+    ('binutils', '2.40', '', ('GCCcore', version)),
+]
+
+moduleclass = 'compiler'

test/framework/easyconfigs/test_ecs/g/gompi/gompi-2023a.eb

@@ -0,0 +1,20 @@
+easyblock = 'Toolchain'
+
+name = 'gompi'
+version = '2023'
+
+homepage = '(none)'
+description = """GNU Compiler Collection (GCC) based compiler toolchain,
+ including OpenMPI for MPI support."""
+
+toolchain = SYSTEM
+
+local_comp = ('GCC', '12.3.0')
+
+# compiler toolchain dependencies
+dependencies = [
+    local_comp,
+    ('OpenMPI', '4.1.5', '', local_comp),
+]
+
+moduleclass = 'toolchain'

test/framework/easyconfigs/test_ecs/o/OpenMPI/OpenMPI-4.1.5-GCC-12.3.0.eb

@@ -0,0 +1,13 @@
+# minimal easyconfig, only really neededd so toolchain hierarchy for gompi/2023a can be determined
+
+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.5'
+
+homepage = 'http://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-2 implementation."""
+
+toolchain = {'name': 'GCC', 'version': '12.3.0'}
+
+moduleclass = 'mpi'

test/framework/filetools.py

@@ -2294,8 +2294,8 @@ def test_copy_dir(self):
 
         ft.copy_dir(to_copy, target_dir, ignore=lambda src, names: [x for x in names if '6.4.0-2.28' in x])
         self.assertExists(target_dir)
-        expected = ['GCC-10.2.0.eb', 'GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', 'GCC-4.9.2.eb',
-                    'GCC-4.9.3-2.25.eb', 'GCC-4.9.3-2.26.eb', 'GCC-7.3.0-2.30.eb']
+        expected = ['GCC-10.2.0.eb', 'GCC-12.3.0.eb', 'GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.8.2.eb',
+                    'GCC-4.8.3.eb', 'GCC-4.9.2.eb', 'GCC-4.9.3-2.25.eb', 'GCC-4.9.3-2.26.eb', 'GCC-7.3.0-2.30.eb']
         self.assertEqual(sorted(os.listdir(target_dir)), expected)
         # GCC-6.4.0-2.28.eb should not get copied, since it's specified as file too ignore
         self.assertNotExists(os.path.join(target_dir, 'GCC-6.4.0-2.28.eb'))
@@ -2702,7 +2702,7 @@ def test_index_functions(self):
         # test with specified path with and without trailing '/'s
         for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
             index = ft.create_index(path)
-            self.assertEqual(len(index), 100)
+            self.assertEqual(len(index), 104)
 
             expected = [
                 os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),
@@ -2752,7 +2752,7 @@ def test_index_functions(self):
         regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir)
         self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))
 
-        self.assertEqual(len(index), 29)
+        self.assertEqual(len(index), 31)
         for fn in expected:
             self.assertIn(fn, index)
 
@@ -2782,7 +2782,7 @@ def test_index_functions(self):
         regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir)
         self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))
 
-        self.assertEqual(len(index), 29)
+        self.assertEqual(len(index), 31)
         for fn in expected:
             self.assertIn(fn, index)
 

test/framework/tweak.py

@@ -80,7 +80,7 @@ def test_find_matching_easyconfigs(self):
             self.assertTrue(len(ecs) == 1 and ecs[0].endswith('/%s-%s.eb' % (name, installver)))
 
         ecs = find_matching_easyconfigs('GCC', '*', [test_easyconfigs_path])
-        gccvers = ['10.2.0', '4.6.3', '4.6.4', '4.8.2', '4.8.3', '4.9.2', '4.9.3-2.25',
+        gccvers = ['10.2.0', '12.3.0', '4.6.3', '4.6.4', '4.8.2', '4.8.3', '4.9.2', '4.9.3-2.25',
                    '4.9.3-2.26', '6.4.0-2.28', '7.3.0-2.30']
         self.assertEqual(len(ecs), len(gccvers))
         ecs_basename = [os.path.basename(ec) for ec in ecs]
@@ -128,7 +128,7 @@ def test_obtain_ec_for(self):
         }
         (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
         self.assertFalse(generated)
-        self.assertEqual(os.path.basename(ec_file), 'GCC-10.2.0.eb')
+        self.assertEqual(os.path.basename(ec_file), 'GCC-12.3.0.eb')
 
         # generate non-existing easyconfig
         change_dir(self.test_prefix)