allow extensions to add text to the module file

easybuild/framework/easyblock.py

@@ -166,7 +166,6 @@ def __init__(self, ec):
 
         # extensions
         self.exts = []
-        self.exts_all = None
         self.ext_instances = []
         self.skip = None
         self.module_extra_extensions = ''  # extra stuff for module file required by extensions
@@ -1463,48 +1462,58 @@ def make_module_extra_extensions(self):
         Sets optional variables for extensions.
         """
         # add stuff specific to individual extensions
-        lines = [self.module_extra_extensions]
+        txt = self.module_extra_extensions
+
+        if not self.ext_instances:
+            self.prepare_for_extensions()
+            self.init_ext_instances()
+
+        for ext in self.ext_instances:
+            ext_txt = ext.make_extension_module_extra()
+            if ext_txt:
+                txt += ext_txt
 
         # set environment variable that specifies list of extensions
         # We need only name and version, so don't resolve templates
         exts_list = self.make_extension_string(ext_sep=',', sort=False)
         env_var_name = convert_name(self.name, upper=True)
-        lines.append(self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list))
+        txt += self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list)
 
-        return ''.join(lines)
+        return txt
 
     def make_module_footer(self):
         """
         Insert a footer section in the module file, primarily meant for contextual information
         """
-        footer = [self.module_generator.comment("Built with EasyBuild version %s" % VERBOSE_VERSION)]
+        footer = []
 
         # add extra stuff for extensions (if any)
         if self.cfg.get_ref('exts_list'):
             footer.append(self.make_module_extra_extensions())
 
         # include modules footer if one is specified
-        if self.modules_footer is not None:
+        if self.modules_footer:
             self.log.debug("Including specified footer into module: '%s'" % self.modules_footer)
             footer.append(self.modules_footer)
 
         if self.cfg['modtclfooter']:
             if isinstance(self.module_generator, ModuleGeneratorTcl):
                 self.log.debug("Including Tcl footer in module: %s", self.cfg['modtclfooter'])
-                footer.extend([self.cfg['modtclfooter'], '\n'])
+                footer.append(self.cfg['modtclfooter'])
             else:
                 self.log.warning("Not including footer in Tcl syntax in non-Tcl module file: %s",
                                  self.cfg['modtclfooter'])
 
         if self.cfg['modluafooter']:
             if isinstance(self.module_generator, ModuleGeneratorLua):
                 self.log.debug("Including Lua footer in module: %s", self.cfg['modluafooter'])
-                footer.extend([self.cfg['modluafooter'], '\n'])
+                footer.append(self.cfg['modluafooter'])
             else:
                 self.log.warning("Not including footer in Lua syntax in non-Lua module file: %s",
                                  self.cfg['modluafooter'])
 
-        return ''.join(footer)
+        footer.append(self.module_generator.comment("Built with EasyBuild version %s" % VERBOSE_VERSION))
+        return '\n'.join(footer)
 
     def make_module_extend_modpath(self):
         """
@@ -1974,7 +1983,7 @@ def install_extensions_parallel(self, install=True):
         running_exts = []
         installed_ext_names = []
 
-        all_ext_names = [x['name'] for x in self.exts_all]
+        all_ext_names = [x['name'] for x in self.exts]
         self.log.debug("List of names of all extensions: %s", all_ext_names)
 
         # take into account that some extensions may be installed already
@@ -2877,6 +2886,17 @@ def extensions_step(self, fetch=False, install=True):
             self.log.debug("No extensions in exts_list")
             return
 
+        self.prepare_for_extensions()
+
+        if fetch:
+            self.update_exts_progress_bar("fetching extension sources/patches")
+            self.exts = self.collect_exts_file_info(fetch_files=True)
+
+        # we really need a default class
+        if not self.cfg['exts_defaultclass']:
+            raise EasyBuildError("ERROR: No default extension class set for %s", self.name)
+        self.init_ext_instances()
+
         # load fake module
         fake_mod_data = None
         if install and not self.dry_run:
@@ -2888,25 +2908,10 @@ def extensions_step(self, fetch=False, install=True):
 
         start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg['exts_list']))
 
-        self.prepare_for_extensions()
-
-        if fetch:
-            self.update_exts_progress_bar("fetching extension sources/patches")
-            self.exts = self.collect_exts_file_info(fetch_files=True)
-
-        self.exts_all = self.exts[:]  # retain a copy of all extensions, regardless of filtering/skipping
-
         # actually install extensions
         if install:
             self.log.info("Installing extensions")
 
-        # we really need a default class
-        if not self.cfg['exts_defaultclass'] and fake_mod_data:
-            self.clean_up_fake_module(fake_mod_data)
-            raise EasyBuildError("ERROR: No default extension class set for %s", self.name)
-
-        self.init_ext_instances()
-
         if self.skip:
             self.skip_extensions()
 

easybuild/framework/extension.py

@@ -247,6 +247,10 @@ def toolchain(self):
         """
         return self.master.toolchain
 
+    def make_extension_module_extra(self):
+        """Similar to make_module_extra but only called for extensions"""
+        return ''
+
     def sanity_check_step(self):
         """
         Sanity check to run after installing extension

easybuild/framework/extensioneasyblock.py

@@ -217,6 +217,6 @@ def make_module_extra(self, extra=None):
         """Add custom entries to module."""
 
         txt = EasyBlock.make_module_extra(self)
-        if extra is not None:
+        if extra:
             txt += extra
         return txt

test/framework/easyconfig.py

@@ -475,6 +475,7 @@ def test_exts_list(self):
             homepage = "http://example.com"
             description = "test easyconfig"
             toolchain = SYSTEM
+            exts_defaultclass = "DummyExtension"
             exts_default_options = {
                 "source_tmpl": "gzip-1.4.eb", # dummy source template to avoid download fail
                 "source_urls": ["http://example.com/%(name)s/%(version)s"]

test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py

@@ -99,6 +99,15 @@ def postrun(self):
 
         EB_toy.install_step(self.master, name=self.name)
 
+    def make_extension_module_extra(self):
+        """Extra stuff for toy extensions"""
+        txt = super(Toy_Extension, self).make_extension_module_extra()
+        value = self.name
+        if self.version:
+            value += '-' + self.version
+        txt += self.module_generator.set_environment('TOY_EXT_VAR', value)
+        return txt
+
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for toy extensions."""
         self.log.info("Loaded modules: %s", self.modules_tool.list())

test/framework/toy_build.py

@@ -1105,6 +1105,8 @@ def test_toy_advanced(self):
             # set by ToyExtension easyblock used to install extensions
             '^setenv.*TOY_EXT_BAR.*bar',
             '^setenv.*TOY_EXT_BARBAR.*barbar',
+            '^setenv.*TOY_EXT_VAR.*bar',
+            '^setenv.*TOY_EXT_VAR.*barbar',
         ]
         for pattern in patterns:
             self.assertTrue(re.search(pattern, toy_mod_txt, re.M), "Pattern '%s' found in: %s" % (pattern, toy_mod_txt))
@@ -1530,8 +1532,8 @@ def test_toy_module_fulltxt(self):
             ] + modloadmsg_lua + [
                 r'end',
                 r'setenv\("TOY", "toy-0.0"\)',
+                r'io.stderr:write\("oh hai\!"\)',
                 r'-- Built with EasyBuild version .*',
-                r'io.stderr:write\("oh hai\!"\)$',
             ])
         elif get_module_syntax() == 'Tcl':
             mod_txt_regex_pattern = '\n'.join([
@@ -1571,15 +1573,13 @@ def test_toy_module_fulltxt(self):
             ] + modloadmsg_tcl + [
                 r'}',
                 r'setenv	TOY		"toy-0.0"',
+                r'puts stderr "oh hai\!"',
                 r'# Built with EasyBuild version .*',
-                r'puts stderr "oh hai\!"$',
             ])
         else:
             self.fail("Unknown module syntax: %s" % get_module_syntax())
 
-        mod_txt_regex = re.compile(mod_txt_regex_pattern)
-        msg = "Pattern '%s' matches with: %s" % (mod_txt_regex.pattern, toy_mod_txt)
-        self.assertTrue(mod_txt_regex.match(toy_mod_txt), msg)
+        self.assertRegex(toy_mod_txt, mod_txt_regex_pattern)
 
     def test_external_dependencies(self):
         """Test specifying external (build) dependencies."""
@@ -1804,6 +1804,8 @@ def test_module_only_extensions(self):
         # first try normal --module-only, should work fine
         self.eb_main([test_ec, '--module-only'], do_build=True, raise_error=True)
         self.assertExists(toy_mod)
+        # Extra stuff from extension(s) is included
+        self.assertRegex(read_file(toy_mod), 'TOY_EXT_VAR.*barbar-0.0')
         remove_file(toy_mod)
 
         # rename file required for barbar extension, so we can check whether sanity check catches it
@@ -1858,6 +1860,13 @@ def test_toy_exts_parallel(self):
         stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
         self.assertEqual(stderr, '')
 
+        # Extra stuff from extension(s) should be included
+        ext_var_patterns = [
+            'TOY_EXT_VAR.*ls',
+            'TOY_EXT_VAR.*bar-0.0',
+            'TOY_EXT_VAR.*barbar-0.0',
+        ]
+
         # take into account that each of these lines may appear multiple times,
         # in case no progress was made between checks
         patterns = [
@@ -1870,7 +1879,11 @@ def test_toy_exts_parallel(self):
         for pattern in patterns:
             regex = re.compile(pattern, re.M)
             error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout)
-            self.assertTrue(regex.search(stdout), error_msg)
+            self.assertRegex(stdout, regex)
+
+        module_contents = read_file(toy_mod)
+        for pattern in ext_var_patterns:
+            self.assertRegex(module_contents, pattern)
 
         # also test skipping of extensions in parallel
         args.append('--skip')
@@ -1890,6 +1903,10 @@ def test_toy_exts_parallel(self):
             error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout)
             self.assertTrue(regex.search(stdout), error_msg)
 
+        module_contents = read_file(toy_mod)
+        for pattern in ext_var_patterns:
+            self.assertRegex(module_contents, pattern)
+
         # check behaviour when using Toy_Extension easyblock that doesn't implement required_deps method;
         # framework should fall back to installing extensions sequentially
         toy_ext_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'toy_extension.py')
@@ -1917,6 +1934,10 @@ def test_toy_exts_parallel(self):
             error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout)
             self.assertTrue(regex.search(stdout), error_msg)
 
+        module_contents = read_file(toy_mod)
+        for pattern in ext_var_patterns:
+            self.assertRegex(module_contents, pattern)
+
     def test_backup_modules(self):
         """Test use of backing up of modules with --module-only."""
 
@@ -3012,13 +3033,13 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
             ' gcc toy.c -o toy  && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it!
             in post-install hook for toy v0.0
             bin, lib
-            in module-write hook hook for {mod_name}
             toy 0.0
             ['%(name)s-%(version)s.tar.gz']
             echo toy
             toy 0.0
             ['%(name)s-%(version)s.tar.gz']
             echo toy
+            in module-write hook hook for {mod_name}
             installing of extension bar is done!
             pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy '
             ' gcc toy.c -o toy  && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it!

test/framework/utilities.py

@@ -419,6 +419,10 @@ def setup_categorized_hmns_modules(self):
                 sys.stdout.write(line)
 
 
+if sys.version_info[0] < 3 or sys.version_info[1] < 1:
+    EnhancedTestCase.assertRegex = EnhancedTestCase.assertRegexpMatches
+
+
 class TestLoaderFiltered(unittest.TestLoader):
     """Test load that supports filtering of tests based on name."""