add support for developer mode

easybuild/framework/easyblock.py

@@ -83,7 +83,7 @@
 from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info
 from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
 from easybuild.tools.filetools import encode_class_name, extract_file, find_backup_name_candidate
-from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url
+from easybuild.tools.filetools import copy_dir, get_cwd, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
@@ -182,6 +182,19 @@ def __init__(self, ec):
         else:
             raise EasyBuildError("Value of incorrect type passed to EasyBlock constructor: %s ('%s')", type(ec), ec)
 
+        # are we running in developer mode?
+        self.developer = build_option('developer')
+
+        # This is needed in case custom easyblocks do a reparse of the EC file (e.g. quantum espresso)
+        if self.developer and not self.version.endswith('-dev'):
+            old_vers = ec['version']
+            new_vers = old_vers + '-dev'
+            ec['version'] = new_vers
+            for key in ['short_mod_name', 'full_mod_name']:
+                prev = getattr(ec, key)
+                new = prev.replace(f'/{old_vers}', f'/{new_vers}')
+                setattr(ec, key, new)
+
         # modules interface with default MODULEPATH
         self.modules_tool = self.cfg.modules_tool
         # module generator
@@ -2654,15 +2667,31 @@ def extract_step(self):
         """
         Unpack the source files.
         """
-        for src in self.src:
-            self.log.info("Unpacking source %s" % src['name'])
-            srcdir = extract_file(src['path'], self.builddir, cmd=src['cmd'],
-                                  extra_options=self.cfg['unpack_options'], change_into_dir=False)
-            change_dir(srcdir)
-            if srcdir:
-                self.src[self.src.index(src)]['finalpath'] = srcdir
+        developer_pth = self.developer
+        if developer_pth:
+            if self.start_dir:
+                dst_name = self.start_dir
             else:
-                raise EasyBuildError("Unpacking source %s failed", src['name'])
+                dst_name = os.path.basename(developer_pth)
+            dest = os.path.join(self.builddir, dst_name)
+            copy_dir(
+                developer_pth, dest,
+                ignore=lambda pth, elem: [_ for _ in elem if _ in ['.git', '.svn', '.hg']]
+            )
+            self.cfg['start_dir'] = dest
+            self.log.info("Content of develop path %s copied to %s (new start_dir)", developer_pth, dest)
+        else:
+            for src in self.src:
+                self.log.info("Unpacking source %s" % src['name'])
+                srcdir = extract_file(
+                    src['path'], self.builddir, cmd=src['cmd'],
+                    extra_options=self.cfg['unpack_options'], change_into_dir=False
+                )
+                change_dir(srcdir)
+                if srcdir:
+                    self.src[self.src.index(src)]['finalpath'] = srcdir
+                else:
+                    raise EasyBuildError("Unpacking source %s failed", src['name'])
 
     def patch_step(self, beginpath=None, patches=None):
         """
@@ -4036,6 +4065,12 @@ def skip_step(self, step, skippable):
         skip_test_step = build_option('skip_test_step')
         skipsteps = self.cfg['skipsteps']
 
+        developer_skip = (
+            FETCH_STEP,
+            PATCH_STEP,
+            EXTENSIONS_STEP,
+        )
+
         # under --skip, sanity check is not skipped
         cli_skip = self.skip and step != SANITYCHECK_STEP
 
@@ -4071,6 +4106,9 @@ def skip_step(self, step, skippable):
         elif skip_test_step and step == TEST_STEP:
             self.log.info("Skipping %s step as requested via skip-test-step", step)
             skip = True
+        elif self.developer and step in developer_skip:
+            self.log.info("Skipping %s step because of --developer", step)
+            skip = True
 
         else:
             msg = "Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s, "

easybuild/main.py

@@ -60,7 +60,7 @@
 from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
 from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
-from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
+from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option, update_build_option
 from easybuild.tools.containers.common import containerize
 from easybuild.tools.docs import list_software
 from easybuild.tools.environment import restore_env
@@ -445,6 +445,18 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
         options.inject_checksums, options.inject_checksums_to_json, options.sanity_check_only
     ))
 
+    if build_option('developer'):
+        for ec in easyconfigs:
+            _ec = ec['ec']
+            old_vers = _ec['version']
+            new_vers = old_vers + '-dev'
+            _ec['version'] = new_vers
+            for key in ['short_mod_name', 'full_mod_name']:
+                prev = getattr(_ec, key)
+                new = prev.replace(f'/{old_vers}', f'/{new_vers}')
+                ec[key] = new
+                setattr(_ec, key, new)
+
     # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
     if not keep_available_modules:
         retained_ecs = skip_available(easyconfigs, modtool)
@@ -691,6 +703,16 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
         index = load_index(options.create_index)
         print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False)
 
+    if options.developer:
+        pth = build_option('developer')
+        if not os.path.exists(pth):
+            raise EasyBuildError("Developer mode path %s does not exist" % pth)
+
+        pth = os.path.abspath(pth)
+        options.developer = pth
+        update_build_option('developer', pth)
+        print_msg("Developer mode running from %s" % build_option('developer'), log=_log)
+
     # non-verbose cleanup after handling GitHub integration stuff or printing terse info
     early_stop_options = [
         options.add_pr_labels,

easybuild/tools/config.py

@@ -218,6 +218,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'cuda_cache_dir',
         'cuda_cache_maxsize',
         'cuda_compute_capabilities',
+        'developer',
         'dump_test_report',
         'easyblock',
         'envvars_user_modules',

easybuild/tools/options.py

@@ -265,6 +265,7 @@ def basic_options(self):
         descr = ("Basic options", "Basic runtime options for EasyBuild.")
 
         opts = OrderedDict({
+            'developer': ("This is a test option", None, 'store_or_None', '.'),
             'dry-run': ("Print build overview incl. dependencies (full paths)", None, 'store_true', False),
             'dry-run-short': ("Print build overview incl. dependencies (short paths)", None, 'store_true', False, 'D'),
             'extended-dry-run': ("Print build environment and (expected) build procedure that will be performed",