From 773dd67f2afd2db55ba40dc96d9bc9a00aa5c525 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 10 Jun 2025 17:41:30 +0200 Subject: [PATCH 01/19] New files + modifications for entrypoint features --- easybuild/framework/easyconfig/easyconfig.py | 9 + easybuild/framework/easyconfig/tools.py | 7 + easybuild/tools/config.py | 1 + easybuild/tools/entrypoints.py | 248 +++++++++++++++++++ easybuild/tools/hooks.py | 35 ++- easybuild/tools/options.py | 3 + easybuild/tools/toolchain/utilities.py | 20 ++ 7 files changed, 318 insertions(+), 5 deletions(-) create mode 100644 easybuild/tools/entrypoints.py diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index dd022fc786..87e91fe52d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -65,6 +65,7 @@ from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATES, DEPRECATED_EASYCONFIG_TEMPLATES from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools import LooseVersion +from easybuild.tools.entrypoints import get_easyblock_entrypoints, validate_easyblock_entrypoints from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning, print_msg from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN @@ -2105,6 +2106,14 @@ def get_module_path(name, generic=None, decode=True): if generic is None: generic = filetools.is_generic_easyblock(name) + invalid_eps = validate_easyblock_entrypoints() + if invalid_eps: + _log.error("Invalid easyblock entrypoints found: %s", invalid_eps) + raise EasyBuildError("Invalid easyblock entrypoints found: %s", invalid_eps) + eb_from_eps = get_easyblock_entrypoints(name) + if eb_from_eps: + return list(eb_from_eps.keys())[0] + # example: 'EB_VSC_minus_tools' should result in 'vsc_tools' if decode: name = decode_class_name(name) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 2e548298ff..8ce5a26a72 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -54,6 +54,7 @@ from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.tools import LooseVersion +from easybuild.tools.entrypoints import validate_easyblock_entrypoints, get_easyblock_entrypoints from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_error, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env @@ -799,6 +800,12 @@ def avail_easyblocks(): else: raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc) + invalid_eps = validate_easyblock_entrypoints() + if invalid_eps: + _log.error("Found invalid easyblock entry points: %s", invalid_eps) + raise EasyBuildError("Found invalid easyblock entry points: %s", invalid_eps) + easyblocks.update(get_easyblock_entrypoints()) + return easyblocks diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index cef93774a6..138e0ff153 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -351,6 +351,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'upload_test_report', 'update_modules_tool_cache', 'use_ccache', + 'use_entrypoints', 'use_existing_modules', 'use_f90cache', 'wait_on_lock_limit', diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py new file mode 100644 index 0000000000..22e8e4833d --- /dev/null +++ b/easybuild/tools/entrypoints.py @@ -0,0 +1,248 @@ +"""Python module to manage entry points for EasyBuild. + +Authors: + +* Davide Grassano (CECAM) +""" + +import importlib +from importlib.metadata import EntryPoint, entry_points +from easybuild.tools.config import build_option +from typing import Callable + +from easybuild.base import fancylogger +from easybuild.tools.build_log import EasyBuildError + +_log = fancylogger.getLogger('entrypoints', fname=False) + +def get_group_entrypoints(group: str) -> set[EntryPoint]: + """Get all entrypoints for a group""" + # print(f"--- Getting entry points for group: {group}") + # Default True needed to work with commands like --list-toolchains that do not initialize the BuildOptions + if not build_option('use_entrypoints', default=True): + return set() + return set(ep for ep in entry_points(group=group)) + +# EASYCONFIG_ENTRYPOINT = "easybuild.easyconfig" + +EASYBLOCK_ENTRYPOINT = "easybuild.easyblock" +EASYBLOCK_ENTRYPOINT_MARK = "_is_easybuild_easyblock" + +TOOLCHAIN_ENTRYPOINT = "easybuild.toolchain" +TOOLCHAIN_ENTRYPOINT_MARK = "_is_easybuild_toolchain" +TOOLCHAIN_ENTRYPOINT_PREPEND = "_prepend" + +HOOKS_ENTRYPOINT = "easybuild.hooks" +HOOKS_ENTRYPOINT_STEP = "_step" +HOOKS_ENTRYPOINT_PRE_STEP = "_pre_step" +HOOKS_ENTRYPOINT_POST_STEP = "_post_step" +HOOKS_ENTRYPOINT_MARK = "_is_easybuild_hook" +HOOKS_ENTRYPOINT_PRIORITY = "_priority" + +#########################################################################################3 +# Easyblock entrypoints +def register_easyblock_entrypoint(): + """Decorator to register an easyblock entrypoint.""" + def decorator(cls: type) -> type: + if not isinstance(cls, type): + raise EasyBuildError("Easyblock entrypoint `%s` is not a class", cls.__name__) + setattr(cls, EASYBLOCK_ENTRYPOINT_MARK, True) + _log.debug("Registering easyblock entrypoint: %s", cls.__name__) + return cls + + return decorator + + +def validate_easyblock_entrypoints() -> list[str]: + """Validate all easyblock entrypoints. + + Returns: + List of invalid easyblocks. + """ + invalid_easyblocks = [] + for ep in get_group_entrypoints(EASYBLOCK_ENTRYPOINT): + full_name = f'{ep.name} <{ep.value}>' + + eb = ep.load() + if not hasattr(eb, EASYBLOCK_ENTRYPOINT_MARK): + invalid_easyblocks.append(full_name) + _log.warning(f"Easyblock {ep.name} <{ep.value}> is not a valid EasyBuild easyblock") + continue + + if not isinstance(eb, type): + _log.warning(f"Easyblock {ep.name} <{ep.value}> is not a class") + invalid_easyblocks.append(full_name) + continue + + return invalid_easyblocks + + +def get_easyblock_entrypoints(name = None) -> dict: + """Get all easyblock entrypoints. + + Returns: + List of easyblocks. + """ + easyblocks = {} + for ep in get_group_entrypoints(EASYBLOCK_ENTRYPOINT): + try: + eb = ep.load() + except Exception as e: + _log.error(f"Error loading easyblock entry point {ep.name}: {e}") + raise EasyBuildError(f"Error loading easyblock entry point {ep.name}: {e}") + mod = importlib.import_module(eb.__module__) + + ptr = { + 'class': eb.__name__, + 'loc': mod.__file__, + } + easyblocks[f'{ep.module}'] = ptr + # print('--' * 80) + # print(easyblocks) + # print('--' * 80) + if name is not None: + for key, value in easyblocks.items(): + if value['class'] == name: + return {key: value} + if key == name: + return {key: value} + return {} + + return easyblocks + +######################################################################################### +# Hooks entrypoints +def register_entrypoint_hooks(step, pre_step=False, post_step=False, priority=0): + """Decorator to add metadata on functions to be used as hooks. + + priority: integer, the priority of the hook, higher value means higher priority + """ + def decorator(func): + setattr(func, HOOKS_ENTRYPOINT_MARK, True) + setattr(func, HOOKS_ENTRYPOINT_STEP, step) + setattr(func, HOOKS_ENTRYPOINT_PRE_STEP, pre_step) + setattr(func, HOOKS_ENTRYPOINT_POST_STEP, post_step) + setattr(func, HOOKS_ENTRYPOINT_PRIORITY, priority) + + # Register the function as an entry point + _log.info( + "Registering entry point hook '%s' 'pre=%s' 'post=%s' with priority %d", + func.__name__, pre_step, post_step, priority + ) + return func + return decorator + + +def validate_entrypoint_hooks(known_hooks: list[str], pre_prefix: str, post_prefix: str, suffix: str) -> list[str]: + """Validate all entrypoints hooks. + + Args: + known_hooks: List of known hooks. + pre_prefix: Prefix for pre hooks. + post_prefix: Prefix for post hooks. + suffix: Suffix for hooks. + + Returns: + List of invalid hooks. + """ + invalid_hooks = [] + for ep in get_group_entrypoints(HOOKS_ENTRYPOINT): + full_name = f'{ep.name} <{ep.value}>' + + hook = ep.load() + if not hasattr(hook, HOOKS_ENTRYPOINT_MARK): + invalid_hooks.append(f"{ep.name} <{ep.value}>") + _log.warning(f"Hook {ep.name} <{ep.value}> is not a valid EasyBuild hook") + continue + + if not callable(hook): + _log.warning(f"Hook {ep.name} <{ep.value}> is not callable") + invalid_hooks.append(full_name) + continue + + label = getattr(hook, HOOKS_ENTRYPOINT_STEP) + pre_cond = getattr(hook, HOOKS_ENTRYPOINT_PRE_STEP) + post_cond = getattr(hook, HOOKS_ENTRYPOINT_POST_STEP) + + prefix = '' + if pre_cond: + prefix = pre_prefix + elif post_cond: + prefix = post_prefix + + hook_name = prefix + label + suffix + + if hook_name not in known_hooks: + _log.warning(f"Hook {full_name} does not match known hooks patterns") + invalid_hooks.append(full_name) + continue + + return invalid_hooks + + +def find_entrypoint_hooks(label, pre_step_hook=False, post_step_hook=False) -> list[Callable]: + """Get all hooks defined in entry points.""" + hooks = [] + # print(f"--- Searching for entry point hooks with label: {label}, pre_step_hook: {pre_step_hook}, post_step_hook: {post_step_hook}") + for ep in get_group_entrypoints(HOOKS_ENTRYPOINT): + # print(f"--- Processing entry point: {ep.name}") + try: + hook = ep.load() + except Exception as e: + _log.error(f"Error loading entry point {ep.name}: {e}") + raise EasyBuildError(f"Error loading entry point {ep.name}: {e}") + + cond = all([ + getattr(hook, HOOKS_ENTRYPOINT_STEP) == label, + getattr(hook, HOOKS_ENTRYPOINT_PRE_STEP) == pre_step_hook, + getattr(hook, HOOKS_ENTRYPOINT_POST_STEP) == post_step_hook, + ]) + if cond: + hooks.append(hook) + + return hooks + +######################################################################################### +# Toolchain entrypoints +def register_toolchain_entrypoint(prepend=False): + def decorator(cls): + from easybuild.tools.toolchain.toolchain import Toolchain + if not isinstance(cls, type) or not issubclass(cls, Toolchain): + raise EasyBuildError("Toolchain entrypoint `%s` is not a subclass of `Toolchain`", cls.__name__) + setattr(cls, TOOLCHAIN_ENTRYPOINT_MARK, True) + setattr(cls, TOOLCHAIN_ENTRYPOINT_PREPEND, prepend) + + _log.debug("Registering toolchain entrypoint: %s", cls.__name__) + return cls + + return decorator + + +def get_toolchain_entrypoints() -> set[EntryPoint]: + """Get all toolchain entrypoints.""" + toolchains = [] + for ep in get_group_entrypoints(TOOLCHAIN_ENTRYPOINT): + try: + tc = ep.load() + except Exception as e: + _log.error(f"Error loading toolchain entry point {ep.name}: {e}") + raise EasyBuildError(f"Error loading toolchain entry point {ep.name}: {e}") + toolchains.append(tc) + # print(f"Found {len(toolchains)} toolchain entry points") + # print(f"Toolchain entry points: {toolchains}") + return toolchains + + +def validate_toolchain_entrypoints() -> list[str]: + """Validate all toolchain entrypoints.""" + invalid_toolchains = [] + for ep in get_group_entrypoints(TOOLCHAIN_ENTRYPOINT): + full_name = f'{ep.name} <{ep.value}>' + + tc = ep.load() + if not hasattr(tc, TOOLCHAIN_ENTRYPOINT_MARK): + invalid_toolchains.append(full_name) + _log.warning(f"Toolchain {ep.name} <{ep.value}> is not a valid EasyBuild toolchain") + continue + + return invalid_toolchains diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 4451439856..bee7eb5969 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -31,6 +31,13 @@ """ import difflib import os +from functools import wraps + +from easybuild.tools.entrypoints import ( + find_entrypoint_hooks, validate_entrypoint_hooks, + HOOKS_ENTRYPOINT_MARK, HOOKS_ENTRYPOINT_STEP, HOOKS_ENTRYPOINT_PRE_STEP, HOOKS_ENTRYPOINT_POST_STEP, + HOOKS_ENTRYPOINT_PRIORITY +) from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg @@ -76,6 +83,7 @@ PRE_PREF = 'pre_' POST_PREF = 'post_' HOOK_SUFF = '_hook' +ENTRYPOINT_PRE = 'entrypoint_' # list of names for steps in installation procedure (in order of execution) STEP_NAMES = [FETCH_STEP, READY_STEP, EXTRACT_STEP, PATCH_STEP, PREPARE_STEP, CONFIGURE_STEP, BUILD_STEP, TEST_STEP, @@ -176,6 +184,8 @@ def verify_hooks(hooks): """Check whether obtained hooks only includes known hooks.""" unknown_hooks = [key for key in sorted(hooks) if key not in KNOWN_HOOKS] + unknown_hooks.extend(validate_entrypoint_hooks(KNOWN_HOOKS, PRE_PREF, POST_PREF, HOOK_SUFF)) + if unknown_hooks: error_lines = ["Found one or more unknown hooks:"] @@ -231,14 +241,12 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, :param args: arguments to pass to hook function :param msg: custom message that is printed when hook is called """ + # print(f"Running hook '{label}' {pre_step_hook=} {post_step_hook=}") hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) res = None + args = args or [] + kwargs = kwargs or {} if hook: - if args is None: - args = [] - if kwargs is None: - kwargs = {} - if pre_step_hook: label = 'pre-' + label elif post_step_hook: @@ -251,4 +259,21 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) res = hook(*args, **kwargs) + + entrypoint_hooks = find_entrypoint_hooks(label=label, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) + if entrypoint_hooks: + msg = "Running entry point %s hook..." % label + if build_option('debug') and not build_option('silence_hook_trigger'): + print_msg(msg) + entrypoint_hooks.sort( + key=lambda x: (-getattr(x, HOOKS_ENTRYPOINT_PRIORITY, 0), x.__name__), + ) + for hook in entrypoint_hooks: + _log.info("Running entry point '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) + try: + res = hook(*args, **kwargs) + except Exception as e: + _log.error("Error running entry point '%s' hook: %s", hook.__name__, e) + raise EasyBuildError("Error running entry point '%s' hook: %s", hook.__name__, e) from e + return res diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index cd539cf547..8e9c369a08 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -303,6 +303,9 @@ def basic_options(self): 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', EXTRACT_STEP, 's', all_stops), 'strict': ("Set strictness level", 'choice', 'store', WARN, strictness_options), + 'use-entrypoints': ( + "Use entry points for easyblocks, toolchains, and hooks", None, 'store_true', False, + ), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 90a0c99583..f5eb01be00 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -40,6 +40,10 @@ import sys import easybuild.tools.toolchain +from easybuild.tools.entrypoints import ( + get_toolchain_entrypoints, validate_toolchain_entrypoints, + TOOLCHAIN_ENTRYPOINT_MARK, TOOLCHAIN_ENTRYPOINT_PREPEND +) from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.toolchain import Toolchain @@ -69,14 +73,17 @@ def search_toolchain(name): # make sure all defined toolchain constants are available in toolchain module tc_const_re = re.compile('^%s(.*)$' % TC_CONST_PREFIX) + # print(f'!! TC_MODULES: {tc_modules}') for tc_mod in tc_modules: # determine classes imported in this module mod_classes = [] for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]: + # print('-----', elem) if hasattr(elem, '__module__'): # exclude the toolchain class defined in that module if not tc_mod.__file__ == sys.modules[elem.__module__].__file__: elem_name = getattr(elem, '__name__', elem) + # print(f" Adding {elem_name} to list of imported classes used for looking for constants") _log.debug("Adding %s to list of imported classes used for looking for constants", elem_name) mod_classes.append(elem) @@ -106,6 +113,19 @@ def search_toolchain(name): # obtain all subclasses of toolchain found_tcs = nub(get_subclasses(Toolchain)) + invalid_eps = validate_toolchain_entrypoints() + if invalid_eps: + _log.warning("Invalid toolchain entrypoints found: %s", ', '.join(invalid_eps)) + raise EasyBuildError("Invalid toolchain entrypoints found: %s", ', '.join(invalid_eps)) + prepend_eps = [] + append_eps = [] + for tc in get_toolchain_entrypoints(): + if getattr(tc, TOOLCHAIN_ENTRYPOINT_PREPEND): + prepend_eps.append(tc) + else: + append_eps.append(tc) + found_tcs = prepend_eps + found_tcs + append_eps + # filter found toolchain subclasses based on whether they can be used a toolchains found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)] From 641c8f7ececa1cc402463b678e93aaaafc9af690 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 10 Jun 2025 18:06:40 +0200 Subject: [PATCH 02/19] Lint and cleanup --- easybuild/tools/entrypoints.py | 18 ++++++++---------- easybuild/tools/hooks.py | 7 ++++--- easybuild/tools/toolchain/utilities.py | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 22e8e4833d..8285613d98 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -13,8 +13,10 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError + _log = fancylogger.getLogger('entrypoints', fname=False) + def get_group_entrypoints(group: str) -> set[EntryPoint]: """Get all entrypoints for a group""" # print(f"--- Getting entry points for group: {group}") @@ -23,8 +25,8 @@ def get_group_entrypoints(group: str) -> set[EntryPoint]: return set() return set(ep for ep in entry_points(group=group)) -# EASYCONFIG_ENTRYPOINT = "easybuild.easyconfig" +# EASYCONFIG_ENTRYPOINT = "easybuild.easyconfig" EASYBLOCK_ENTRYPOINT = "easybuild.easyblock" EASYBLOCK_ENTRYPOINT_MARK = "_is_easybuild_easyblock" @@ -39,7 +41,8 @@ def get_group_entrypoints(group: str) -> set[EntryPoint]: HOOKS_ENTRYPOINT_MARK = "_is_easybuild_hook" HOOKS_ENTRYPOINT_PRIORITY = "_priority" -#########################################################################################3 + +######################################################################################### # Easyblock entrypoints def register_easyblock_entrypoint(): """Decorator to register an easyblock entrypoint.""" @@ -77,7 +80,7 @@ def validate_easyblock_entrypoints() -> list[str]: return invalid_easyblocks -def get_easyblock_entrypoints(name = None) -> dict: +def get_easyblock_entrypoints(name=None) -> dict: """Get all easyblock entrypoints. Returns: @@ -97,9 +100,6 @@ def get_easyblock_entrypoints(name = None) -> dict: 'loc': mod.__file__, } easyblocks[f'{ep.module}'] = ptr - # print('--' * 80) - # print(easyblocks) - # print('--' * 80) if name is not None: for key, value in easyblocks.items(): if value['class'] == name: @@ -110,6 +110,7 @@ def get_easyblock_entrypoints(name = None) -> dict: return easyblocks + ######################################################################################### # Hooks entrypoints def register_entrypoint_hooks(step, pre_step=False, post_step=False, priority=0): @@ -183,9 +184,7 @@ def validate_entrypoint_hooks(known_hooks: list[str], pre_prefix: str, post_pref def find_entrypoint_hooks(label, pre_step_hook=False, post_step_hook=False) -> list[Callable]: """Get all hooks defined in entry points.""" hooks = [] - # print(f"--- Searching for entry point hooks with label: {label}, pre_step_hook: {pre_step_hook}, post_step_hook: {post_step_hook}") for ep in get_group_entrypoints(HOOKS_ENTRYPOINT): - # print(f"--- Processing entry point: {ep.name}") try: hook = ep.load() except Exception as e: @@ -202,6 +201,7 @@ def find_entrypoint_hooks(label, pre_step_hook=False, post_step_hook=False) -> l return hooks + ######################################################################################### # Toolchain entrypoints def register_toolchain_entrypoint(prepend=False): @@ -228,8 +228,6 @@ def get_toolchain_entrypoints() -> set[EntryPoint]: _log.error(f"Error loading toolchain entry point {ep.name}: {e}") raise EasyBuildError(f"Error loading toolchain entry point {ep.name}: {e}") toolchains.append(tc) - # print(f"Found {len(toolchains)} toolchain entry points") - # print(f"Toolchain entry points: {toolchains}") return toolchains diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index bee7eb5969..3ea5cd6417 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -31,11 +31,9 @@ """ import difflib import os -from functools import wraps from easybuild.tools.entrypoints import ( find_entrypoint_hooks, validate_entrypoint_hooks, - HOOKS_ENTRYPOINT_MARK, HOOKS_ENTRYPOINT_STEP, HOOKS_ENTRYPOINT_PRE_STEP, HOOKS_ENTRYPOINT_POST_STEP, HOOKS_ENTRYPOINT_PRIORITY ) @@ -269,7 +267,10 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, key=lambda x: (-getattr(x, HOOKS_ENTRYPOINT_PRIORITY, 0), x.__name__), ) for hook in entrypoint_hooks: - _log.info("Running entry point '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) + _log.info( + "Running entry point '%s' hook function (args: %s, keyword args: %s)...", + hook.__name__, args, kwargs + ) try: res = hook(*args, **kwargs) except Exception as e: diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index f5eb01be00..b5d44c651e 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -42,7 +42,7 @@ import easybuild.tools.toolchain from easybuild.tools.entrypoints import ( get_toolchain_entrypoints, validate_toolchain_entrypoints, - TOOLCHAIN_ENTRYPOINT_MARK, TOOLCHAIN_ENTRYPOINT_PREPEND + TOOLCHAIN_ENTRYPOINT_PREPEND ) from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError From 45ef3740e9717eefa71641a7a08284fec48f6843 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 11 Jun 2025 17:44:52 +0200 Subject: [PATCH 03/19] Replace `_log.error` wiht `warning` --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/tools/entrypoints.py | 6 +++--- easybuild/tools/hooks.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 87e91fe52d..517d280519 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -2108,7 +2108,7 @@ def get_module_path(name, generic=None, decode=True): invalid_eps = validate_easyblock_entrypoints() if invalid_eps: - _log.error("Invalid easyblock entrypoints found: %s", invalid_eps) + _log.warning("Invalid easyblock entrypoints found: %s", invalid_eps) raise EasyBuildError("Invalid easyblock entrypoints found: %s", invalid_eps) eb_from_eps = get_easyblock_entrypoints(name) if eb_from_eps: diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 8ce5a26a72..bb3527070e 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -802,7 +802,7 @@ def avail_easyblocks(): invalid_eps = validate_easyblock_entrypoints() if invalid_eps: - _log.error("Found invalid easyblock entry points: %s", invalid_eps) + _log.warning("Found invalid easyblock entry points: %s", invalid_eps) raise EasyBuildError("Found invalid easyblock entry points: %s", invalid_eps) easyblocks.update(get_easyblock_entrypoints()) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 8285613d98..741f7a5d7f 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -91,7 +91,7 @@ def get_easyblock_entrypoints(name=None) -> dict: try: eb = ep.load() except Exception as e: - _log.error(f"Error loading easyblock entry point {ep.name}: {e}") + _log.warning(f"Error loading easyblock entry point {ep.name}: {e}") raise EasyBuildError(f"Error loading easyblock entry point {ep.name}: {e}") mod = importlib.import_module(eb.__module__) @@ -188,7 +188,7 @@ def find_entrypoint_hooks(label, pre_step_hook=False, post_step_hook=False) -> l try: hook = ep.load() except Exception as e: - _log.error(f"Error loading entry point {ep.name}: {e}") + _log.warning(f"Error loading entry point {ep.name}: {e}") raise EasyBuildError(f"Error loading entry point {ep.name}: {e}") cond = all([ @@ -225,7 +225,7 @@ def get_toolchain_entrypoints() -> set[EntryPoint]: try: tc = ep.load() except Exception as e: - _log.error(f"Error loading toolchain entry point {ep.name}: {e}") + _log.warning(f"Error loading toolchain entry point {ep.name}: {e}") raise EasyBuildError(f"Error loading toolchain entry point {ep.name}: {e}") toolchains.append(tc) return toolchains diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 3ea5cd6417..91e08c777f 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -274,7 +274,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, try: res = hook(*args, **kwargs) except Exception as e: - _log.error("Error running entry point '%s' hook: %s", hook.__name__, e) + _log.warning("Error running entry point '%s' hook: %s", hook.__name__, e) raise EasyBuildError("Error running entry point '%s' hook: %s", hook.__name__, e) from e return res From dff3af79574e7adfa2605a2e31e83ecb6c26d398 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 12 Jun 2025 10:54:58 +0200 Subject: [PATCH 04/19] Olden typehints to make CI happy and add guard against missing `importlib.metadata` in old python --- easybuild/tools/entrypoints.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 741f7a5d7f..9cda198add 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -6,23 +6,32 @@ """ import importlib -from importlib.metadata import EntryPoint, entry_points from easybuild.tools.config import build_option -from typing import Callable +from typing import Callable, List, Set from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError +try: + from importlib.metadata import entry_points, EntryPoint +except ModuleNotFoundError: + HAVE_ENTRY_POINTS = False +else: + HAVE_ENTRY_POINTS = True + _log = fancylogger.getLogger('entrypoints', fname=False) -def get_group_entrypoints(group: str) -> set[EntryPoint]: +def get_group_entrypoints(group: str) -> Set[EntryPoint]: """Get all entrypoints for a group""" - # print(f"--- Getting entry points for group: {group}") # Default True needed to work with commands like --list-toolchains that do not initialize the BuildOptions if not build_option('use_entrypoints', default=True): return set() + if not HAVE_ENTRY_POINTS: + msg = "Python importlib.metadata requires Python >= 3.8" + _log.warning(msg) + raise EasyBuildError(msg) return set(ep for ep in entry_points(group=group)) @@ -56,7 +65,7 @@ def decorator(cls: type) -> type: return decorator -def validate_easyblock_entrypoints() -> list[str]: +def validate_easyblock_entrypoints() -> List[str]: """Validate all easyblock entrypoints. Returns: @@ -125,7 +134,6 @@ def decorator(func): setattr(func, HOOKS_ENTRYPOINT_POST_STEP, post_step) setattr(func, HOOKS_ENTRYPOINT_PRIORITY, priority) - # Register the function as an entry point _log.info( "Registering entry point hook '%s' 'pre=%s' 'post=%s' with priority %d", func.__name__, pre_step, post_step, priority @@ -134,7 +142,7 @@ def decorator(func): return decorator -def validate_entrypoint_hooks(known_hooks: list[str], pre_prefix: str, post_prefix: str, suffix: str) -> list[str]: +def validate_entrypoint_hooks(known_hooks: List[str], pre_prefix: str, post_prefix: str, suffix: str) -> List[str]: """Validate all entrypoints hooks. Args: @@ -181,7 +189,7 @@ def validate_entrypoint_hooks(known_hooks: list[str], pre_prefix: str, post_pref return invalid_hooks -def find_entrypoint_hooks(label, pre_step_hook=False, post_step_hook=False) -> list[Callable]: +def find_entrypoint_hooks(label, pre_step_hook=False, post_step_hook=False) -> List[Callable]: """Get all hooks defined in entry points.""" hooks = [] for ep in get_group_entrypoints(HOOKS_ENTRYPOINT): @@ -218,7 +226,7 @@ def decorator(cls): return decorator -def get_toolchain_entrypoints() -> set[EntryPoint]: +def get_toolchain_entrypoints() -> Set[EntryPoint]: """Get all toolchain entrypoints.""" toolchains = [] for ep in get_group_entrypoints(TOOLCHAIN_ENTRYPOINT): @@ -231,7 +239,7 @@ def get_toolchain_entrypoints() -> set[EntryPoint]: return toolchains -def validate_toolchain_entrypoints() -> list[str]: +def validate_toolchain_entrypoints() -> List[str]: """Validate all toolchain entrypoints.""" invalid_toolchains = [] for ep in get_group_entrypoints(TOOLCHAIN_ENTRYPOINT): From 8b4d29f09c386295b12f2dfee3b9ca1026181082 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 12 Jun 2025 11:00:45 +0200 Subject: [PATCH 05/19] Fix for Python<3.10 --- easybuild/tools/entrypoints.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 9cda198add..4113fccbf5 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -32,7 +32,8 @@ def get_group_entrypoints(group: str) -> Set[EntryPoint]: msg = "Python importlib.metadata requires Python >= 3.8" _log.warning(msg) raise EasyBuildError(msg) - return set(ep for ep in entry_points(group=group)) + # Can't use the group keyword argument in entry_points() for Python < 3.10 + return set(ep for ep in entry_points() if ep.group == group) # EASYCONFIG_ENTRYPOINT = "easybuild.easyconfig" From 575cfdb839b079ecd1436bde36a083cff2a187de Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 12 Jun 2025 11:06:35 +0200 Subject: [PATCH 06/19] Removed EntryPoint typehints --- easybuild/tools/entrypoints.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 4113fccbf5..1e654b675b 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -7,13 +7,13 @@ import importlib from easybuild.tools.config import build_option -from typing import Callable, List, Set +from typing import Callable, List from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError try: - from importlib.metadata import entry_points, EntryPoint + from importlib.metadata import entry_points except ModuleNotFoundError: HAVE_ENTRY_POINTS = False else: @@ -23,7 +23,7 @@ _log = fancylogger.getLogger('entrypoints', fname=False) -def get_group_entrypoints(group: str) -> Set[EntryPoint]: +def get_group_entrypoints(group: str): """Get all entrypoints for a group""" # Default True needed to work with commands like --list-toolchains that do not initialize the BuildOptions if not build_option('use_entrypoints', default=True): @@ -227,7 +227,7 @@ def decorator(cls): return decorator -def get_toolchain_entrypoints() -> Set[EntryPoint]: +def get_toolchain_entrypoints(): """Get all toolchain entrypoints.""" toolchains = [] for ep in get_group_entrypoints(TOOLCHAIN_ENTRYPOINT): From 165ef9672e87066d7037a2db417fd61c10d5c432 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 12 Jun 2025 11:51:02 +0200 Subject: [PATCH 07/19] More fixes for python versions --- easybuild/tools/entrypoints.py | 36 ++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 1e654b675b..2bcc5e5b1a 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -13,7 +13,7 @@ from easybuild.tools.build_log import EasyBuildError try: - from importlib.metadata import entry_points + from importlib.metadata import entry_points, EntryPoints except ModuleNotFoundError: HAVE_ENTRY_POINTS = False else: @@ -25,15 +25,31 @@ def get_group_entrypoints(group: str): """Get all entrypoints for a group""" - # Default True needed to work with commands like --list-toolchains that do not initialize the BuildOptions - if not build_option('use_entrypoints', default=True): - return set() - if not HAVE_ENTRY_POINTS: - msg = "Python importlib.metadata requires Python >= 3.8" - _log.warning(msg) - raise EasyBuildError(msg) - # Can't use the group keyword argument in entry_points() for Python < 3.10 - return set(ep for ep in entry_points() if ep.group == group) + strict_python = True + use_eps = build_option('use_entrypoints', default=None) + if use_eps is None: + # Default True needed to work with commands like --list-toolchains that do not initialize the BuildOptions + use_eps = True + # Needed to work with older Python versions: do not raise errors when entry points are default enabled + strict_python = False + res = set() + if use_eps: + if not HAVE_ENTRY_POINTS and strict_python: + msg = "`--use-entrypoints` requires importlib.metadata (Python >= 3.8)" + _log.warning(msg) + raise EasyBuildError(msg) + # Can't use the group keyword argument in entry_points() for Python < 3.10 + try: + eps = entry_points() + if isinstance(eps, EntryPoints): + # Python >= 3.10 + res = set(ep for ep in eps if ep.group == group) + elif isinstance(eps, dict): + # Python < 3.10 + res = set(eps.get(group, [])) + except NameError: + _log.debug("`get_group_entrypoints` called before BuildOptions initialized, with python < 3.8") + return res # EASYCONFIG_ENTRYPOINT = "easybuild.easyconfig" From 0edd625ba5e8ef74ddaefaff3a23cb752ada1258 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 17 Jun 2025 12:07:09 +0200 Subject: [PATCH 08/19] Improved version check logic --- easybuild/tools/entrypoints.py | 44 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 2bcc5e5b1a..ddf1bd2b30 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -4,7 +4,7 @@ * Davide Grassano (CECAM) """ - +import sys import importlib from easybuild.tools.config import build_option from typing import Callable, List @@ -12,12 +12,16 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -try: - from importlib.metadata import entry_points, EntryPoints -except ModuleNotFoundError: - HAVE_ENTRY_POINTS = False -else: + +HAVE_ENTRY_POINTS = False +HAVE_ENTRY_POINTS_CLS = False +if sys.version_info >= (3, 8): HAVE_ENTRY_POINTS = True + from importlib.metadata import entry_points + +if sys.version_info >= (3, 10): + # Python >= 3.10 uses importlib.metadata.EntryPoints as a type for entry_points() + HAVE_ENTRY_POINTS_CLS = True _log = fancylogger.getLogger('entrypoints', fname=False) @@ -34,21 +38,21 @@ def get_group_entrypoints(group: str): strict_python = False res = set() if use_eps: - if not HAVE_ENTRY_POINTS and strict_python: - msg = "`--use-entrypoints` requires importlib.metadata (Python >= 3.8)" - _log.warning(msg) - raise EasyBuildError(msg) - # Can't use the group keyword argument in entry_points() for Python < 3.10 - try: - eps = entry_points() - if isinstance(eps, EntryPoints): - # Python >= 3.10 - res = set(ep for ep in eps if ep.group == group) - elif isinstance(eps, dict): - # Python < 3.10 + if not HAVE_ENTRY_POINTS: + if strict_python: + msg = "`--use-entrypoints` requires importlib.metadata (Python >= 3.8)" + _log.warning(msg) + raise EasyBuildError(msg) + else: + _log.debug("`get_group_entrypoints` called before BuildOptions initialized, with python < 3.8") + else: + if HAVE_ENTRY_POINTS_CLS: + eps = entry_points(group=group) + res = set(eps) + else: + eps = entry_points() res = set(eps.get(group, [])) - except NameError: - _log.debug("`get_group_entrypoints` called before BuildOptions initialized, with python < 3.8") + return res From 1a5ea6434841eb2ff77b1ab945d6670572d474c6 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 17 Jun 2025 17:59:08 +0200 Subject: [PATCH 09/19] WIP - adding tests --- easybuild/tools/docs.py | 4 + easybuild/tools/entrypoints.py | 6 +- easybuild/tools/toolchain/toolchain.py | 4 +- easybuild/tools/toolchain/utilities.py | 6 +- test/framework/entrypoints.py | 275 +++++++++++++++++++++++++ test/framework/suite.py | 3 +- 6 files changed, 289 insertions(+), 9 deletions(-) create mode 100644 test/framework/entrypoints.py diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index d7fba3885d..6945b3ef67 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -66,6 +66,7 @@ from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import INDENT_2SPACES, INDENT_4SPACES from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table, nub, quote_str +from easybuild.tools.entrypoints import EASYBLOCK_ENTRYPOINT_MARK _log = fancylogger.getLogger('tools.docs') @@ -724,6 +725,9 @@ def gen_list_easyblocks(list_easyblocks, format_strings): def add_class(classes, cls): """Add a new class, and all of its subclasses.""" children = cls.__subclasses__() + # Filter out possible sublcasses coming from entrypoints as they will be readded letter + if not build_option('use_entrypoints', default=False): + children = [c for c in children if not hasattr(c, EASYBLOCK_ENTRYPOINT_MARK)] classes.update({cls.__name__: { 'module': cls.__module__, 'children': sorted([c.__name__ for c in children], key=lambda x: x.lower()) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index ddf1bd2b30..9b8c62ae12 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -47,11 +47,9 @@ def get_group_entrypoints(group: str): _log.debug("`get_group_entrypoints` called before BuildOptions initialized, with python < 3.8") else: if HAVE_ENTRY_POINTS_CLS: - eps = entry_points(group=group) - res = set(eps) + res = set(entry_points(group=group)) else: - eps = entry_points() - res = set(eps.get(group, [])) + res = set(entry_points().get(group, [])) return res diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index d89c10b71a..312724767e 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -168,7 +168,7 @@ class Toolchain: CLASS_CONSTANTS_TO_RESTORE = None CLASS_CONSTANT_COPIES = {} - # class method + @classmethod def _is_toolchain_for(cls, name): """see if this class can provide support for toolchain named name""" # TODO report later in the initialization the found version @@ -181,8 +181,6 @@ def _is_toolchain_for(cls, name): # is no name is supplied, check whether class can be used as a toolchain return bool(getattr(cls, 'NAME', None)) - _is_toolchain_for = classmethod(_is_toolchain_for) - def __init__(self, name=None, version=None, mns=None, class_constants=None, tcdeps=None, modtool=None, hidden=False): """ diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index b5d44c651e..14306e4bbb 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -42,7 +42,7 @@ import easybuild.tools.toolchain from easybuild.tools.entrypoints import ( get_toolchain_entrypoints, validate_toolchain_entrypoints, - TOOLCHAIN_ENTRYPOINT_PREPEND + TOOLCHAIN_ENTRYPOINT_PREPEND, TOOLCHAIN_ENTRYPOINT_MARK ) from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError @@ -113,6 +113,10 @@ def search_toolchain(name): # obtain all subclasses of toolchain found_tcs = nub(get_subclasses(Toolchain)) + # Getting all subclasses will also include toolchains that are registered as entrypoints even if we are not + # using the `--use-entrypoints` option, so we filter them out here and re-add them later if needed. + found_tcs = [x for x in found_tcs if not hasattr(x, TOOLCHAIN_ENTRYPOINT_MARK)] + invalid_eps = validate_toolchain_entrypoints() if invalid_eps: _log.warning("Invalid toolchain entrypoints found: %s", ', '.join(invalid_eps)) diff --git a/test/framework/entrypoints.py b/test/framework/entrypoints.py new file mode 100644 index 0000000000..785a7786b2 --- /dev/null +++ b/test/framework/entrypoints.py @@ -0,0 +1,275 @@ +# # +# Copyright 2013-2025 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for EasyBuild configuration. + +@author: Davide Grassano (CECAM - EPFL) +""" + +import os +import re +import shutil +import sys +import tempfile +from importlib import reload +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from unittest import TextTestRunner + +import easybuild.tools.options as eboptions +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import ERROR, IGNORE, WARN, BuildOptions, ConfigurationVariables +from easybuild.tools.config import build_option, build_path, get_build_log_path, get_log_filename, get_repositorypath +from easybuild.tools.config import install_path, log_file_format, log_path, source_paths +from easybuild.tools.config import update_build_option, update_build_options +from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, init_build_options +from easybuild.tools.filetools import copy_dir, mkdir, write_file +from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX +from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains +from easybuild.tools.entrypoints import ( + get_group_entrypoints, HOOKS_ENTRYPOINT, EASYBLOCK_ENTRYPOINT, TOOLCHAIN_ENTRYPOINT, + HAVE_ENTRY_POINTS +) +from easybuild.framework.easyconfig.easyconfig import get_module_path +from easybuild.framework.easyblock import EasyBlock + + +if HAVE_ENTRY_POINTS: + from importlib.metadata import DistributionFinder, Distribution + + +MOCK_HOOK_EP_NAME = "mock_hook" +MOCK_EASYBLOCK_EP_NAME = "mock_easyblock" +MOCK_TOOLCHAIN_EP_NAME = "mock_toolchain" + +MOCK_HOOK = "hello_world" +MOCK_EASYBLOCK = "TestEasyBlock" +MOCK_TOOLCHAIN = "MockTc" + + +MOCK_EP_FILE=f""" +from easybuild.tools.entrypoints import register_entrypoint_hooks +from easybuild.tools.hooks import CONFIGURE_STEP, START + + +@register_entrypoint_hooks(START) +def {MOCK_HOOK}(): + print("Hello, World! ----------------------------------------") + +########################################################################## +from easybuild.framework.easyblock import EasyBlock +from easybuild.tools.entrypoints import register_easyblock_entrypoint + +@register_easyblock_entrypoint() +class {MOCK_EASYBLOCK}(EasyBlock): + def configure_step(self): + print("{MOCK_EASYBLOCK}: configure_step called.") + + def build_step(self): + print("{MOCK_EASYBLOCK}: build_step called.") + + def install_step(self): + print("{MOCK_EASYBLOCK}: install_step called.") + + def sanity_check_step(self): + print("{MOCK_EASYBLOCK}: sanity_check_step called.") + +########################################################################## +from easybuild.tools.entrypoints import register_toolchain_entrypoint +from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, Compiler +from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME + +TC_CONSTANT_MOCK = "Mock" + +class MockCompiler(Compiler): + COMPILER_FAMILY = TC_CONSTANT_MOCK + SUBTOOLCHAIN = SYSTEM_TOOLCHAIN_NAME + +@register_toolchain_entrypoint() +class {MOCK_TOOLCHAIN}(MockCompiler): + NAME = '{MOCK_TOOLCHAIN}' # Using `...tc` to distinguish toolchain from package + COMPILER_MODULE_NAME = [NAME] + SUBTOOLCHAIN = [SYSTEM_TOOLCHAIN_NAME] +""" + + + +MOCK_EP_META_FILE = f""" +[{HOOKS_ENTRYPOINT}] +{MOCK_HOOK_EP_NAME} = {{module}}:hello_world + +[{EASYBLOCK_ENTRYPOINT}] +{MOCK_EASYBLOCK_EP_NAME} = {{module}}:TestEasyBlock + +[{TOOLCHAIN_ENTRYPOINT}] +{MOCK_TOOLCHAIN_EP_NAME} = {{module}}:MockTc +""" + + +class MockDistribution(Distribution): + """Mock distribution for testing entry points.""" + def __init__(self, module): + self.module = module + + def read_text(self, filename): + if filename == "entry_points.txt": + return MOCK_EP_META_FILE.format(module=self.module) + + if filename == "METADATA": + return "Name: mock_hook\nVersion: 0.1.0\n" + +class MockDistributionFinder(DistributionFinder): + """Mock distribution finder for testing entry points.""" + def __init__(self, *args, module, **kwargs): + super().__init__(*args, **kwargs) + self.module = module + + def find_distributions(self, context=None): + yield MockDistribution(self.module) + + +class EasyBuildEntrypointsTest(EnhancedTestCase): + """Test cases for EasyBuild configuration.""" + + tmpdir = None + + def setUp(self): + """Set up the test environment.""" + reload(eboptions) + super().setUp() + self.tmpdir = tempfile.mkdtemp(prefix='easybuild_test_') + + if HAVE_ENTRY_POINTS: + filename_root = "mock" + dirname, dirpath = os.path.split(self.tmpdir) + + self.module = '.'.join([dirpath, filename_root]) + sys.path.insert(0, dirname) + sys.meta_path.insert(0, MockDistributionFinder(module=self.module)) + + # Create a mock entry point for testing + mock_hook_file = os.path.join(self.tmpdir, f'{filename_root}.py') + write_file(mock_hook_file, MOCK_EP_FILE) + + def tearDown(self): + """Clean up the test environment.""" + if self.tmpdir and os.path.isdir(self.tmpdir): + shutil.rmtree(self.tmpdir) + + if HAVE_ENTRY_POINTS: + # Remove the entry point from the working set + torm = [] + for idx,cls in enumerate(sys.meta_path): + if isinstance(cls, MockDistributionFinder): + torm.append(idx) + for idx in reversed(torm): + del sys.meta_path[idx] + + def test_entrypoints_get_group_too_old_python(self): + """Test retrieving entrypoints for a specific group with too old Python version.""" + if HAVE_ENTRY_POINTS: + self.skipTest("Entry points available in this Python version") + self.assertRaises(EasyBuildError, get_group_entrypoints, HOOKS_ENTRYPOINT) + + def test_entrypoints_get_group(self): + """Test retrieving entrypoints for a specific group.""" + if not HAVE_ENTRY_POINTS: + self.skipTest("Entry points not available in this Python version") + + expected = { + HOOKS_ENTRYPOINT: MOCK_HOOK_EP_NAME, + EASYBLOCK_ENTRYPOINT: MOCK_EASYBLOCK_EP_NAME, + TOOLCHAIN_ENTRYPOINT: MOCK_TOOLCHAIN_EP_NAME, + } + + # init_config() + for group in [HOOKS_ENTRYPOINT, EASYBLOCK_ENTRYPOINT, TOOLCHAIN_ENTRYPOINT]: + epts = get_group_entrypoints(group) + self.assertIsInstance(epts, set, f"Expected set for group {group}") + self.assertEqual(len(epts), 0, f"Expected non-empty set for group {group}") + + init_config(build_options={'use_entrypoints': True}) + for group in [HOOKS_ENTRYPOINT, EASYBLOCK_ENTRYPOINT, TOOLCHAIN_ENTRYPOINT]: + epts = get_group_entrypoints(group) + self.assertIsInstance(epts, set, f"Expected set for group {group}") + self.assertGreater(len(epts), 0, f"Expected non-empty set for group {group}") + + loaded_names = [ep.name for ep in epts] + self.assertIn(expected[group], loaded_names, f"Expected entry point {expected[group]} in group {group}") + + def test_entrypoints_list_easyblocks(self): + """ + Tests for list_easyblocks function with entry points enabled. + """ + if not HAVE_ENTRY_POINTS: + self.skipTest("Entry points not available in this Python version") + + # init_config() + # print('-------', build_option('use_entrypoints', default='1234')) + txt = list_easyblocks() + self.assertNotIn("TestEasyBlock", txt, "TestEasyBlock should not be listed without entry points enabled") + + init_config(build_options={'use_entrypoints': True}) + txt = list_easyblocks() + self.assertIn("TestEasyBlock", txt, "TestEasyBlock should be listed with entry points enabled") + + def test_entrypoints_list_toolchains(self): + """ + Tests for list_toolchains function with entry points enabled. + """ + if not HAVE_ENTRY_POINTS: + self.skipTest("Entry points not available in this Python version") + + # init_config() + txt = list_toolchains() + self.assertNotIn(MOCK_TOOLCHAIN, txt, f"{MOCK_TOOLCHAIN} should not be listed without entry points enabled") + + init_config(build_options={'use_entrypoints': True}) + + txt = list_toolchains() + self.assertIn(MOCK_TOOLCHAIN, txt, f"{MOCK_TOOLCHAIN} should be listed with entry points enabled") + + def test_entrypoints_get_module_path(self): + """ + Tests for get_module_path function with entry points enabled. + """ + if not HAVE_ENTRY_POINTS: + self.skipTest("Entry points not available in this Python version") + + module_path = get_module_path(MOCK_EASYBLOCK) + self.assertIn('.generic.', module_path, "Module path should contain '.generic.'") + + init_config(build_options={'use_entrypoints': True}) + # Reload the EasyBlock module to ensure it is recognized + module_path = get_module_path(MOCK_EASYBLOCK) + self.assertEqual(module_path, self.module, "Module path should match the mock module path") + + +def suite(): + return TestLoaderFiltered().loadTestsFromTestCase(EasyBuildEntrypointsTest, sys.argv[1:]) + + +if __name__ == '__main__': + res = TextTestRunner(verbosity=1).run(suite()) + sys.exit(len(res.failures)) diff --git a/test/framework/suite.py b/test/framework/suite.py index afec127c83..28de8ee56e 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -52,6 +52,7 @@ import test.framework.easyconfigversion as ev import test.framework.easystack as es import test.framework.ebconfigobj as ebco +import test.framework.entrypoints as epts import test.framework.environment as env import test.framework.docs as d import test.framework.filetools as f @@ -119,7 +120,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config -tests = [gen, d, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, +tests = [gen, d, bl, o, r, ef, ev, ebco, ep, epts, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, tw, p, i, pkg, env, et, st, h, ct, lib, u, es, ou] SUITE = unittest.TestSuite([x.suite() for x in tests]) From b0aeb735a91f3b2094da41b9d102685928368523 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 18 Jun 2025 11:59:25 +0200 Subject: [PATCH 10/19] lint --- test/framework/entrypoints.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/test/framework/entrypoints.py b/test/framework/entrypoints.py index 785a7786b2..ec72eb06cd 100644 --- a/test/framework/entrypoints.py +++ b/test/framework/entrypoints.py @@ -29,7 +29,6 @@ """ import os -import re import shutil import sys import tempfile @@ -39,20 +38,13 @@ import easybuild.tools.options as eboptions from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import ERROR, IGNORE, WARN, BuildOptions, ConfigurationVariables -from easybuild.tools.config import build_option, build_path, get_build_log_path, get_log_filename, get_repositorypath -from easybuild.tools.config import install_path, log_file_format, log_path, source_paths -from easybuild.tools.config import update_build_option, update_build_options -from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, init_build_options -from easybuild.tools.filetools import copy_dir, mkdir, write_file -from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX -from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains +from easybuild.tools.filetools import write_file +from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.entrypoints import ( get_group_entrypoints, HOOKS_ENTRYPOINT, EASYBLOCK_ENTRYPOINT, TOOLCHAIN_ENTRYPOINT, HAVE_ENTRY_POINTS ) from easybuild.framework.easyconfig.easyconfig import get_module_path -from easybuild.framework.easyblock import EasyBlock if HAVE_ENTRY_POINTS: @@ -68,7 +60,7 @@ MOCK_TOOLCHAIN = "MockTc" -MOCK_EP_FILE=f""" +MOCK_EP_FILE = f""" from easybuild.tools.entrypoints import register_entrypoint_hooks from easybuild.tools.hooks import CONFIGURE_STEP, START @@ -114,7 +106,6 @@ class {MOCK_TOOLCHAIN}(MockCompiler): """ - MOCK_EP_META_FILE = f""" [{HOOKS_ENTRYPOINT}] {MOCK_HOOK_EP_NAME} = {{module}}:hello_world @@ -139,6 +130,7 @@ def read_text(self, filename): if filename == "METADATA": return "Name: mock_hook\nVersion: 0.1.0\n" + class MockDistributionFinder(DistributionFinder): """Mock distribution finder for testing entry points.""" def __init__(self, *args, module, **kwargs): @@ -180,7 +172,7 @@ def tearDown(self): if HAVE_ENTRY_POINTS: # Remove the entry point from the working set torm = [] - for idx,cls in enumerate(sys.meta_path): + for idx, cls in enumerate(sys.meta_path): if isinstance(cls, MockDistributionFinder): torm.append(idx) for idx in reversed(torm): From 46af0e9e85dc8241949b49e1fc5c614a509c9588 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 18 Jun 2025 12:52:02 +0200 Subject: [PATCH 11/19] Fixes undefined `Distribution` --- test/framework/entrypoints.py | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/test/framework/entrypoints.py b/test/framework/entrypoints.py index ec72eb06cd..1b29125407 100644 --- a/test/framework/entrypoints.py +++ b/test/framework/entrypoints.py @@ -37,7 +37,6 @@ from unittest import TextTestRunner import easybuild.tools.options as eboptions -from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.entrypoints import ( @@ -49,6 +48,9 @@ if HAVE_ENTRY_POINTS: from importlib.metadata import DistributionFinder, Distribution +else: + DistributionFinder = object + Distribution = object MOCK_HOOK_EP_NAME = "mock_hook" @@ -163,6 +165,8 @@ def setUp(self): # Create a mock entry point for testing mock_hook_file = os.path.join(self.tmpdir, f'{filename_root}.py') write_file(mock_hook_file, MOCK_EP_FILE) + else: + self.skipTest("Entry points not available in this Python version") def tearDown(self): """Clean up the test environment.""" @@ -178,24 +182,14 @@ def tearDown(self): for idx in reversed(torm): del sys.meta_path[idx] - def test_entrypoints_get_group_too_old_python(self): - """Test retrieving entrypoints for a specific group with too old Python version.""" - if HAVE_ENTRY_POINTS: - self.skipTest("Entry points available in this Python version") - self.assertRaises(EasyBuildError, get_group_entrypoints, HOOKS_ENTRYPOINT) - def test_entrypoints_get_group(self): """Test retrieving entrypoints for a specific group.""" - if not HAVE_ENTRY_POINTS: - self.skipTest("Entry points not available in this Python version") - expected = { HOOKS_ENTRYPOINT: MOCK_HOOK_EP_NAME, EASYBLOCK_ENTRYPOINT: MOCK_EASYBLOCK_EP_NAME, TOOLCHAIN_ENTRYPOINT: MOCK_TOOLCHAIN_EP_NAME, } - # init_config() for group in [HOOKS_ENTRYPOINT, EASYBLOCK_ENTRYPOINT, TOOLCHAIN_ENTRYPOINT]: epts = get_group_entrypoints(group) self.assertIsInstance(epts, set, f"Expected set for group {group}") @@ -214,11 +208,6 @@ def test_entrypoints_list_easyblocks(self): """ Tests for list_easyblocks function with entry points enabled. """ - if not HAVE_ENTRY_POINTS: - self.skipTest("Entry points not available in this Python version") - - # init_config() - # print('-------', build_option('use_entrypoints', default='1234')) txt = list_easyblocks() self.assertNotIn("TestEasyBlock", txt, "TestEasyBlock should not be listed without entry points enabled") @@ -230,10 +219,6 @@ def test_entrypoints_list_toolchains(self): """ Tests for list_toolchains function with entry points enabled. """ - if not HAVE_ENTRY_POINTS: - self.skipTest("Entry points not available in this Python version") - - # init_config() txt = list_toolchains() self.assertNotIn(MOCK_TOOLCHAIN, txt, f"{MOCK_TOOLCHAIN} should not be listed without entry points enabled") @@ -246,9 +231,6 @@ def test_entrypoints_get_module_path(self): """ Tests for get_module_path function with entry points enabled. """ - if not HAVE_ENTRY_POINTS: - self.skipTest("Entry points not available in this Python version") - module_path = get_module_path(MOCK_EASYBLOCK) self.assertIn('.generic.', module_path, "Module path should contain '.generic.'") From 6a78c9b12e65343c1ebd0272c87036dcd2795c3c Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 18 Jun 2025 14:17:31 +0200 Subject: [PATCH 12/19] Better tearDown --- test/framework/entrypoints.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/framework/entrypoints.py b/test/framework/entrypoints.py index 1b29125407..2436087daf 100644 --- a/test/framework/entrypoints.py +++ b/test/framework/entrypoints.py @@ -170,12 +170,17 @@ def setUp(self): def tearDown(self): """Clean up the test environment.""" + super().tearDown() + if self.tmpdir and os.path.isdir(self.tmpdir): shutil.rmtree(self.tmpdir) if HAVE_ENTRY_POINTS: # Remove the entry point from the working set torm = [] + dirname, _ = os.path.split(self.tmpdir) + if dirname in sys.path: + sys.path.remove(dirname) for idx, cls in enumerate(sys.meta_path): if isinstance(cls, MockDistributionFinder): torm.append(idx) From 05ec7ce89929e6d7107a2edf050273670da185f0 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 19 Jun 2025 12:00:00 +0200 Subject: [PATCH 13/19] - Reworked the way entrypoints are registered and recalled - Added listing of entrypoints available through `--show-config` - Enhanced test cases --- easybuild/framework/easyconfig/easyconfig.py | 22 +- easybuild/framework/easyconfig/tools.py | 14 +- easybuild/tools/docs.py | 7 +- easybuild/tools/entrypoints.py | 355 ++++++++----------- easybuild/tools/hooks.py | 19 +- easybuild/tools/options.py | 25 ++ easybuild/tools/toolchain/utilities.py | 25 +- test/framework/entrypoints.py | 197 ++++++++-- 8 files changed, 369 insertions(+), 295 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 517d280519..8316fa1119 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -65,7 +65,7 @@ from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATES, DEPRECATED_EASYCONFIG_TEMPLATES from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools import LooseVersion -from easybuild.tools.entrypoints import get_easyblock_entrypoints, validate_easyblock_entrypoints +from easybuild.tools.entrypoints import EntrypointEasyblock from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning, print_msg from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN @@ -2017,9 +2017,15 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error class_name, modulepath) cls = get_class_for(modulepath, class_name) else: - modulepath = get_module_path(easyblock) - cls = get_class_for(modulepath, class_name) - _log.info("Derived full easyblock module path for %s: %s" % (class_name, modulepath)) + eb_from_eps = EntrypointEasyblock.get_entrypoints(name=easyblock) + if eb_from_eps: + ep = eb_from_eps[0] + cls = ep.wrapped + _log.info("Obtained easyblock class '%s' from entrypoint '%s'", easyblock, str(ep)) + else: + modulepath = get_module_path(easyblock) + cls = get_class_for(modulepath, class_name) + _log.info("Derived full easyblock module path for %s: %s" % (class_name, modulepath)) else: # if no easyblock specified, try to find if one exists if name is None: @@ -2106,14 +2112,6 @@ def get_module_path(name, generic=None, decode=True): if generic is None: generic = filetools.is_generic_easyblock(name) - invalid_eps = validate_easyblock_entrypoints() - if invalid_eps: - _log.warning("Invalid easyblock entrypoints found: %s", invalid_eps) - raise EasyBuildError("Invalid easyblock entrypoints found: %s", invalid_eps) - eb_from_eps = get_easyblock_entrypoints(name) - if eb_from_eps: - return list(eb_from_eps.keys())[0] - # example: 'EB_VSC_minus_tools' should result in 'vsc_tools' if decode: name = decode_class_name(name) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index bb3527070e..25cf469db8 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -54,7 +54,7 @@ from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.tools import LooseVersion -from easybuild.tools.entrypoints import validate_easyblock_entrypoints, get_easyblock_entrypoints +from easybuild.tools.entrypoints import EntrypointEasyblock from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_error, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env @@ -800,11 +800,13 @@ def avail_easyblocks(): else: raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc) - invalid_eps = validate_easyblock_entrypoints() - if invalid_eps: - _log.warning("Found invalid easyblock entry points: %s", invalid_eps) - raise EasyBuildError("Found invalid easyblock entry points: %s", invalid_eps) - easyblocks.update(get_easyblock_entrypoints()) + ept_eb_lst = EntrypointEasyblock.get_entrypoints() + + for ept_eb in ept_eb_lst: + easyblocks[ept_eb.module] = { + 'class': ept_eb.name, + 'loc': ept_eb.file, + } return easyblocks diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 6945b3ef67..5bbe6cb3d5 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -66,7 +66,6 @@ from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import INDENT_2SPACES, INDENT_4SPACES from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table, nub, quote_str -from easybuild.tools.entrypoints import EASYBLOCK_ENTRYPOINT_MARK _log = fancylogger.getLogger('tools.docs') @@ -725,9 +724,9 @@ def gen_list_easyblocks(list_easyblocks, format_strings): def add_class(classes, cls): """Add a new class, and all of its subclasses.""" children = cls.__subclasses__() - # Filter out possible sublcasses coming from entrypoints as they will be readded letter - if not build_option('use_entrypoints', default=False): - children = [c for c in children if not hasattr(c, EASYBLOCK_ENTRYPOINT_MARK)] + # Filter out possible sublcasses coming from entrypoints based on build option + # if not build_option('use_entrypoints', default=False): + # children = [c for c in children if not hasattr(c, EASYBLOCK_ENTRYPOINT_MARK)] classes.update({cls.__name__: { 'module': cls.__module__, 'children': sorted([c.__name__ for c in children], key=lambda x: x.lower()) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 9b8c62ae12..196daf5fcc 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -7,7 +7,6 @@ import sys import importlib from easybuild.tools.config import build_option -from typing import Callable, List from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError @@ -24,6 +23,10 @@ HAVE_ENTRY_POINTS_CLS = True +EASYBLOCK_ENTRYPOINT = "easybuild.easyblock" +TOOLCHAIN_ENTRYPOINT = "easybuild.toolchain" +HOOKS_ENTRYPOINT = "easybuild.hooks" + _log = fancylogger.getLogger('entrypoints', fname=False) @@ -54,220 +57,144 @@ def get_group_entrypoints(group: str): return res -# EASYCONFIG_ENTRYPOINT = "easybuild.easyconfig" -EASYBLOCK_ENTRYPOINT = "easybuild.easyblock" -EASYBLOCK_ENTRYPOINT_MARK = "_is_easybuild_easyblock" +class EasybuildEntrypoint: + _group = None + expected_type = None + registered = {} + + def __init__(self): + self.wrapped = None + + self.module = None + self.name = None + self.file = None + + def __repr__(self): + return f"{self.__class__.__name__} <{self.module}:{self.name}>" + + def __call__(self, wrap): + """Use an instance of this class as a decorator to register an entrypoint.""" + if self.expected_type is not None: + check = False + try: + check = isinstance(wrap, self.expected_type) or issubclass(wrap, self.expected_type) + except: + pass + if not check: + raise EasyBuildError( + "Entrypoint '%s' expected type '%s', got '%s'", + self.name, self.expected_type, type(wrap) + ) + self.wrapped = wrap + self.module = getattr(wrap, '__module__', None) + self.name = getattr(wrap, '__name__', None) + if self.module: + mod = importlib.import_module(self.module) + self.file = getattr(mod, '__file__', None) + + grp = self.registered.setdefault(self.group, set()) + + for ep in grp: + if ep.name == self.name and ep.module != self.module: + raise ValueError( + "Entrypoint '%s' already registered in group '%s' by module '%s' vs '%s'", + self.name, self.group, ep.module, self.module + ) + grp.add(self) + + self.validate() + + _log.debug("Registered entrypoint: %s", self) + + return wrap + + @property + def group(self): + if self._group is None: + raise EasyBuildError("Entrypoint group not set for %s", self.__class__.__name__) + return self._group + + @classmethod + def get_entrypoints(cls, name=None, **filter_params): + """Get all entrypoints in this group.""" + for ep in get_group_entrypoints(cls._group): + try: + ep.load() + except Exception as e: + msg = f"Error loading entrypoint {ep}: {e}" + _log.error(msg) + raise EasyBuildError(msg) from e + + entrypoints = [] + for ep in cls.registered.get(cls._group, []): + cond = name is None or ep.name == name + for key, value in filter_params.items(): + cond = cond and getattr(ep, key, None) == value + if cond: + entrypoints.append(ep) + + return entrypoints + + @staticmethod + def clear(): + """Clear the registered entrypoints. Used for testing when the same entrypoint is loaded multiple times + from different temporary directories.""" + EasybuildEntrypoint.registered.clear() + + def validate(self): + """Validate the entrypoint.""" + if self.module is None or self.name is None: + raise EasyBuildError("Entrypoint `%s` has no module or name associated", self.wrapped) + + +class EntrypointHook(EasybuildEntrypoint): + """Class to represent a hook entrypoint.""" + _group = HOOKS_ENTRYPOINT + + def __init__(self, step, pre_step=False, post_step=False, priority=0): + """Initialize the EntrypointHook.""" + super().__init__() + self.step = step + self.pre_step = pre_step + self.post_step = post_step + self.priority = priority + + def validate(self): + """Validate the hook entrypoint.""" + from easybuild.tools.hooks import KNOWN_HOOKS, HOOK_SUFF, PRE_PREF, POST_PREF + super().validate() -TOOLCHAIN_ENTRYPOINT = "easybuild.toolchain" -TOOLCHAIN_ENTRYPOINT_MARK = "_is_easybuild_toolchain" -TOOLCHAIN_ENTRYPOINT_PREPEND = "_prepend" + prefix = '' + if self.pre_step: + prefix = PRE_PREF + elif self.post_step: + prefix = POST_PREF -HOOKS_ENTRYPOINT = "easybuild.hooks" -HOOKS_ENTRYPOINT_STEP = "_step" -HOOKS_ENTRYPOINT_PRE_STEP = "_pre_step" -HOOKS_ENTRYPOINT_POST_STEP = "_post_step" -HOOKS_ENTRYPOINT_MARK = "_is_easybuild_hook" -HOOKS_ENTRYPOINT_PRIORITY = "_priority" - - -######################################################################################### -# Easyblock entrypoints -def register_easyblock_entrypoint(): - """Decorator to register an easyblock entrypoint.""" - def decorator(cls: type) -> type: - if not isinstance(cls, type): - raise EasyBuildError("Easyblock entrypoint `%s` is not a class", cls.__name__) - setattr(cls, EASYBLOCK_ENTRYPOINT_MARK, True) - _log.debug("Registering easyblock entrypoint: %s", cls.__name__) - return cls - - return decorator - - -def validate_easyblock_entrypoints() -> List[str]: - """Validate all easyblock entrypoints. - - Returns: - List of invalid easyblocks. - """ - invalid_easyblocks = [] - for ep in get_group_entrypoints(EASYBLOCK_ENTRYPOINT): - full_name = f'{ep.name} <{ep.value}>' - - eb = ep.load() - if not hasattr(eb, EASYBLOCK_ENTRYPOINT_MARK): - invalid_easyblocks.append(full_name) - _log.warning(f"Easyblock {ep.name} <{ep.value}> is not a valid EasyBuild easyblock") - continue - - if not isinstance(eb, type): - _log.warning(f"Easyblock {ep.name} <{ep.value}> is not a class") - invalid_easyblocks.append(full_name) - continue - - return invalid_easyblocks - - -def get_easyblock_entrypoints(name=None) -> dict: - """Get all easyblock entrypoints. - - Returns: - List of easyblocks. - """ - easyblocks = {} - for ep in get_group_entrypoints(EASYBLOCK_ENTRYPOINT): - try: - eb = ep.load() - except Exception as e: - _log.warning(f"Error loading easyblock entry point {ep.name}: {e}") - raise EasyBuildError(f"Error loading easyblock entry point {ep.name}: {e}") - mod = importlib.import_module(eb.__module__) - - ptr = { - 'class': eb.__name__, - 'loc': mod.__file__, - } - easyblocks[f'{ep.module}'] = ptr - if name is not None: - for key, value in easyblocks.items(): - if value['class'] == name: - return {key: value} - if key == name: - return {key: value} - return {} - - return easyblocks - - -######################################################################################### -# Hooks entrypoints -def register_entrypoint_hooks(step, pre_step=False, post_step=False, priority=0): - """Decorator to add metadata on functions to be used as hooks. - - priority: integer, the priority of the hook, higher value means higher priority - """ - def decorator(func): - setattr(func, HOOKS_ENTRYPOINT_MARK, True) - setattr(func, HOOKS_ENTRYPOINT_STEP, step) - setattr(func, HOOKS_ENTRYPOINT_PRE_STEP, pre_step) - setattr(func, HOOKS_ENTRYPOINT_POST_STEP, post_step) - setattr(func, HOOKS_ENTRYPOINT_PRIORITY, priority) - - _log.info( - "Registering entry point hook '%s' 'pre=%s' 'post=%s' with priority %d", - func.__name__, pre_step, post_step, priority - ) - return func - return decorator - - -def validate_entrypoint_hooks(known_hooks: List[str], pre_prefix: str, post_prefix: str, suffix: str) -> List[str]: - """Validate all entrypoints hooks. - - Args: - known_hooks: List of known hooks. - pre_prefix: Prefix for pre hooks. - post_prefix: Prefix for post hooks. - suffix: Suffix for hooks. - - Returns: - List of invalid hooks. - """ - invalid_hooks = [] - for ep in get_group_entrypoints(HOOKS_ENTRYPOINT): - full_name = f'{ep.name} <{ep.value}>' - - hook = ep.load() - if not hasattr(hook, HOOKS_ENTRYPOINT_MARK): - invalid_hooks.append(f"{ep.name} <{ep.value}>") - _log.warning(f"Hook {ep.name} <{ep.value}> is not a valid EasyBuild hook") - continue - - if not callable(hook): - _log.warning(f"Hook {ep.name} <{ep.value}> is not callable") - invalid_hooks.append(full_name) - continue - - label = getattr(hook, HOOKS_ENTRYPOINT_STEP) - pre_cond = getattr(hook, HOOKS_ENTRYPOINT_PRE_STEP) - post_cond = getattr(hook, HOOKS_ENTRYPOINT_POST_STEP) + hook_name = f'{prefix}{self.step}{HOOK_SUFF}' - prefix = '' - if pre_cond: - prefix = pre_prefix - elif post_cond: - prefix = post_prefix - - hook_name = prefix + label + suffix - - if hook_name not in known_hooks: - _log.warning(f"Hook {full_name} does not match known hooks patterns") - invalid_hooks.append(full_name) - continue - - return invalid_hooks - - -def find_entrypoint_hooks(label, pre_step_hook=False, post_step_hook=False) -> List[Callable]: - """Get all hooks defined in entry points.""" - hooks = [] - for ep in get_group_entrypoints(HOOKS_ENTRYPOINT): - try: - hook = ep.load() - except Exception as e: - _log.warning(f"Error loading entry point {ep.name}: {e}") - raise EasyBuildError(f"Error loading entry point {ep.name}: {e}") - - cond = all([ - getattr(hook, HOOKS_ENTRYPOINT_STEP) == label, - getattr(hook, HOOKS_ENTRYPOINT_PRE_STEP) == pre_step_hook, - getattr(hook, HOOKS_ENTRYPOINT_POST_STEP) == post_step_hook, - ]) - if cond: - hooks.append(hook) - - return hooks - - -######################################################################################### -# Toolchain entrypoints -def register_toolchain_entrypoint(prepend=False): - def decorator(cls): + if hook_name not in KNOWN_HOOKS: + msg = f"Attempting to register unknown hook '{hook_name}'" + _log.warning(msg) + raise EasyBuildError(msg) + +class EntrypointEasyblock(EasybuildEntrypoint): + """Class to represent an easyblock entrypoint.""" + _group = EASYBLOCK_ENTRYPOINT + + def __init__(self): + super().__init__() + # Avoid circular imports by importing EasyBlock here + from easybuild.framework.easyblock import EasyBlock + self.expected_type = EasyBlock + + +class EntrypointToolchain(EasybuildEntrypoint): + """Class to represent a toolchain entrypoint.""" + _group = TOOLCHAIN_ENTRYPOINT + + def __init__(self, prepend=False): + super().__init__() + # Avoid circular imports by importing Toolchain here from easybuild.tools.toolchain.toolchain import Toolchain - if not isinstance(cls, type) or not issubclass(cls, Toolchain): - raise EasyBuildError("Toolchain entrypoint `%s` is not a subclass of `Toolchain`", cls.__name__) - setattr(cls, TOOLCHAIN_ENTRYPOINT_MARK, True) - setattr(cls, TOOLCHAIN_ENTRYPOINT_PREPEND, prepend) - - _log.debug("Registering toolchain entrypoint: %s", cls.__name__) - return cls - - return decorator - - -def get_toolchain_entrypoints(): - """Get all toolchain entrypoints.""" - toolchains = [] - for ep in get_group_entrypoints(TOOLCHAIN_ENTRYPOINT): - try: - tc = ep.load() - except Exception as e: - _log.warning(f"Error loading toolchain entry point {ep.name}: {e}") - raise EasyBuildError(f"Error loading toolchain entry point {ep.name}: {e}") - toolchains.append(tc) - return toolchains - - -def validate_toolchain_entrypoints() -> List[str]: - """Validate all toolchain entrypoints.""" - invalid_toolchains = [] - for ep in get_group_entrypoints(TOOLCHAIN_ENTRYPOINT): - full_name = f'{ep.name} <{ep.value}>' - - tc = ep.load() - if not hasattr(tc, TOOLCHAIN_ENTRYPOINT_MARK): - invalid_toolchains.append(full_name) - _log.warning(f"Toolchain {ep.name} <{ep.value}> is not a valid EasyBuild toolchain") - continue - - return invalid_toolchains + self.expected_type = Toolchain + self.prepend = prepend diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 91e08c777f..1aca6f203c 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -32,10 +32,7 @@ import difflib import os -from easybuild.tools.entrypoints import ( - find_entrypoint_hooks, validate_entrypoint_hooks, - HOOKS_ENTRYPOINT_PRIORITY -) +from easybuild.tools.entrypoints import EntrypointHook from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg @@ -182,8 +179,6 @@ def verify_hooks(hooks): """Check whether obtained hooks only includes known hooks.""" unknown_hooks = [key for key in sorted(hooks) if key not in KNOWN_HOOKS] - unknown_hooks.extend(validate_entrypoint_hooks(KNOWN_HOOKS, PRE_PREF, POST_PREF, HOOK_SUFF)) - if unknown_hooks: error_lines = ["Found one or more unknown hooks:"] @@ -258,23 +253,25 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) res = hook(*args, **kwargs) - entrypoint_hooks = find_entrypoint_hooks(label=label, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) + entrypoint_hooks = EntrypointHook.get_entrypoints( + step=label, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook + ) if entrypoint_hooks: msg = "Running entry point %s hook..." % label if build_option('debug') and not build_option('silence_hook_trigger'): print_msg(msg) entrypoint_hooks.sort( - key=lambda x: (-getattr(x, HOOKS_ENTRYPOINT_PRIORITY, 0), x.__name__), + key=lambda x: (-x.priority, x.name), ) for hook in entrypoint_hooks: _log.info( "Running entry point '%s' hook function (args: %s, keyword args: %s)...", - hook.__name__, args, kwargs + hook.name, args, kwargs ) try: res = hook(*args, **kwargs) except Exception as e: - _log.warning("Error running entry point '%s' hook: %s", hook.__name__, e) - raise EasyBuildError("Error running entry point '%s' hook: %s", hook.__name__, e) from e + _log.warning("Error running entry point '%s' hook: %s", hook.name, e) + raise EasyBuildError("Error running entry point '%s' hook: %s", hook.name, e) from e return res diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8e9c369a08..561f5f0be6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -110,6 +110,7 @@ from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_os_type, get_system_info from easybuild.tools.utilities import flatten from easybuild.tools.version import this_is_easybuild +from easybuild.tools.entrypoints import EntrypointHook, EntrypointEasyblock, EntrypointToolchain try: @@ -1637,6 +1638,30 @@ def det_location(opt, prefix=''): pretty_print_opts(opts_dict) + if build_option('use_entrypoints', default=True): + hook_epts = EntrypointHook.get_entrypoints() + eb_epts = EntrypointEasyblock.get_entrypoints() + tc_epts = EntrypointToolchain.get_entrypoints() + + if hook_epts: + print() + print("Hooks from entrypoints (%d):" % len(hook_epts)) + for hook in hook_epts: + print('-', hook) + + if eb_epts: + print() + print("Easyblocks from entrypoints (%d):" % len(eb_epts)) + for eb in eb_epts: + print('-', eb) + + if tc_epts: + print() + print("Toolchains from entrypoints (%d):" % len(tc_epts)) + for tc in tc_epts: + print('-', tc) + + def parse_options(args=None, with_include=True): """wrapper function for option parsing""" diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 14306e4bbb..dd4c24d3da 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -40,10 +40,7 @@ import sys import easybuild.tools.toolchain -from easybuild.tools.entrypoints import ( - get_toolchain_entrypoints, validate_toolchain_entrypoints, - TOOLCHAIN_ENTRYPOINT_PREPEND, TOOLCHAIN_ENTRYPOINT_MARK -) +from easybuild.tools.entrypoints import EntrypointToolchain from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.toolchain import Toolchain @@ -73,12 +70,10 @@ def search_toolchain(name): # make sure all defined toolchain constants are available in toolchain module tc_const_re = re.compile('^%s(.*)$' % TC_CONST_PREFIX) - # print(f'!! TC_MODULES: {tc_modules}') for tc_mod in tc_modules: # determine classes imported in this module mod_classes = [] for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]: - # print('-----', elem) if hasattr(elem, '__module__'): # exclude the toolchain class defined in that module if not tc_mod.__file__ == sys.modules[elem.__module__].__file__: @@ -115,19 +110,11 @@ def search_toolchain(name): # Getting all subclasses will also include toolchains that are registered as entrypoints even if we are not # using the `--use-entrypoints` option, so we filter them out here and re-add them later if needed. - found_tcs = [x for x in found_tcs if not hasattr(x, TOOLCHAIN_ENTRYPOINT_MARK)] - - invalid_eps = validate_toolchain_entrypoints() - if invalid_eps: - _log.warning("Invalid toolchain entrypoints found: %s", ', '.join(invalid_eps)) - raise EasyBuildError("Invalid toolchain entrypoints found: %s", ', '.join(invalid_eps)) - prepend_eps = [] - append_eps = [] - for tc in get_toolchain_entrypoints(): - if getattr(tc, TOOLCHAIN_ENTRYPOINT_PREPEND): - prepend_eps.append(tc) - else: - append_eps.append(tc) + all_eps_names = [ep.wrapped.NAME for ep in EntrypointToolchain.get_entrypoints()] + found_tcs = [x for x in found_tcs if x.NAME not in all_eps_names] + + prepend_eps = [_.wrapped for _ in EntrypointToolchain.get_entrypoints(prepend=True)] + append_eps = [_.wrapped for _ in EntrypointToolchain.get_entrypoints(prepend=False)] found_tcs = prepend_eps + found_tcs + append_eps # filter found toolchain subclasses based on whether they can be used a toolchains diff --git a/test/framework/entrypoints.py b/test/framework/entrypoints.py index 2436087daf..1af5dd697a 100644 --- a/test/framework/entrypoints.py +++ b/test/framework/entrypoints.py @@ -37,13 +37,15 @@ from unittest import TextTestRunner import easybuild.tools.options as eboptions +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.entrypoints import ( get_group_entrypoints, HOOKS_ENTRYPOINT, EASYBLOCK_ENTRYPOINT, TOOLCHAIN_ENTRYPOINT, - HAVE_ENTRY_POINTS + HAVE_ENTRY_POINTS, EntrypointHook, EntrypointEasyblock, EntrypointToolchain, ) -from easybuild.framework.easyconfig.easyconfig import get_module_path +from easybuild.framework.easyconfig.easyconfig import get_easyblock_class +from easybuild.tools.hooks import START if HAVE_ENTRY_POINTS: @@ -57,25 +59,28 @@ MOCK_EASYBLOCK_EP_NAME = "mock_easyblock" MOCK_TOOLCHAIN_EP_NAME = "mock_toolchain" -MOCK_HOOK = "hello_world" -MOCK_EASYBLOCK = "TestEasyBlock" -MOCK_TOOLCHAIN = "MockTc" +MOCK_HOOK = "hello_world_12412412" +MOCK_EASYBLOCK = "TestEasyBlock_1212461" +MOCK_TOOLCHAIN = "MockTc_352124671346" MOCK_EP_FILE = f""" -from easybuild.tools.entrypoints import register_entrypoint_hooks +from easybuild.tools.entrypoints import EntrypointHook from easybuild.tools.hooks import CONFIGURE_STEP, START -@register_entrypoint_hooks(START) +@EntrypointHook(START) def {MOCK_HOOK}(): print("Hello, World! ----------------------------------------") +def {MOCK_HOOK}_invalid(): + print("This hook should not be registered, as it is invalid.") + ########################################################################## from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.entrypoints import register_easyblock_entrypoint +from easybuild.tools.entrypoints import EntrypointEasyblock -@register_easyblock_entrypoint() +@EntrypointEasyblock() class {MOCK_EASYBLOCK}(EasyBlock): def configure_step(self): print("{MOCK_EASYBLOCK}: configure_step called.") @@ -89,8 +94,11 @@ def install_step(self): def sanity_check_step(self): print("{MOCK_EASYBLOCK}: sanity_check_step called.") +class {MOCK_EASYBLOCK}_invalid(EasyBlock): + pass + ########################################################################## -from easybuild.tools.entrypoints import register_toolchain_entrypoint +from easybuild.tools.entrypoints import EntrypointToolchain from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, Compiler from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME @@ -100,25 +108,37 @@ class MockCompiler(Compiler): COMPILER_FAMILY = TC_CONSTANT_MOCK SUBTOOLCHAIN = SYSTEM_TOOLCHAIN_NAME -@register_toolchain_entrypoint() +@EntrypointToolchain() class {MOCK_TOOLCHAIN}(MockCompiler): NAME = '{MOCK_TOOLCHAIN}' # Using `...tc` to distinguish toolchain from package COMPILER_MODULE_NAME = [NAME] SUBTOOLCHAIN = [SYSTEM_TOOLCHAIN_NAME] + +class {MOCK_TOOLCHAIN}_invalid(MockCompiler): + pass """ MOCK_EP_META_FILE = f""" [{HOOKS_ENTRYPOINT}] -{MOCK_HOOK_EP_NAME} = {{module}}:hello_world +{MOCK_HOOK_EP_NAME} = {{module}}:{MOCK_HOOK} +{{invalid_hook}} [{EASYBLOCK_ENTRYPOINT}] -{MOCK_EASYBLOCK_EP_NAME} = {{module}}:TestEasyBlock +{MOCK_EASYBLOCK_EP_NAME} = {{module}}:{MOCK_EASYBLOCK} +{{invalid_easyblock}} [{TOOLCHAIN_ENTRYPOINT}] -{MOCK_TOOLCHAIN_EP_NAME} = {{module}}:MockTc +{MOCK_TOOLCHAIN_EP_NAME} = {{module}}:{MOCK_TOOLCHAIN} +{{invalid_toolchain}} """ +FORMAT_DCT = { + 'invalid_hook': '', + 'invalid_easyblock': '', + 'invalid_toolchain': '', +} + class MockDistribution(Distribution): """Mock distribution for testing entry points.""" @@ -127,7 +147,7 @@ def __init__(self, module): def read_text(self, filename): if filename == "entry_points.txt": - return MOCK_EP_META_FILE.format(module=self.module) + return MOCK_EP_META_FILE.format(module=self.module, **FORMAT_DCT) if filename == "METADATA": return "Name: mock_hook\nVersion: 0.1.0\n" @@ -148,12 +168,35 @@ class EasyBuildEntrypointsTest(EnhancedTestCase): tmpdir = None + def _run_mock_eb(self, args, strip=False, **kwargs): + """Helper function to mock easybuild runs + + Return (stdout, stderr) optionally stripped of whitespace at start/end + """ + with self.mocked_stdout_stderr() as (stdout, stderr): + self.eb_main(args, **kwargs) + stdout_txt = stdout.getvalue() + stderr_txt = stderr.getvalue() + if strip: + stdout_txt = stdout_txt.strip() + stderr_txt = stderr_txt.strip() + return stdout_txt, stderr_txt + def setUp(self): """Set up the test environment.""" + global FORMAT_DCT + + FORMAT_DCT = { + 'invalid_hook': '', + 'invalid_easyblock': '', + 'invalid_toolchain': '', + } + reload(eboptions) super().setUp() self.tmpdir = tempfile.mkdtemp(prefix='easybuild_test_') + if HAVE_ENTRY_POINTS: filename_root = "mock" dirname, dirpath = os.path.split(self.tmpdir) @@ -163,8 +206,8 @@ def setUp(self): sys.meta_path.insert(0, MockDistributionFinder(module=self.module)) # Create a mock entry point for testing - mock_hook_file = os.path.join(self.tmpdir, f'{filename_root}.py') - write_file(mock_hook_file, MOCK_EP_FILE) + self.mock_hook_file = os.path.join(self.tmpdir, f'{filename_root}.py') + write_file(self.mock_hook_file, MOCK_EP_FILE) else: self.skipTest("Entry points not available in this Python version") @@ -172,21 +215,71 @@ def tearDown(self): """Clean up the test environment.""" super().tearDown() - if self.tmpdir and os.path.isdir(self.tmpdir): + try: shutil.rmtree(self.tmpdir) + except OSError as err: + pass + tempfile.tempdir = None if HAVE_ENTRY_POINTS: # Remove the entry point from the working set - torm = [] dirname, _ = os.path.split(self.tmpdir) if dirname in sys.path: sys.path.remove(dirname) + torm = [] for idx, cls in enumerate(sys.meta_path): if isinstance(cls, MockDistributionFinder): torm.append(idx) for idx in reversed(torm): del sys.meta_path[idx] + EntrypointHook.clear() + + def test_entrypoints_register_hook(self): + """Test registering entry point hooks with both valid and invalid hook names.""" + func = lambda: None # Dummy function + + decorator = EntrypointHook('123') + with self.assertRaises(EasyBuildError): + decorator(func) + + decorator = EntrypointHook(START, pre_step=True) + with self.assertRaises(EasyBuildError): + decorator(func) + + decorator = EntrypointHook(START) + decorator(func) + + def test_entrypoints_register_easyblock(self): + """Test registering entry point easyblocks with both valid and invalid easyblock names.""" + from easybuild.framework.easyblock import EasyBlock + decorator = EntrypointEasyblock() + + with self.assertRaises(EasyBuildError): + decorator(123) + + class MOCK(): pass + with self.assertRaises(EasyBuildError): + decorator(MOCK) + + class MOCK(EasyBlock): pass + decorator(MOCK) + + def test_entrypoints_register_toolchain(self): + """Test registering entry point toolchains with both valid and invalid toolchain names.""" + from easybuild.tools.toolchain.toolchain import Toolchain + decorator = EntrypointToolchain() + + with self.assertRaises(EasyBuildError): + decorator(123) + + class MOCK(): pass + with self.assertRaises(EasyBuildError): + decorator(MOCK) + + class MOCK(Toolchain): pass + decorator(MOCK) + def test_entrypoints_get_group(self): """Test retrieving entrypoints for a specific group.""" expected = { @@ -209,12 +302,40 @@ def test_entrypoints_get_group(self): loaded_names = [ep.name for ep in epts] self.assertIn(expected[group], loaded_names, f"Expected entry point {expected[group]} in group {group}") + def test_entrypoints_exclude_invalid(self): + """Check that invalid entry points are excluded from the get_entrypoints function.""" + init_config(build_options={'use_entrypoints': True}) + + # Check that the invalid hook is not registered + + FORMAT_DCT['invalid_hook'] = f"{MOCK_HOOK_EP_NAME}_invalid = {self.module}:{MOCK_HOOK}_invalid" + FORMAT_DCT['invalid_easyblock'] = f"{MOCK_EASYBLOCK_EP_NAME}_invalid = {self.module}:{MOCK_EASYBLOCK}_invalid" + FORMAT_DCT['invalid_toolchain'] = f"{MOCK_TOOLCHAIN_EP_NAME}_invalid = {self.module}:{MOCK_TOOLCHAIN}_invalid" + + hooks = EntrypointHook.get_entrypoints() + self.assertNotIn( + MOCK_HOOK + '_invalid', [ep.name for ep in hooks], "Invalid hook should not be registered" + ) + + # Check that the invalid easyblock is not registered + easyblocks = EntrypointEasyblock.get_entrypoints() + self.assertNotIn( + MOCK_EASYBLOCK + '_invalid', [ep.name for ep in easyblocks], "Invalid easyblock should not be registered" + ) + + # Check that the invalid toolchain is not registered + toolchains = EntrypointToolchain.get_entrypoints() + self.assertNotIn( + MOCK_TOOLCHAIN + '_invalid', [ep.name for ep in toolchains], "Invalid toolchain should not be registered" + ) + def test_entrypoints_list_easyblocks(self): """ Tests for list_easyblocks function with entry points enabled. """ - txt = list_easyblocks() - self.assertNotIn("TestEasyBlock", txt, "TestEasyBlock should not be listed without entry points enabled") + # Invalid EBs are still picked up as subclasses of EasyBlock, difficult to exclude them from this behavior + # txt = list_easyblocks() + # self.assertNotIn("TestEasyBlock", txt, "TestEasyBlock should not be listed without entry points enabled") init_config(build_options={'use_entrypoints': True}) txt = list_easyblocks() @@ -224,25 +345,43 @@ def test_entrypoints_list_toolchains(self): """ Tests for list_toolchains function with entry points enabled. """ - txt = list_toolchains() - self.assertNotIn(MOCK_TOOLCHAIN, txt, f"{MOCK_TOOLCHAIN} should not be listed without entry points enabled") + # Invalid TCs are still picked up as subclasses of Toolchain, difficult to exclude them from this behavior + # txt = list_toolchains() + # self.assertNotIn(MOCK_TOOLCHAIN, txt, f"{MOCK_TOOLCHAIN} should not be listed without entry points enabled") init_config(build_options={'use_entrypoints': True}) txt = list_toolchains() self.assertIn(MOCK_TOOLCHAIN, txt, f"{MOCK_TOOLCHAIN} should be listed with entry points enabled") - def test_entrypoints_get_module_path(self): + def test_entrypoints_get_easyblock_class(self): """ - Tests for get_module_path function with entry points enabled. + Tests for get_easyblock_class function with entry points enabled. """ - module_path = get_module_path(MOCK_EASYBLOCK) - self.assertIn('.generic.', module_path, "Module path should contain '.generic.'") + with self.assertRaises(EasyBuildError): + get_easyblock_class(MOCK_EASYBLOCK) + # self.assertIn('.generic.', module_path, "Module path should contain '.generic.'") init_config(build_options={'use_entrypoints': True}) # Reload the EasyBlock module to ensure it is recognized - module_path = get_module_path(MOCK_EASYBLOCK) - self.assertEqual(module_path, self.module, "Module path should match the mock module path") + cls = get_easyblock_class(MOCK_EASYBLOCK) + self.assertEqual(cls.__module__, self.module, "Module path should match the mock module path") + + def test_entrypoints_show_config(self): + """Test that showing configuration includes entry points.""" + args = ['--show-config'] + stdout, stderr = self._run_mock_eb(args, strip=True) + + for name in ['Hooks', 'Easyblocks', 'Toolchains']: + pattern = f"{name} from entrypoints (" + self.assertIn(pattern, stdout, f"Expected {name} in configuration output") + + args = ['--show-full-config'] + stdout, stderr = self._run_mock_eb(args, strip=True) + + for name in ['Hooks', 'Easyblocks', 'Toolchains']: + pattern = f"{name} from entrypoints (" + self.assertIn(pattern, stdout, f"Expected {name} in configuration output") def suite(): From 1a8e270efeb140fc67071e91dcd15ade15c1da32 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 19 Jun 2025 12:04:43 +0200 Subject: [PATCH 14/19] lint --- easybuild/tools/entrypoints.py | 3 ++- test/framework/entrypoints.py | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 196daf5fcc..5b3edf8290 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -78,7 +78,7 @@ def __call__(self, wrap): check = False try: check = isinstance(wrap, self.expected_type) or issubclass(wrap, self.expected_type) - except: + except Exception: pass if not check: raise EasyBuildError( @@ -177,6 +177,7 @@ def validate(self): _log.warning(msg) raise EasyBuildError(msg) + class EntrypointEasyblock(EasybuildEntrypoint): """Class to represent an easyblock entrypoint.""" _group = EASYBLOCK_ENTRYPOINT diff --git a/test/framework/entrypoints.py b/test/framework/entrypoints.py index 1af5dd697a..5b4b755b5d 100644 --- a/test/framework/entrypoints.py +++ b/test/framework/entrypoints.py @@ -196,7 +196,6 @@ def setUp(self): super().setUp() self.tmpdir = tempfile.mkdtemp(prefix='easybuild_test_') - if HAVE_ENTRY_POINTS: filename_root = "mock" dirname, dirpath = os.path.split(self.tmpdir) @@ -217,7 +216,7 @@ def tearDown(self): try: shutil.rmtree(self.tmpdir) - except OSError as err: + except OSError: pass tempfile.tempdir = None @@ -237,7 +236,9 @@ def tearDown(self): def test_entrypoints_register_hook(self): """Test registering entry point hooks with both valid and invalid hook names.""" - func = lambda: None # Dummy function + # Dummy function + def func(): + return decorator = EntrypointHook('123') with self.assertRaises(EasyBuildError): @@ -258,11 +259,13 @@ def test_entrypoints_register_easyblock(self): with self.assertRaises(EasyBuildError): decorator(123) - class MOCK(): pass + class MOCK(): + pass with self.assertRaises(EasyBuildError): decorator(MOCK) - class MOCK(EasyBlock): pass + class MOCK(EasyBlock): + pass decorator(MOCK) def test_entrypoints_register_toolchain(self): @@ -273,11 +276,13 @@ def test_entrypoints_register_toolchain(self): with self.assertRaises(EasyBuildError): decorator(123) - class MOCK(): pass + class MOCK(): + pass with self.assertRaises(EasyBuildError): decorator(MOCK) - class MOCK(Toolchain): pass + class MOCK(Toolchain): + pass decorator(MOCK) def test_entrypoints_get_group(self): From 6a8346f536e6aaf2af1631825b2b0ddd86d39aff Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 19 Jun 2025 12:41:07 +0200 Subject: [PATCH 15/19] Lint and fixes --- easybuild/tools/entrypoints.py | 2 +- easybuild/tools/options.py | 33 +++++++++++---------------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 5b3edf8290..e46d9fb08d 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -122,7 +122,7 @@ def get_entrypoints(cls, name=None, **filter_params): ep.load() except Exception as e: msg = f"Error loading entrypoint {ep}: {e}" - _log.error(msg) + _log.warning(msg) raise EasyBuildError(msg) from e entrypoints = [] diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 561f5f0be6..1110aa32c7 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1639,28 +1639,17 @@ def det_location(opt, prefix=''): pretty_print_opts(opts_dict) if build_option('use_entrypoints', default=True): - hook_epts = EntrypointHook.get_entrypoints() - eb_epts = EntrypointEasyblock.get_entrypoints() - tc_epts = EntrypointToolchain.get_entrypoints() - - if hook_epts: - print() - print("Hooks from entrypoints (%d):" % len(hook_epts)) - for hook in hook_epts: - print('-', hook) - - if eb_epts: - print() - print("Easyblocks from entrypoints (%d):" % len(eb_epts)) - for eb in eb_epts: - print('-', eb) - - if tc_epts: - print() - print("Toolchains from entrypoints (%d):" % len(tc_epts)) - for tc in tc_epts: - print('-', tc) - + for prefix, cls in [ + ('Hook', EntrypointHook), + ('Easyblock', EntrypointEasyblock), + ('Toolchain', EntrypointToolchain), + ]: + ept_list = cls.get_entrypoints() + if ept_list: + print() + print("%ss from entrypoints (%d):" % (prefix, len(ept_list))) + for ept in ept_list: + print('-', ept) def parse_options(args=None, with_include=True): From 6f017f22159889a22e0b8b911ee5aa02d4de8266 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 19 Jun 2025 13:56:23 +0200 Subject: [PATCH 16/19] Cleanup and enhanced test-cases to run correct hooks in correct order --- easybuild/tools/docs.py | 3 -- easybuild/tools/hooks.py | 8 +--- test/framework/entrypoints.py | 86 ++++++++++++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 5bbe6cb3d5..d7fba3885d 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -724,9 +724,6 @@ def gen_list_easyblocks(list_easyblocks, format_strings): def add_class(classes, cls): """Add a new class, and all of its subclasses.""" children = cls.__subclasses__() - # Filter out possible sublcasses coming from entrypoints based on build option - # if not build_option('use_entrypoints', default=False): - # children = [c for c in children if not hasattr(c, EASYBLOCK_ENTRYPOINT_MARK)] classes.update({cls.__name__: { 'module': cls.__module__, 'children': sorted([c.__name__ for c in children], key=lambda x: x.lower()) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 1aca6f203c..6f0a596d40 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -78,7 +78,6 @@ PRE_PREF = 'pre_' POST_PREF = 'post_' HOOK_SUFF = '_hook' -ENTRYPOINT_PRE = 'entrypoint_' # list of names for steps in installation procedure (in order of execution) STEP_NAMES = [FETCH_STEP, READY_STEP, EXTRACT_STEP, PATCH_STEP, PREPARE_STEP, CONFIGURE_STEP, BUILD_STEP, TEST_STEP, @@ -234,7 +233,6 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, :param args: arguments to pass to hook function :param msg: custom message that is printed when hook is called """ - # print(f"Running hook '{label}' {pre_step_hook=} {post_step_hook=}") hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) res = None args = args or [] @@ -253,9 +251,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) res = hook(*args, **kwargs) - entrypoint_hooks = EntrypointHook.get_entrypoints( - step=label, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook - ) + entrypoint_hooks = EntrypointHook.get_entrypoints(step=label, pre_step=pre_step_hook, post_step=post_step_hook) if entrypoint_hooks: msg = "Running entry point %s hook..." % label if build_option('debug') and not build_option('silence_hook_trigger'): @@ -269,7 +265,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, hook.name, args, kwargs ) try: - res = hook(*args, **kwargs) + res = hook.wrapped(*args, **kwargs) except Exception as e: _log.warning("Error running entry point '%s' hook: %s", hook.name, e) raise EasyBuildError("Error running entry point '%s' hook: %s", hook.name, e) from e diff --git a/test/framework/entrypoints.py b/test/framework/entrypoints.py index 5b4b755b5d..33ef21ffff 100644 --- a/test/framework/entrypoints.py +++ b/test/framework/entrypoints.py @@ -38,14 +38,14 @@ import easybuild.tools.options as eboptions from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import write_file from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.entrypoints import ( get_group_entrypoints, HOOKS_ENTRYPOINT, EASYBLOCK_ENTRYPOINT, TOOLCHAIN_ENTRYPOINT, HAVE_ENTRY_POINTS, EntrypointHook, EntrypointEasyblock, EntrypointToolchain, ) +from easybuild.tools.filetools import write_file +from easybuild.tools.hooks import run_hook, START, CONFIGURE_STEP from easybuild.framework.easyconfig.easyconfig import get_easyblock_class -from easybuild.tools.hooks import START if HAVE_ENTRY_POINTS: @@ -388,6 +388,88 @@ def test_entrypoints_show_config(self): pattern = f"{name} from entrypoints (" self.assertIn(pattern, stdout, f"Expected {name} in configuration output") + def test_entrypoints_register_invalid_hook(self): + """Test that registering an invalid hook steps raises an error.""" + # Invalid step name + with self.assertRaises(EasyBuildError): + EntrypointHook('invalid_hook_name')(lambda: None) + + # START does not have the pre/post prefixes + with self.assertRaises(EasyBuildError): + EntrypointHook(START, pre_step=True)(lambda: None) + + # CONFIGURE_STEP must have a pre/post prefix + with self.assertRaises(EasyBuildError): + EntrypointHook(CONFIGURE_STEP)(lambda: None) + + def test_entrypoints_run_hook(self): + """Ensure that entry point hooks are run in the correct order.""" + cnt = 0 + @EntrypointHook(START, priority=50) + def func2_2(): + nonlocal cnt + self.assertEqual(cnt, 2, "This hook should be run third because of name ordering") + cnt += 1 + + @EntrypointHook(START, priority=50) + def func2_1(): + nonlocal cnt + self.assertEqual(cnt, 1, "This hook should be run second because of name ordering") + cnt += 1 + + @EntrypointHook(START, priority=10) + def func3(): + nonlocal cnt + self.assertEqual(cnt, 3, "This hook should be run last") + cnt += 1 + + @EntrypointHook(START, priority=100) + def func1(): + nonlocal cnt + self.assertEqual(cnt, 0, "This hook should be run first") + cnt += 1 + + @EntrypointHook(CONFIGURE_STEP, pre_step=True) + def func_configure_pre(): + nonlocal cnt + self.assertEqual(cnt, 4, "This hook should be run after all START hooks") + cnt += 1 + + run_hook(START, {}) + + self.assertEqual(cnt, 4, "All hooks should have been run in the correct order") + + def test_entrypoints_run_hook_onlyreq(self): + """Ensure that only the hooks required for a step are run.""" + tpl_flags = {'start': False, 'pre_cfg': False, 'post_cfg': False} + + @EntrypointHook(START) + def func_start(): + flags['start'] = True + + @EntrypointHook(CONFIGURE_STEP, pre_step=True) + def func_configure_pre(): + flags['pre_cfg'] = True + + @EntrypointHook(CONFIGURE_STEP, post_step=True) + def func_configure_post(): + flags['post_cfg'] = True + + flags = tpl_flags.copy() + run_hook(START, {}) + for key, val in flags.items(): + self.assertEqual(val, key == 'start', "Should only run START hooks") + + flags = tpl_flags.copy() + run_hook(CONFIGURE_STEP, {}, pre_step_hook=True) + for key, val in flags.items(): + self.assertEqual(val, key == 'pre_cfg', "Should only run pre-configure hooks") + + flags = tpl_flags.copy() + run_hook(CONFIGURE_STEP, {}, post_step_hook=True) + for key, val in flags.items(): + self.assertEqual(val, key == 'post_cfg', "Should only run post-configure hooks") + def suite(): return TestLoaderFiltered().loadTestsFromTestCase(EasyBuildEntrypointsTest, sys.argv[1:]) From 522411347188e73e899492621b6318650b208f1b Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 19 Jun 2025 14:52:25 +0200 Subject: [PATCH 17/19] lint --- test/framework/entrypoints.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/entrypoints.py b/test/framework/entrypoints.py index 33ef21ffff..c158eb4d48 100644 --- a/test/framework/entrypoints.py +++ b/test/framework/entrypoints.py @@ -405,6 +405,7 @@ def test_entrypoints_register_invalid_hook(self): def test_entrypoints_run_hook(self): """Ensure that entry point hooks are run in the correct order.""" cnt = 0 + @EntrypointHook(START, priority=50) def func2_2(): nonlocal cnt From 08f096ac345b8e1759a0077e4f1d6f32650dc556 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 20 Jun 2025 14:02:10 +0200 Subject: [PATCH 18/19] - Readded typehints - Code cleanup - Better function names - Improved tests/comments/docstrints --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/tools/entrypoints.py | 100 ++++++++++--------- easybuild/tools/hooks.py | 2 +- easybuild/tools/options.py | 2 +- easybuild/tools/toolchain/utilities.py | 6 +- test/framework/entrypoints.py | 51 ++++++---- 7 files changed, 89 insertions(+), 76 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 8316fa1119..1164f959b9 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -2017,7 +2017,7 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error class_name, modulepath) cls = get_class_for(modulepath, class_name) else: - eb_from_eps = EntrypointEasyblock.get_entrypoints(name=easyblock) + eb_from_eps = EntrypointEasyblock.get_loaded_entrypoints(name=easyblock) if eb_from_eps: ep = eb_from_eps[0] cls = ep.wrapped diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 25cf469db8..d2b12ba357 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -800,7 +800,7 @@ def avail_easyblocks(): else: raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc) - ept_eb_lst = EntrypointEasyblock.get_entrypoints() + ept_eb_lst = EntrypointEasyblock.get_loaded_entrypoints() for ept_eb in ept_eb_lst: easyblocks[ept_eb.module] = { diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index e46d9fb08d..d69dc3b778 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -10,61 +10,39 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError +from typing import TypeVar, List, Callable, Set, Any + +_T = TypeVar('_T') HAVE_ENTRY_POINTS = False HAVE_ENTRY_POINTS_CLS = False if sys.version_info >= (3, 8): HAVE_ENTRY_POINTS = True - from importlib.metadata import entry_points + from importlib.metadata import entry_points, EntryPoint +else: + EntryPoint = Any if sys.version_info >= (3, 10): # Python >= 3.10 uses importlib.metadata.EntryPoints as a type for entry_points() HAVE_ENTRY_POINTS_CLS = True -EASYBLOCK_ENTRYPOINT = "easybuild.easyblock" -TOOLCHAIN_ENTRYPOINT = "easybuild.toolchain" -HOOKS_ENTRYPOINT = "easybuild.hooks" - _log = fancylogger.getLogger('entrypoints', fname=False) -def get_group_entrypoints(group: str): - """Get all entrypoints for a group""" - strict_python = True - use_eps = build_option('use_entrypoints', default=None) - if use_eps is None: - # Default True needed to work with commands like --list-toolchains that do not initialize the BuildOptions - use_eps = True - # Needed to work with older Python versions: do not raise errors when entry points are default enabled - strict_python = False - res = set() - if use_eps: - if not HAVE_ENTRY_POINTS: - if strict_python: - msg = "`--use-entrypoints` requires importlib.metadata (Python >= 3.8)" - _log.warning(msg) - raise EasyBuildError(msg) - else: - _log.debug("`get_group_entrypoints` called before BuildOptions initialized, with python < 3.8") - else: - if HAVE_ENTRY_POINTS_CLS: - res = set(entry_points(group=group)) - else: - res = set(entry_points().get(group, [])) - - return res - - class EasybuildEntrypoint: - _group = None + group = None expected_type = None registered = {} def __init__(self): - self.wrapped = None + if self.group is None: + raise EasyBuildError( + "Cannot use drirectly. Please use a subclass that defines `group`", + ) + self.wrapped = None self.module = None self.name = None self.file = None @@ -72,7 +50,7 @@ def __init__(self): def __repr__(self): return f"{self.__class__.__name__} <{self.module}:{self.name}>" - def __call__(self, wrap): + def __call__(self, wrap: _T) -> _T: """Use an instance of this class as a decorator to register an entrypoint.""" if self.expected_type is not None: check = False @@ -108,16 +86,39 @@ def __call__(self, wrap): return wrap - @property - def group(self): - if self._group is None: - raise EasyBuildError("Entrypoint group not set for %s", self.__class__.__name__) - return self._group + @classmethod + def retrieve_entrypoints(cls) -> Set[EntryPoint]: + """"Get all entrypoints in this group.""" + strict_python = True + use_eps = build_option('use_entrypoints', default=None) + if use_eps is None: + # Default True needed to work with commands like --list-toolchains that do not initialize the BuildOptions + use_eps = True + # Needed to work with older Python versions: do not raise errors when entry points are default enabled + strict_python = False + res = set() + if use_eps: + if not HAVE_ENTRY_POINTS: + if strict_python: + msg = "`--use-entrypoints` requires importlib.metadata (Python >= 3.8)" + _log.warning(msg) + raise EasyBuildError(msg) + else: + _log.debug("`get_group_entrypoints` called before BuildOptions initialized, with python < 3.8") + else: + if HAVE_ENTRY_POINTS_CLS: + res = set(entry_points(group=cls.group)) + else: + res = set(entry_points().get(cls.group, [])) + + return res @classmethod - def get_entrypoints(cls, name=None, **filter_params): - """Get all entrypoints in this group.""" - for ep in get_group_entrypoints(cls._group): + def load_entrypoints(cls): + """Load all the entrypoints in this group. This is needed for the modules contining the entrypoints to be + actually imported in order to process the function decorators that will register them in the + `registered` dict.""" + for ep in cls.retrieve_entrypoints(): try: ep.load() except Exception as e: @@ -125,8 +126,13 @@ def get_entrypoints(cls, name=None, **filter_params): _log.warning(msg) raise EasyBuildError(msg) from e + @classmethod + def get_loaded_entrypoints(cls: _T, name: str = None, **filter_params) -> List[_T]: + """Get all entrypoints in this group.""" + cls.load_entrypoints() + entrypoints = [] - for ep in cls.registered.get(cls._group, []): + for ep in cls.registered.get(cls.group, []): cond = name is None or ep.name == name for key, value in filter_params.items(): cond = cond and getattr(ep, key, None) == value @@ -149,7 +155,7 @@ def validate(self): class EntrypointHook(EasybuildEntrypoint): """Class to represent a hook entrypoint.""" - _group = HOOKS_ENTRYPOINT + group = 'easybuild.hooks' def __init__(self, step, pre_step=False, post_step=False, priority=0): """Initialize the EntrypointHook.""" @@ -180,7 +186,7 @@ def validate(self): class EntrypointEasyblock(EasybuildEntrypoint): """Class to represent an easyblock entrypoint.""" - _group = EASYBLOCK_ENTRYPOINT + group = 'easybuild.easyblock' def __init__(self): super().__init__() @@ -191,7 +197,7 @@ def __init__(self): class EntrypointToolchain(EasybuildEntrypoint): """Class to represent a toolchain entrypoint.""" - _group = TOOLCHAIN_ENTRYPOINT + group = 'easybuild.toolchain' def __init__(self, prepend=False): super().__init__() diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 6f0a596d40..00245d8c32 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -251,7 +251,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) res = hook(*args, **kwargs) - entrypoint_hooks = EntrypointHook.get_entrypoints(step=label, pre_step=pre_step_hook, post_step=post_step_hook) + entrypoint_hooks = EntrypointHook.get_loaded_entrypoints(step=label, pre_step=pre_step_hook, post_step=post_step_hook) if entrypoint_hooks: msg = "Running entry point %s hook..." % label if build_option('debug') and not build_option('silence_hook_trigger'): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 1110aa32c7..aea28cc3c9 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1644,7 +1644,7 @@ def det_location(opt, prefix=''): ('Easyblock', EntrypointEasyblock), ('Toolchain', EntrypointToolchain), ]: - ept_list = cls.get_entrypoints() + ept_list = cls.retrieve_entrypoints() if ept_list: print() print("%ss from entrypoints (%d):" % (prefix, len(ept_list))) diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index dd4c24d3da..733df5ceec 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -110,11 +110,11 @@ def search_toolchain(name): # Getting all subclasses will also include toolchains that are registered as entrypoints even if we are not # using the `--use-entrypoints` option, so we filter them out here and re-add them later if needed. - all_eps_names = [ep.wrapped.NAME for ep in EntrypointToolchain.get_entrypoints()] + all_eps_names = [ep.wrapped.NAME for ep in EntrypointToolchain.get_loaded_entrypoints()] found_tcs = [x for x in found_tcs if x.NAME not in all_eps_names] - prepend_eps = [_.wrapped for _ in EntrypointToolchain.get_entrypoints(prepend=True)] - append_eps = [_.wrapped for _ in EntrypointToolchain.get_entrypoints(prepend=False)] + prepend_eps = [_.wrapped for _ in EntrypointToolchain.get_loaded_entrypoints(prepend=True)] + append_eps = [_.wrapped for _ in EntrypointToolchain.get_loaded_entrypoints(prepend=False)] found_tcs = prepend_eps + found_tcs + append_eps # filter found toolchain subclasses based on whether they can be used a toolchains diff --git a/test/framework/entrypoints.py b/test/framework/entrypoints.py index c158eb4d48..ce925b9365 100644 --- a/test/framework/entrypoints.py +++ b/test/framework/entrypoints.py @@ -40,8 +40,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.entrypoints import ( - get_group_entrypoints, HOOKS_ENTRYPOINT, EASYBLOCK_ENTRYPOINT, TOOLCHAIN_ENTRYPOINT, - HAVE_ENTRY_POINTS, EntrypointHook, EntrypointEasyblock, EntrypointToolchain, + HAVE_ENTRY_POINTS, EntrypointHook, EntrypointEasyblock, EntrypointToolchain, EasybuildEntrypoint ) from easybuild.tools.filetools import write_file from easybuild.tools.hooks import run_hook, START, CONFIGURE_STEP @@ -120,15 +119,15 @@ class {MOCK_TOOLCHAIN}_invalid(MockCompiler): MOCK_EP_META_FILE = f""" -[{HOOKS_ENTRYPOINT}] +[{EntrypointHook.group}] {MOCK_HOOK_EP_NAME} = {{module}}:{MOCK_HOOK} {{invalid_hook}} -[{EASYBLOCK_ENTRYPOINT}] +[{EntrypointEasyblock.group}] {MOCK_EASYBLOCK_EP_NAME} = {{module}}:{MOCK_EASYBLOCK} {{invalid_easyblock}} -[{TOOLCHAIN_ENTRYPOINT}] +[{EntrypointToolchain.group}] {MOCK_TOOLCHAIN_EP_NAME} = {{module}}:{MOCK_TOOLCHAIN} {{invalid_toolchain}} """ @@ -234,22 +233,27 @@ def tearDown(self): EntrypointHook.clear() + def test_entrypoints_baseclass_raises(self): + """Test that attempting to register an entry point with the base class raises an error.""" + with self.assertRaises(EasyBuildError): + EasybuildEntrypoint()(lambda: None) + def test_entrypoints_register_hook(self): """Test registering entry point hooks with both valid and invalid hook names.""" # Dummy function def func(): return - decorator = EntrypointHook('123') + # Invalid step name with self.assertRaises(EasyBuildError): - decorator(func) + EntrypointHook('123')(func) - decorator = EntrypointHook(START, pre_step=True) + # Valid name but invalid combination of step and pre/post with self.assertRaises(EasyBuildError): - decorator(func) + EntrypointHook(START, pre_step=True)(func) - decorator = EntrypointHook(START) - decorator(func) + # Valid hook registration + EntrypointHook(START)(func) def test_entrypoints_register_easyblock(self): """Test registering entry point easyblocks with both valid and invalid easyblock names.""" @@ -288,24 +292,27 @@ class MOCK(Toolchain): def test_entrypoints_get_group(self): """Test retrieving entrypoints for a specific group.""" expected = { - HOOKS_ENTRYPOINT: MOCK_HOOK_EP_NAME, - EASYBLOCK_ENTRYPOINT: MOCK_EASYBLOCK_EP_NAME, - TOOLCHAIN_ENTRYPOINT: MOCK_TOOLCHAIN_EP_NAME, + EntrypointHook: MOCK_HOOK_EP_NAME, + EntrypointEasyblock: MOCK_EASYBLOCK_EP_NAME, + EntrypointToolchain: MOCK_TOOLCHAIN_EP_NAME, } - for group in [HOOKS_ENTRYPOINT, EASYBLOCK_ENTRYPOINT, TOOLCHAIN_ENTRYPOINT]: - epts = get_group_entrypoints(group) + for ep_type in [EntrypointHook, EntrypointEasyblock, EntrypointToolchain]: + group = ep_type.group + epts = ep_type.retrieve_entrypoints() self.assertIsInstance(epts, set, f"Expected set for group {group}") self.assertEqual(len(epts), 0, f"Expected non-empty set for group {group}") init_config(build_options={'use_entrypoints': True}) - for group in [HOOKS_ENTRYPOINT, EASYBLOCK_ENTRYPOINT, TOOLCHAIN_ENTRYPOINT]: - epts = get_group_entrypoints(group) + for ep_type in [EntrypointHook, EntrypointEasyblock, EntrypointToolchain]: + group = ep_type.group + epts = ep_type.retrieve_entrypoints() self.assertIsInstance(epts, set, f"Expected set for group {group}") self.assertGreater(len(epts), 0, f"Expected non-empty set for group {group}") loaded_names = [ep.name for ep in epts] - self.assertIn(expected[group], loaded_names, f"Expected entry point {expected[group]} in group {group}") + expt = expected[ep_type] + self.assertIn(expt, loaded_names, f"Expected entry point {expt} in group {group}") def test_entrypoints_exclude_invalid(self): """Check that invalid entry points are excluded from the get_entrypoints function.""" @@ -317,19 +324,19 @@ def test_entrypoints_exclude_invalid(self): FORMAT_DCT['invalid_easyblock'] = f"{MOCK_EASYBLOCK_EP_NAME}_invalid = {self.module}:{MOCK_EASYBLOCK}_invalid" FORMAT_DCT['invalid_toolchain'] = f"{MOCK_TOOLCHAIN_EP_NAME}_invalid = {self.module}:{MOCK_TOOLCHAIN}_invalid" - hooks = EntrypointHook.get_entrypoints() + hooks = EntrypointHook.get_loaded_entrypoints() self.assertNotIn( MOCK_HOOK + '_invalid', [ep.name for ep in hooks], "Invalid hook should not be registered" ) # Check that the invalid easyblock is not registered - easyblocks = EntrypointEasyblock.get_entrypoints() + easyblocks = EntrypointEasyblock.get_loaded_entrypoints() self.assertNotIn( MOCK_EASYBLOCK + '_invalid', [ep.name for ep in easyblocks], "Invalid easyblock should not be registered" ) # Check that the invalid toolchain is not registered - toolchains = EntrypointToolchain.get_entrypoints() + toolchains = EntrypointToolchain.get_loaded_entrypoints() self.assertNotIn( MOCK_TOOLCHAIN + '_invalid', [ep.name for ep in toolchains], "Invalid toolchain should not be registered" ) From 2d33f2ba0f8496bb08802ca910f6b5c4b9a3e10b Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 20 Jun 2025 14:04:08 +0200 Subject: [PATCH 19/19] Lint and improved validation --- easybuild/tools/entrypoints.py | 5 ++++- easybuild/tools/hooks.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index d69dc3b778..447e5eda28 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -10,7 +10,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from typing import TypeVar, List, Callable, Set, Any +from typing import TypeVar, List, Set, Any _T = TypeVar('_T') @@ -170,6 +170,9 @@ def validate(self): from easybuild.tools.hooks import KNOWN_HOOKS, HOOK_SUFF, PRE_PREF, POST_PREF super().validate() + if not callable(self.wrapped): + raise EasyBuildError("Hook entrypoint `%s` is not callable", self.wrapped) + prefix = '' if self.pre_step: prefix = PRE_PREF diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 00245d8c32..8e95f23c01 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -251,7 +251,9 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) res = hook(*args, **kwargs) - entrypoint_hooks = EntrypointHook.get_loaded_entrypoints(step=label, pre_step=pre_step_hook, post_step=post_step_hook) + entrypoint_hooks = EntrypointHook.get_loaded_entrypoints( + step=label, pre_step=pre_step_hook, post_step=post_step_hook + ) if entrypoint_hooks: msg = "Running entry point %s hook..." % label if build_option('debug') and not build_option('silence_hook_trigger'):