diff --git a/plux/core/entrypoint.py b/plux/core/entrypoint.py index 1049e38..2ac6c33 100644 --- a/plux/core/entrypoint.py +++ b/plux/core/entrypoint.py @@ -6,9 +6,19 @@ class EntryPoint(t.NamedTuple): + """ + Lightweight data structure to represent an entry point. You can find out more about the data model here: + https://packaging.python.org/en/latest/specifications/entry-points/#data-model + """ + name: str + """The name identifies this entry point within its group. The precise meaning of this is up to the consumer. + Within a group, entry point names should be unique.""" value: str + """The object reference points to a Python object. It is either in the form ``importable.module``, or + importable.module:object.attr.""" group: str + """The group that an entry point belongs to indicates what sort of object it provides.""" EntryPointDict = dict[str, list[str]] diff --git a/plux/core/plugin.py b/plux/core/plugin.py index 8a05d36..9affc21 100644 --- a/plux/core/plugin.py +++ b/plux/core/plugin.py @@ -3,8 +3,15 @@ import inspect import typing as t +if t.TYPE_CHECKING: + from importlib.metadata import EntryPoint + class PluginException(Exception): + """ + Generic ``PluginException`` that may be raised by the ``PluginManager``. + """ + def __init__(self, message, namespace: str = None, name: str = None) -> None: super().__init__(message) self.namespace = namespace @@ -12,6 +19,12 @@ def __init__(self, message, namespace: str = None, name: str = None) -> None: class PluginDisabled(PluginException): + """ + Exception that can be raised by a ``Plugin`` or a ``PluginLifecycleListener`` to indicate to the ``PluginManager`` + that this plugin should be disabled. Later calls to ``PluginManager.load(...)``, will then also raise a + ``PluginDisabled`` exception. + """ + reason: str def __init__(self, namespace: str, name: str, reason: str = None): @@ -25,24 +38,64 @@ def __init__(self, namespace: str, name: str, reason: str = None): class Plugin(abc.ABC): - """A generic LocalStack plugin. + """ + A generic Plugin. - A Plugin's purpose is to be loaded dynamically at runtime and defer code imports into the Plugin::load method. + A Plugin's purpose is to be loaded dynamically at runtime and defer code imports into the ``Plugin.load`` method. Abstract subtypes of plugins (e.g., a LocalstackCliPlugin) may overwrite the load method with concrete call arguments that they agree upon with the PluginManager. In other words, a Plugin and a PluginManager for that - particular Plugin have an informal contracts to use the same argument types when load is invoked. + particular Plugin have an informal contract to use the same argument types when ``load`` is invoked. + + Note that ``PluginManagers`` will instantiate plugins without constructor args, but can pass arguments to the load + method. + + Here's an example plugin that lazily imports server code and returns a ``Server`` instance--an example + abstraction in your application:: + + class TwistedServerPlugin(Plugin): + namespace = "my.server_plugins" + name = "twisted" + + def load(self, host: str, port: int) -> Server: + from .twisted import TwistedServer + + return TwistedServer(host, port) + + Later through a plugin manager, this plugin can be loaded as:: + + server_name = "twisted" + pm = PluginManager("my.server_plugins", load_args=("localhost", 8080)) + server = pm.load(server_name) + server.start() """ namespace: str + """The namespace of the plugin (e.g., ``cli.commands``). Maps to the ``group`` attribute of an entrypoint.""" name: str + """The name of the plugin (e.g., ``my_command``). Maps to the ``name`` attribute of an entrypoint.""" requirements: list[str] + """The list of "depends on" clauses of the entry point, which is simply a list of strings and the meaning is up + to the consumer.""" def should_load(self) -> bool: + """ + Whether the ``PluginManager`` should load this plugin. This is called before the ``load`` method. Note that, + at the point where the ``PluginManager`` calls this function, the plugin has already been *resolved*, which + means that all ``import`` statements of the module where this plugin lives, have been executed. If you want to + defer imports, make sure to put them into the ``load`` method. + + By default, this method always returns True. + + :return: True if the plugin should be loaded, False otherwise. + """ return True - def load(self, *args, **kwargs): + def load(self, *args, **kwargs) -> t.Any | None: """ - Called by a PluginLoader when it loads the Plugin. + Called by a ``PluginLoader`` when it loads the Plugin. Ideally, any optional imports needed to run the plugin + are deferred to this method. + + :return: An optional return value of the load method, which is passed to ``PluginManager.load(...)``. """ return None @@ -57,15 +110,19 @@ class PluginSpec: imported code that can instantiate the plugin (a PluginFactory). In the simplest case, the PluginFactory that can just be the Plugin's class. - Internally a PluginSpec is essentially a wrapper around an importlib EntryPoint. An entrypoint is a tuple: ( - "name", "module:object") inside a namespace that can be loaded. The entrypoint object of a Plugin can point to a - PluginSpec, or a Plugin that defines its own namespace and name, in which case the PluginSpec will be instantiated - dynamically by, e.g., a PluginSpecResolver. + Internally, a PluginSpec is essentially a wrapper around an importlib EntryPoint. An entrypoint is a tuple: + ("group", "name", "module:object") inside a namespace that can be loaded. The entrypoint object of a Plugin can + point to a PluginSpec, or a Plugin that defines its own namespace and name, in which case the PluginSpec will be + instantiated dynamically by, e.g., a ``PluginSpecResolver``. """ namespace: str + """The namespace of the plugin (e.g., ``cli.commands``). Maps to the ``group`` attribute of an entrypoint.""" name: str + """The name of the plugin (e.g., ``my_command``). Maps to the ``name`` attribute of an entrypoint.""" factory: PluginFactory + """The factory that can instantiate the plugin. This is the imported code referenced in the ``value`` attribute of + the entrypoint.""" def __init__( self, @@ -90,11 +147,17 @@ def __eq__(self, other): class PluginFinder(abc.ABC): """ - Basic abstractions to find plugins, either at build time (e.g., using the PackagePathPluginFinder) or at run time - (e.g., using ``MetadataPluginFinder`` that finds plugins from entrypoints) + High-level abstractions to find plugins, either at build time (e.g., using the PackagePathPluginFinder) or at run + time (e.g., using ``MetadataPluginFinder`` that finds plugins from entrypoints) """ def find_plugins(self) -> list[PluginSpec]: + """ + Find and return a list of ``PluginSpec`` instances. The implementation will vary drastically depending on the + context in which the specific ``PluginFinder`` is used. + + :return: A list of ``PluginSpec`` instances. + """ raise NotImplementedError # pragma: no cover @@ -107,7 +170,7 @@ def resolve(self, source: t.Any) -> PluginSpec: """ Tries to create a PluginSpec from the given source. - :param source: anything that can produce a PluginSpec (Plugin class, ...) + :param source: Anything that can produce a PluginSpec (Plugin class, ...) :return: a PluginSpec instance """ if isinstance(source, PluginSpec): @@ -129,19 +192,60 @@ def resolve(self, source: t.Any) -> PluginSpec: class PluginLifecycleListener: # pragma: no cover """ - Listener that can be attached to a PluginManager to react to plugin lifecycle events. + Listener that can be attached to a ``PluginManager`` to react to plugin lifecycle events. + + A ``Plugin`` managed by a ``PluginManager`` can be in three lifecycle phases: + + * resolved: The entrypoint pointing to the PluginSpec was imported and the PluginSpec instance was created. + In technical terms, resolution is the process of calling importlib ``EntryPoint.load()`` and creating a + ``PluginSpec`` instance from the result. + * init: The ``PluginFactory`` of the ``PluginSpec`` was successfully invoked + * loaded: The load method of the ``Plugin`` was successfully invoked + + If an exception occurs during any of these steps, the plugin will be in an errored state. + + A lifecycle listener can hook into any of these steps and perform custom actions. """ - def on_resolve_exception(self, namespace: str, entrypoint, exception: Exception): + def on_resolve_exception(self, namespace: str, entrypoint: "EntryPoint", exception: Exception): + """ + This hook is called when an exception occurs during the resolution of a ``PluginSpec``. This can happen, for + example, when the module that holds the plugin raises an import exception, and therefore the ``PluginSpec`` + cannot be created. + + :param namespace: The namespace of the plugin that was being resolved + :param entrypoint: The importlib EntryPoint that was being used to resolve the plugin + :param exception: The exception that was raised during resolution + """ pass def on_resolve_after(self, plugin_spec: PluginSpec): + """ + This hook is called after the entry point was successfully loaded, and the ``PluginSpec`` instance was created. + + :param plugin_spec: The created ``PluginSpec`` instance. + """ pass def on_init_exception(self, plugin_spec: PluginSpec, exception: Exception): + """ + This hook is called when an exception occurs during the construction of a ``Plugin`` instance. A Plugin is + instantiated by calling the ``PluginFactory`` of the ``PluginSpec``. If that call raises an exception, this + hook is called. + + :param plugin_spec: The ``PluginSpec`` used to instantiate the plugin.. + :param exception: The exception that was raised during plugin initialization. + """ pass def on_init_after(self, plugin_spec: PluginSpec, plugin: Plugin): + """ + This hook is called after a ``Plugin`` instance has been successfully created from a ``PluginSpec``. + This happens when the ``PluginFactory`` of the ``PluginSpec`` is called and returns a ``Plugin`` instance. + + :param plugin_spec: The ``PluginSpec`` that was used to create the plugin. + :param plugin: The newly created ``Plugin`` instance. + """ pass def on_load_before( @@ -151,17 +255,47 @@ def on_load_before( load_args: list | tuple, load_kwargs: dict, ): + """ + This hook is called just before the ``load`` method of a ``Plugin`` is invoked. It provides access to the + arguments that will be passed to the plugin's load method. + + :param plugin_spec: The ``PluginSpec`` of the plugin being loaded. + :param plugin: The ``Plugin`` instance that is about to be loaded. + :param load_args: The positional arguments that will be passed to the plugin's ``load`` method. + :param load_kwargs: The keyword arguments that will be passed to the plugin's ``load`` method. + """ pass - def on_load_after(self, plugin_spec: PluginSpec, plugin: Plugin, load_result: t.Any = None): + def on_load_after(self, plugin_spec: PluginSpec, plugin: Plugin, load_result: t.Any | None = None): + """ + This hook is called after the ``load`` method of a ``Plugin`` has been successfully invoked. + It provides access to the result returned by the plugin's load method. + + :param plugin_spec: The ``PluginSpec`` of the plugin that was loaded. + :param plugin: The ``Plugin`` instance that was loaded. + :param load_result: The value returned by the plugin's ``load`` method, if any. + """ pass def on_load_exception(self, plugin_spec: PluginSpec, plugin: Plugin, exception: Exception): + """ + This hook is called when an exception occurs during the loading of a ``Plugin``. This happens when the + ``load`` method of the plugin raises an exception. + + :param plugin_spec: The ``PluginSpec`` of the plugin that failed to load. + :param plugin: The ``Plugin`` instance that failed to load. + :param exception: The exception that was raised during the loading process. + """ pass class CompositePluginLifecycleListener(PluginLifecycleListener): - """A PluginLifecycleListener decorator that dispatches to multiple delegates.""" + """ + A PluginLifecycleListener decorator that dispatches to multiple delegates. This can be used to execute multiple + listeners into one and pass them as a single listener to a ``PluginManager``. + + TODO: might be a candidate for removal, since a ``PluginManager`` can be configured with multiple listeners. + """ listeners: list[PluginLifecycleListener] @@ -202,7 +336,10 @@ def on_load_exception(self, *args, **kwargs): class FunctionPlugin(Plugin): """ - Exposes a function as a Plugin. + This class can be used to create plugins that are not derived from the Plugin class, but are simply functions. + Plux provides a built-in mechanism for this with the ``@plugin`` decorator. The ``FunctionPlugin`` is primarily an + internal API, but the class can be extended to customize the behavior of the function plugins, though this is a + very advanced use case. """ fn: t.Callable @@ -242,13 +379,28 @@ def plugin( load: t.Callable = None, ): """ - Expose a function as discoverable and loadable FunctionPlugin. + Expose a function as discoverable and loadable ``Plugin``. + + Here's an example using the ``@plugin`` decorator:: + + from plux import plugin + + @plugin(namespace="localstack.configurators") + def configure_logging(runtime): + logging.basicConfig(level=runtime.config.loglevel) + + + @plugin(namespace="localstack.configurators") + def configure_somethingelse(runtime): + # do other stuff with the runtime object + pass + - :param namespace: the plugin namespace - :param name: the name of the plugin (by default the function name will be used) - :param should_load: optional either a boolean value or a callable returning a boolean - :param load: optional load function - :return: plugin decorator + :param namespace: The plugin namespace + :param name: The name of the plugin (by default, the function name will be used) + :param should_load: Optionally either a boolean value or a callable returning a boolean + :param load: Optional load function + :return: Plugin decorator """ def wrapper(fn): diff --git a/plux/runtime/cache.py b/plux/runtime/cache.py index 5ba8c72..64fe6d3 100644 --- a/plux/runtime/cache.py +++ b/plux/runtime/cache.py @@ -1,3 +1,5 @@ +"""Internal tool to optimize access to entry points that are available in the current sys path.""" + import errno import glob import hashlib @@ -20,9 +22,10 @@ def get_user_cache_dir() -> Path: """ - Returns the path of the user's cache dir (e.g., ~/.cache on Linux, or ~/Library/Caches on Mac). + Returns the path of the user's cache dir. This is ``~/.cache`` on Linux, ``~/Library/Caches`` on Mac, or + ``\\Users\\{UserName}\\AppData\\Local`` on Windows. - :return: a Path pointing to the platform-specific cache dir of the user + :return: A Path pointing to the platform-specific cache dir of the user. """ if "windows" == platform.system().lower(): @@ -39,6 +42,13 @@ def get_user_cache_dir() -> Path: def _get_mtime(path: str) -> float: + """ + Utility to get the mtime of a file or directory. Mtime is the time of the most recent content modification + expressed in seconds. Returns -1.0 if the file does not exist. + + :param path: The file path to check + :return: The mtime of the file or -1.0 if the file does not exist. + """ try: s = os.stat(path) return s.st_mtime @@ -49,15 +59,25 @@ def _get_mtime(path: str) -> float: def _float_to_bytes(f: float) -> bytes: + """ + Utility to convert a float into a byte object. + + :param f: The float value to convert. + :return: A byte object containing the value f according to the format "f". + """ return struct.Struct("f").pack(f) class EntryPointsCache(EntryPointsResolver): """ Special ``EntryPointsResolver`` that uses ``importlib.metadata`` and a file system cache. The basic and - hash key calculation idea is taken from stevedore, but entry point parsing/serializing is simplified - slightly. Instead of writing json files, that includes metadata, we instead write one giant - ``entry_points.txt`` compatible file. + hash key calculation idea is taken from stevedore (see + https://github.com/openstack/stevedore/blob/master/stevedore/_cache.py), but entry point parsing/serializing is + simplified slightly. Instead of writing json files that include metadata, we instead write one giant + ``entry_points.txt`` compatible ini file. + + The cache is stored a ``plux/`` directory in the user's cache dir (e.g., ``~/.cache/plux`` on Linux, or + ``~/Library/Caches/plux`` on Mac) You can get a singleton instance via ``EntryPointsCache.instance()`` that also keeps an in-memory cache for the current ``sys.path``. @@ -77,7 +97,10 @@ def __init__(self): def get_entry_points(self) -> dict[str, list[metadata.EntryPoint]]: """ - Returns a dictionary of entry points for the current ``sys.path``. + Returns a dictionary of entry points for the current ``sys.path``. The first time the method is invoked, + the entrypoint index is created and then cached in-memory for faster subsequent access. + + :return: The dictionary that maps entrypoint group names to a list of entry points. """ path = sys.path key = tuple(path) @@ -118,7 +141,7 @@ def _build_and_store_index(self, cache_dir: Path) -> dict[str, list[metadata.Ent def _calculate_hash_key(self, path: list[str]): """ - Calculates a hash of all modified times of all ``entry_point.txt`` files in the path. Basic idea + Calculates a hash of all modified times of all ``entry_point.txt`` files in the path. The basic idea was taken from ``stevedore._cache._hash_settings_for_path``. The main difference to it is that it also considers entry points files linked through ``entry_points_editable.txt``. @@ -129,8 +152,8 @@ def _calculate_hash_key(self, path: list[str]): * mtimes of all entry_point.txt files within the path * mtimes of all entry_point.txt files linked through entry_points_editable.txt files - :param path: the path (typically ``sys.path``) - :return: a sha256 hash that hashes the path's entry point state + :param path: The path (typically ``sys.path``) + :return: A sha256 hash that hashes the path's entry point state """ h = hashlib.sha256() @@ -156,7 +179,7 @@ def _iter_entry_point_files(self, entry: str): An iterator that returns all entry_point.txt file paths in the given path entry. It transparently resolves editable entry points links. - :param entry: a path entry like ``/home/user/myproject/.venv/lib/python3.11/site-packages`` + :param entry: A path entry like ``/home/user/myproject/.venv/lib/python3.11/site-packages`` """ yield from glob.iglob(os.path.join(entry, "*.dist-info", "entry_points.txt")) yield from glob.iglob(os.path.join(entry, "*.egg-info", "entry_points.txt")) @@ -170,6 +193,11 @@ def _iter_entry_point_files(self, entry: str): @staticmethod def instance() -> "EntryPointsCache": + """ + Returns the singleton instance of the cache. If the instance does not exist yet, it is created. + + :return: A global ``EntryPointsCache`` instance. + """ if EntryPointsCache._instance: return EntryPointsCache._instance diff --git a/plux/runtime/filter.py b/plux/runtime/filter.py index 2a83f8e..2d96ad9 100644 --- a/plux/runtime/filter.py +++ b/plux/runtime/filter.py @@ -13,7 +13,7 @@ def __call__(self, spec: PluginSpec) -> bool: """ Returns True if the plugin should be filtered (disabled), or False otherwise. - :param spec: the spec to check + :param spec: The spec to check :return: True if the plugin should be disabled """ ... diff --git a/plux/runtime/manager.py b/plux/runtime/manager.py index e2fbcf5..3b855ea 100644 --- a/plux/runtime/manager.py +++ b/plux/runtime/manager.py @@ -34,6 +34,7 @@ def _call_safe(func: t.Callable, args: tuple, exception_message: str): try: return func(*args) except PluginException: + # re-raise PluginExceptions, since they should be handled by the caller raise except Exception as e: if LOG.isEnabledFor(logging.DEBUG): @@ -44,7 +45,11 @@ def _call_safe(func: t.Callable, args: tuple, exception_message: str): class PluginLifecycleNotifierMixin: """ - Mixin that provides functions to dispatch calls to a PluginLifecycleListener in a safe way. + Mixin that provides functions to dispatch calls to a ``PluginLifecycleListener`` safely. Safely means that + exceptions that happen in lifecycle listeners are not re-raised but logged instead. Exception ``PluginException``, + which are re-raised to allow the listeners to raise exceptions to the ``PluginManager``. + + Note that this is an internal class used primarily to keep the ``PluginManager`` free from dispatching code. """ listeners: list[PluginLifecycleListener] @@ -119,7 +124,7 @@ class PluginContainer(t.Generic[P]): plugin_spec: PluginSpec plugin: P = None - load_value: t.Any = None + load_value: t.Any | None = None is_init: bool = False is_loaded: bool = False @@ -135,7 +140,7 @@ def distribution(self) -> Distribution: """ Uses metadata from importlib to resolve the distribution information for this plugin. - :return: the importlib.metadata.Distribution object + :return: The importlib.metadata.Distribution object """ return resolve_distribution_information(self.plugin_spec) @@ -143,14 +148,14 @@ def distribution(self) -> Distribution: class PluginManager(PluginLifecycleNotifierMixin, t.Generic[P]): """ Manages Plugins within a namespace discovered by a PluginFinder. The default mechanism is to resolve plugins from - entry points using a ImportlibPluginFinder. + entry points using an ``ImportlibPluginFinder``. - A Plugin that is managed by a PluginManager can be in three states: + A Plugin managed by a PluginManager can be in three states: * resolved: the entrypoint pointing to the PluginSpec was imported and the PluginSpec instance was created * init: the PluginFactory of the PluginSpec was successfully invoked * loaded: the load method of the Plugin was successfully invoked - Internally, the PluginManager uses PluginContainer instances to keep the state of Plugin instances. + Internally, the ``PluginManager`` uses ``PluginContainer`` instances to keep the state of Plugin instances. """ namespace: str @@ -170,16 +175,55 @@ def __init__( filters: list[PluginFilter] = None, ): """ - Create a new PluginManager. + Create a new ``PluginManager`` that can be used to load plugins. The simplest ``PluginManager`` only needs + the namespace of plugins to load:: + + manager = PluginManager("my_namespace") + my_plugin = manager.load("my_plugin") # <- Plugin instance + + + This manager will look up plugins by querying the available entry points in the distribution, and resolve + the plugin through the entrypoint named "my_plugin" in the group "my_namespace". + + If your plugins take load arguments, you can pass them to the manager constructor:: + + class HypercornServerPlugin(Plugin): + namespace = "servers" + name = "hypercorn" + + def load(self, host: str, port: int): + ... + + manager = PluginManager("servers", load_args=("localhost", 8080)) + server_plugin = manager.load("hypercorn") + + + You can react to Plugin lifecycle events by passing a ``PluginLifecycleListener`` to the constructor:: + + class MyListener(PluginLifecycleListener): + def on_load_before(self, plugin_spec: PluginSpec, plugin: Plugin, load_result: t.Any | None = None): + LOG.info("Plugin %s was loaded and returned %s", plugin, load_result) + + manager = PluginManager("servers", listener=MyListener()) + manager.load("...") # will log "Plugin ... was loaded and returned None" + + You can also pass a list of listeners to the constructor. You can read more about the different states in the + documentation of ``PluginLifecycleListener``. + + Suppose you have multiple plugins that are available in your path, but you want to exclude some. You can pass + custom filters:: - :param namespace: the namespace (entry point group) that will be managed - :param load_args: positional arguments passed to ``Plugin.load()`` - :param load_kwargs: keyword arguments passed to ``Plugin.load()`` - :param listener: plugin lifecycle listeners, can either be a single listener or a list - :param finder: the plugin finder to be used, by default it uses a ``MetadataPluginFinder` - :param filters: filters exclude specific plugins. when no filters are provided, a list is created - and ``global_plugin_filter`` is added to it. filters can later be modified via - ``plugin_manager.filters``. + manager = PluginManager("servers", filters=[lambda spec: spec.name == "hypercorn"]) + plugins = manager.load_all() # will load all server plugins except the one named "hypercorn" + + + :param namespace: The namespace (entry point group) that will be managed. + :param load_args: Positional arguments passed to ``Plugin.load()``. + :param load_kwargs: Keyword arguments passed to ``Plugin.load()``. + :param listener: Plugin lifecycle listeners. Can either be a single listener or a list. + :param finder: The plugin finder to be used, by default it uses a ``MetadataPluginFinder`. + :param filters: Filters exclude specific plugins. When no filters are provided, a list is created and + ``global_plugin_filter`` is added to it. Filters can later be modified via ``plugin_manager.filters``. """ self.namespace = namespace @@ -203,6 +247,25 @@ def __init__( self._init_mutex = threading.RLock() def add_listener(self, listener: PluginLifecycleListener): + """ + Adds a lifecycle listener to the plugin manager. The listener will be notified of plugin lifecycle events. + + This method allows you to add listeners after the PluginManager has been created, which is useful when + you want to dynamically add listeners based on certain conditions. + + Example:: + + manager = PluginManager("my_namespace") + + # Later in the code + class MyListener(PluginLifecycleListener): + def on_load_after(self, plugin_spec, plugin, load_result=None): + print(f"Plugin {plugin_spec.name} was loaded") + + manager.add_listener(MyListener()) + + :param listener: The lifecycle listener to add + """ self.listeners.append(listener) def load(self, name: str) -> P: @@ -210,7 +273,39 @@ def load(self, name: str) -> P: Loads the Plugin with the given name using the load args and kwargs set in the plugin manager constructor. If at any point in the lifecycle the plugin loading fails, the load method will raise the respective exception. - Load is idempotent, so once the plugin is loaded, load will return the same instance again. + This method performs several steps: + 1. Checks if the plugin exists in the namespace + 2. Checks if the plugin is disabled + 3. Initializes the plugin if it's not already initialized + 4. Calls the plugin's ``load()`` method if it's not already loaded + + Load is idempotent, so once the plugin is loaded, subsequent calls will return the same instance + without calling the plugin's ``load()`` method again. This is useful for plugins that perform + expensive operations during loading. + + Example:: + + manager = PluginManager("my_namespace") + + # Basic usage + plugin = manager.load("my_plugin") + + # With error handling + try: + plugin = manager.load("another_plugin") + except PluginDisabled as e: + print(f"Plugin is disabled: {e.reason}") + except ValueError as e: + print(f"Plugin doesn't exist: {e}") + except Exception as e: + print(f"Error loading plugin: {e}") + + :param name: The name of the plugin to load + :return: The loaded plugin instance + :raises ValueError: If no plugin with the given name exists in the namespace + :raises PluginDisabled: If the plugin is disabled (either by a filter or by its `should_load()` method) + :raises PluginException: If the plugin failed to load for any other reason + :raises Exception: Any exception that occurred during plugin initialization or loading """ container = self._require_plugin(name) @@ -238,8 +333,36 @@ def load(self, name: str) -> P: def load_all(self, propagate_exceptions=False) -> list[P]: """ - Attempts to load all plugins found in the namespace, and returns those that were loaded successfully. If - propagate_exception is set to True, then the method will re-raise any errors as soon as it encouters them. + Attempts to load all plugins found in the namespace and returns those that were loaded successfully. + + This method iterates through all plugins in the namespace and: + 1. Skips plugins that are already loaded (adding them to the result list) + 2. Attempts to load each plugin that isn't already loaded + 3. Handles exceptions based on the `propagate_exceptions` parameter + + By default, this method silently skips plugins that are disabled or encounter errors during loading, + logging the errors but continuing with the next plugin. If `propagate_exceptions` is set to True, + it will re-raise any exceptions as soon as they are encountered, which can be useful for debugging. + + This method is useful when you want to load all available plugins in a namespace without + having to handle exceptions for each one individually. + + Example:: + + manager = PluginManager("my_namespace") + + # Load all plugins, silently skipping any that fail + plugins = manager.load_all() + print(f"Successfully loaded {len(plugins)} plugins") + + # Load all plugins, but stop and raise an exception if any fail + try: + plugins = manager.load_all(propagate_exceptions=True) + except Exception as e: + print(f"Failed to load all plugins: {e}") + + :param propagate_exceptions: If True, re-raises any exceptions encountered during loading + :return: A list of successfully loaded plugin instances """ plugins = list() @@ -262,21 +385,156 @@ def load_all(self, propagate_exceptions=False) -> list[P]: return plugins def list_plugin_specs(self) -> list[PluginSpec]: + """ + Returns a list of all plugin specifications (PluginSpec objects) in the namespace. + + This method returns all plugin specs that were found by the plugin finder, regardless of + whether they have been loaded or not. It does not trigger any loading of plugins. + + Example:: + + manager = PluginManager("my_namespace") + specs = manager.list_plugin_specs() + for spec in specs: + print(f"Found plugin: {spec.name} from {spec.value}") + + :return: A list of PluginSpec objects + """ return [container.plugin_spec for container in self._plugins.values()] def list_names(self) -> list[str]: + """ + Returns a list of all plugin names in the namespace. + + This is a convenience method that extracts just the names from the plugin specs. + Like `list_plugin_specs()`, this method does not trigger any loading of plugins. + + Example:: + + manager = PluginManager("my_namespace") + names = manager.list_names() + print(f"Available plugins: {', '.join(names)}") + + # Check if a specific plugin is available + if "my_plugin" in names: + plugin = manager.load("my_plugin") + + :return: A list of plugin names (strings) + """ return [spec.name for spec in self.list_plugin_specs()] def list_containers(self) -> list[PluginContainer[P]]: + """ + Returns a list of all plugin containers in the namespace. + + Plugin containers are internal objects that hold the state of plugins, including whether they are + loaded, disabled, or have encountered errors. This method is useful for advanced use cases where + you need to inspect the detailed state of plugins. + + This method does not trigger any loading of plugins. + + Example:: + + manager = PluginManager("my_namespace") + containers = manager.list_containers() + + # Find all loaded plugins + loaded_plugins = [c.plugin for c in containers if c.is_loaded] + + # Find all plugins that had errors during initialization + error_plugins = [c.name for c in containers if c.init_error] + + :return: A list of PluginContainer objects + """ return list(self._plugins.values()) def get_container(self, name: str) -> PluginContainer[P]: + """ + Returns the plugin container for a specific plugin by name. + + This method provides access to the internal container object that holds the state of a plugin. + It's useful when you need to inspect the detailed state of a specific plugin, such as checking + if it has encountered errors during initialization or loading. + + This method does not trigger loading of the plugin, but it will raise a ValueError if the + plugin doesn't exist in the namespace. + + Example:: + + manager = PluginManager("my_namespace") + try: + container = manager.get_container("my_plugin") + + # Check if the plugin is loaded + if container.is_loaded: + print(f"Plugin is loaded: {container.plugin}") + + # Check if there were any errors + if container.init_error: + print(f"Plugin had initialization error: {container.init_error}") + if container.load_error: + print(f"Plugin had loading error: {container.load_error}") + except ValueError: + print("Plugin not found") + + :param name: The name of the plugin + :return: The PluginContainer for the specified plugin + :raises ValueError: If no plugin with the given name exists in the namespace + """ return self._require_plugin(name) def exists(self, name: str) -> bool: + """ + Checks if a plugin with the given name exists in the namespace. + + This method is useful for checking if a plugin is available before attempting to load it. + It does not trigger any loading of plugins but only returns True if the plugin was successfully resolved. + + Example:: + + manager = PluginManager("my_namespace") + + # Check if a plugin exists before trying to load it + if manager.exists("my_plugin"): + plugin = manager.load("my_plugin") + else: + print("Plugin 'my_plugin' is not available") + + # Alternative to using try/except with get_container or load + if not manager.exists("another_plugin"): + print("Plugin 'another_plugin' is not available") + + :param name: The name of the plugin to check + :return: True if the plugin exists, False otherwise + """ return name in self._plugins def is_loaded(self, name: str) -> bool: + """ + Checks if a plugin with the given name is loaded. + + A plugin is considered loaded when its ``load()`` method has been successfully called. This method + is useful for checking the state of a plugin without triggering its loading. + + Note that this method will raise a ValueError if the plugin doesn't exist in the namespace. + + Example:: + + manager = PluginManager("my_namespace") + + # Check if a plugin is already loaded + if manager.exists("my_plugin") and not manager.is_loaded("my_plugin"): + print("Plugin exists but is not loaded yet") + plugin = manager.load("my_plugin") + + # Later in the code, check again + if manager.is_loaded("my_plugin"): + print("Plugin is now loaded") + + :param name: The name of the plugin to check + :return: True if the plugin is loaded, False otherwise + :raises ValueError: If no plugin with the given name exists in the namespace + """ return self._require_plugin(name).is_loaded @property @@ -294,7 +552,24 @@ def _require_plugin(self, name: str) -> PluginContainer[P]: return self._plugins[name] - def _load_plugin(self, container: PluginContainer): + def _load_plugin(self, container: PluginContainer) -> None: + """ + Implements the core algorithm to load a plugin from a ``PluginSpec`` (contained in the ``PluginContainer``), + and stores all relevant results, such as the Plugin instance, load result, or any errors into the passed + ``PluginContainer``. + + The loading algorithm follows these steps: + 1. Checks if any plugin filters would disable the plugin + 2. If not already initialized, instantiates the plugin using the factory from PluginSpec + 3. Checks if the plugin should be loaded by calling its should_load() method + 4. If loading is allowed, calls the plugin's load() method with configured arguments + 5. Updates the container state with load results or any errors that occurred + + Each step is protected by locks and includes appropriate lifecycle event notifications. + + :param container: The ``PluginContainer`` holding the ``PluginSpec`` that defines the ``Plugin`` to load. The + ``PluginContainer`` instance will also be populated with the load result or any errors that occurred. + """ with container.lock: plugin_spec = container.plugin_spec @@ -354,6 +629,13 @@ def _load_plugin(self, container: PluginContainer): container.load_error = e def _plugin_from_spec(self, plugin_spec: PluginSpec) -> P: + """ + Uses the factory from the ``PluginSpec`` to create a new instance of the Plugin. The factory is invoked + without any arguments, and currently there is no way to customize the instantiation process. + + :param plugin_spec: the plugin spec to instantiate + :return: a new instance of the Plugin + """ factory = plugin_spec.factory # functional decorators can overwrite the spec factory (pointing to the decorator) with a custom factory @@ -364,9 +646,23 @@ def _plugin_from_spec(self, plugin_spec: PluginSpec) -> P: return factory() def _init_plugin_index(self) -> dict[str, PluginContainer]: + """ + Initializes the plugin index, which maps plugin names to plugin containers. This method will *resolve* plugins, + meaning it loads the entry point object reference, thereby importing all its code. + + :return: A mapping of plugin names to plugin containers. + """ return {plugin.name: plugin for plugin in self._import_plugins() if plugin} def _import_plugins(self) -> t.Iterable[PluginContainer]: + """ + Finds all ``PluginSpace`` instances in the namespace, creates a container for each spec, and yields them one + by one. The plugin finder will typically load the entry point which involves importing the module it lives in. + Note that loading the entry point does not mean loading the plugin (i.e., calling the ``Plugin.load`` method), + it just means importing the code that defines the plugin. + + :return: Iterable of ``PluginContainer`` instances``. + """ for spec in self.finder.find_plugins(): self._fire_on_resolve_after(spec) @@ -376,6 +672,12 @@ def _import_plugins(self) -> t.Iterable[PluginContainer]: yield self._create_container(spec) def _create_container(self, plugin_spec: PluginSpec) -> PluginContainer: + """ + Factory method to create a ``PluginContainer`` for the given ``PluginSpec``. + + :param plugin_spec: The ``PluginSpec`` to create a container for. + :return: A new ``PluginContainer`` with the basic information of the plugin spec. + """ container = PluginContainer() container.lock = threading.RLock() container.name = plugin_spec.name diff --git a/plux/runtime/metadata.py b/plux/runtime/metadata.py index 9fbae07..5664756 100644 --- a/plux/runtime/metadata.py +++ b/plux/runtime/metadata.py @@ -16,8 +16,12 @@ Distribution = metadata.Distribution -def metadata_packages_distributions(): - """Wrapper around ``importlib.metadata.packages_distributions``, which returns a mapping of top-level packages.""" +def metadata_packages_distributions() -> t.Mapping[str, list[str]]: + """ + Wrapper around ``importlib.metadata.packages_distributions``, which returns a mapping of top-level packages. + + :return: A mapping of top-level packages to their distributions. + """ return metadata.packages_distributions() @@ -33,7 +37,7 @@ def packages_distributions() -> t.Mapping[str, list[str]]: in the site-packages directory created by the editable install. Therefore, a distribution name may appear twice for the same package, which is unhelpful information, so we deduplicate it here. - :return: package to distribution mapping + :return: A mapping of packages to their distributions. """ distributions = dict(metadata_packages_distributions()) @@ -50,8 +54,8 @@ def resolve_distribution_information(plugin_spec: PluginSpec) -> Distribution | error for plugins that come from a namespace package (i.e., when a package is part of multiple distributions). - :param plugin_spec: the plugin spec to resolve - :return: the Distribution metadata if it exists + :param plugin_spec: The plugin spec to resolve + :return: The Distribution metadata if it exists """ package = inspect.getmodule(plugin_spec.factory).__name__ root_package = package.split(".")[0] @@ -76,10 +80,11 @@ def resolve_entry_points( distributions: t.Iterable[Distribution] = None, ) -> list[metadata.EntryPoint]: """ - Resolves all entry points using a combination of ``importlib.metadata``, and also follows entry points - links in ``entry_points_editable.txt`` created by plux while building editable wheels. + Resolves all entry points in the given distributions using a combination of ``importlib.metadata``, and also + follows entry points links in ``entry_points_editable.txt`` created by plux while building editable wheels. + If no distributions are given, all distributions from ``importlib.metadata.distributions()`` are used. - :return: the list of unique entry points + :return: The list of unique entry points """ entry_points = [] distributions = distributions or metadata.distributions() @@ -114,10 +119,11 @@ def build_entry_point_index( ) -> dict[str, list[metadata.EntryPoint]]: """ Organizes the given list of entry points into a dictionary that maps entry point groups to their - respective entry points, which resembles the data structure of an ``entry_points.txt``. + respective entry points, which resembles the data structure of an ``entry_points.txt``. Using, for example, + ``index["distutils.commands"]`` you then get a list of entry points in that group. - :param entry_points: - :return: + :param entry_points: A list of entry points. + :return: A mapping of groups to a list of entrypoints in their group. """ result = collections.defaultdict(list) names = collections.defaultdict(set) # book-keeping to check duplicates @@ -133,8 +139,8 @@ def parse_entry_points_text(text: str) -> list[metadata.EntryPoint]: """ Parses the content of an ``entry_points.txt`` into a list of entry point objects. - :param text: the string to parse - :return: a list of metadata EntryPoint objects + :param text: The string to parse + :return: A list of metadata EntryPoint objects """ return metadata.EntryPoints._from_text(text) @@ -151,8 +157,8 @@ def serialize_entry_points_text(index: dict[str, list[metadata.EntryPoint]]) -> bdist_egg = setuptools.command.bdist_egg:bdist_egg - :param index: the index to serialize - :return: the serialized string + :param index: The index to serialize + :return: The serialized string """ buffer = io.StringIO() diff --git a/plux/runtime/resolve.py b/plux/runtime/resolve.py index f8e6611..0f070f7 100644 --- a/plux/runtime/resolve.py +++ b/plux/runtime/resolve.py @@ -15,7 +15,7 @@ class MetadataPluginFinder(PluginFinder): """ - This is a simple implementation of a PluginFinder that uses by default the ``EntryPointsCache`` singleton. + This is a simple implementation of a ``PluginFinder`` that uses by default the ``EntryPointsCache`` singleton. """ def __init__( @@ -40,9 +40,11 @@ def find_plugins(self) -> list[PluginSpec]: specs.append(spec) return specs - def to_plugin_spec(self, entry_point: EntryPoint) -> PluginSpec: + def to_plugin_spec(self, entry_point: EntryPoint) -> PluginSpec | None: """ - Convert a stevedore extension into a PluginSpec by using a spec_resolver. + Convert an importlib ``EntryPoint`` into a PluginSpec by using the ``spec_resolver``. It does this by first + loading the entry point, which returns the named object referenced by the entry point, and then passes that + to the ``PluginSpecResolver`` instance. """ try: source = entry_point.load() @@ -53,3 +55,5 @@ def to_plugin_spec(self, entry_point: EntryPoint) -> PluginSpec: if self.on_resolve_exception_callback: self.on_resolve_exception_callback(self.namespace, entry_point, e) + + return None