fix: http service extensibility

.github/workflows/tests-worker.yml

@@ -27,16 +27,18 @@ jobs:
         uses: astral-sh/setup-uv@v5
         with:
           version: "0.6.7"
-          python-version: "3.10"
           enable-cache: true
       - name: Setup Python project
         uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: Run tests
+        # TODO: find a way to handle local dev for icij-common properly...
         run: |
           cd icij-worker
-          uv run --dev --all-extras --frozen pytest -vvv --cache-clear --show-capture=all -r A tests
+          uv sync --dev --all-extras --frozen
+          uv pip install -e ../icij-common tests/test-plugin
+          uv run --no-sync --frozen pytest -vvv --cache-clear --show-capture=all -r A tests
     services:
       neo4j:
         image: neo4j:4.4.17

icij-common/icij_common/import_utils.py

@@ -1,21 +1,50 @@
 import importlib
+import warnings
+
 from typing import Any
 
 
-class VariableNotFound(ImportError):
-    pass
+class VariableNotFound(ImportError): ...  # pylint: disable=multiple-statements
 
 
 def import_variable(name: str) -> Any:
+    if ":" in name:
+        return _import_variable(name)
+    return _legacy_import_variable(name)
+
+
+def _import_variable(name: str) -> Any:
+    module, variable_name = name.split(":")
+    if not module:
+        raise VariableNotFound(f"{name} not found in available module")
+    try:
+        module = importlib.import_module(module)
+    except ModuleNotFoundError as e:
+        raise VariableNotFound(e.msg) from e
+    try:
+        variable = getattr(module, variable_name)
+    except AttributeError as e:
+        raise VariableNotFound(e) from e
+    return variable
+
+
+def _legacy_import_variable(name: str) -> Any:
+    msg = (
+        "importing using only dot will be soon deprecated,"
+        " use the new path.to.module:variable syntax"
+    )
+    warnings.warn(msg, DeprecationWarning)
     parts = name.split(".")
     submodule = ".".join(parts[:-1])
+    if not submodule:
+        raise VariableNotFound(f"{name} not found in available module")
     variable_name = parts[-1]
     try:
         module = importlib.import_module(submodule)
     except ModuleNotFoundError as e:
         raise VariableNotFound(e.msg) from e
     try:
-        subclass = getattr(module, variable_name)
+        variable = getattr(module, variable_name)
     except AttributeError as e:
         raise VariableNotFound(e) from e
-    return subclass
+    return variable

icij-worker/docker-compose.yml

@@ -55,6 +55,7 @@ services:
       PORT: 8000
       # Uncomment this and set it to your app path
       #TASK_MANAGER__APP_PATH: path.to.app_module.app_variable
+      # Uncomment this and allow the service to reach the app code
 
     healthcheck:
       test: curl -f http://localhost:8000/health

icij-worker/icij_worker/__init__.py

@@ -43,3 +43,6 @@
 
 from .backend import WorkerBackend
 from .event_publisher import EventPublisher
+
+# APP hook mean to be overridden with plugins
+APP_HOOK = AsyncApp(name="app_hook")

icij-worker/icij_worker/app.py

@@ -2,13 +2,14 @@
 import logging
 from contextlib import asynccontextmanager
 from copy import deepcopy
+from importlib.metadata import entry_points
 from inspect import iscoroutinefunction, signature
 from typing import Callable, final
 
 from pydantic import BaseModel, field_validator, ConfigDict
 from typing_extensions import Self
 
-from icij_common.import_utils import import_variable
+from icij_common.import_utils import VariableNotFound, import_variable
 from icij_common.pydantic_utils import ICIJSettings, icij_config
 from icij_worker.routing_strategy import RoutingStrategy
 from icij_worker.typing_ import Dependency
@@ -194,7 +195,21 @@ def _validate_group(self, task: RegisteredTask):
 
     @classmethod
     def load(cls, app_path: str, config: AsyncAppConfig | None = None) -> Self:
-        app = deepcopy(import_variable(app_path))
+        try:
+            app = import_variable(app_path)
+        except VariableNotFound as e:
+            app_plugins = entry_points(group="icij_worker.APP_HOOK")
+            for entry_point in app_plugins:
+                if entry_point.name == app_path:
+                    app = entry_point.load()
+                    break
+            else:
+                msg = (
+                    f"invalid app path {app_path}, not found in available modules"
+                    f" nor in icij_worker plugins"
+                )
+                raise ValueError(msg) from e
+        app = deepcopy(app)
         if config is not None:
             app.with_config(config)
         return app

icij-worker/tests/test-plugin/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "test-plugin"
+version = "0.1.0"
+description = "Test plugin"
+authors = [
+    { name = "Clément Doumouro", email = "cdoumouro@icij.org" },
+    { name = "ICIJ", email = "engineering@icij.org" },
+]
+readme = "README.md"
+requires-python = "~=3.10"
+dependencies = []
+
+[project.entry-points."icij_worker.APP_HOOK"]
+plugged_app = "test_plugin:app"

icij-worker/tests/test-plugin/test_plugin/__init__.py

@@ -0,0 +1,8 @@
+from icij_worker import AsyncApp
+
+app = AsyncApp(name="plugged")
+
+
+@app.task()
+def plugged_hello_world() -> str:
+    return "Hello Plugged World!"

icij-worker/tests/test_app.py

@@ -31,6 +31,16 @@ def i_m_b():
     return app
 
 
+def test_load_from_plugin():
+    # Given
+    # This is the name in the plugged app registry, see tests/test-plugin/pyproject.toml
+    path = "plugged_app"
+    # When
+    app = AsyncApp.load(path)
+    assert isinstance(app, AsyncApp)
+    assert app.name == "plugged"
+
+
 @pytest.mark.parametrize(
     "group,expected_keys",
     [("", ["i_m_a", "i_m_b"]), ("a", ["i_m_a"]), ("b", ["i_m_b"])],