From f80a137099cec1461d4824b415675a46416611fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cle=CC=81ment=20Doumouro?= Date: Fri, 21 Mar 2025 10:21:17 +0100 Subject: [PATCH 1/2] fix: http service extension --- .github/workflows/tests-worker.yml | 5 ++- icij-common/icij_common/import_utils.py | 37 +++++++++++++++++-- icij-worker/docker-compose.yml | 1 + icij-worker/icij_worker/__init__.py | 3 ++ icij-worker/icij_worker/app.py | 19 +++++++++- icij-worker/tests/test-plugin/README.md | 0 icij-worker/tests/test-plugin/pyproject.toml | 14 +++++++ .../tests/test-plugin/test_plugin/__init__.py | 8 ++++ icij-worker/tests/test_app.py | 10 +++++ 9 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 icij-worker/tests/test-plugin/README.md create mode 100644 icij-worker/tests/test-plugin/pyproject.toml create mode 100644 icij-worker/tests/test-plugin/test_plugin/__init__.py diff --git a/.github/workflows/tests-worker.yml b/.github/workflows/tests-worker.yml index 2cfced18..2fed76ae 100644 --- a/.github/workflows/tests-worker.yml +++ b/.github/workflows/tests-worker.yml @@ -34,9 +34,12 @@ jobs: with: python-version: "3.10" - name: Run tests + # TODO: find a way to handle local dev for icij-common properly... run: | cd icij-worker - uv run --dev --all-extras --frozen pytest -vvv --cache-clear --show-capture=all -r A tests + uv sync --dev --all-extras --frozen + uv pip install ../icij-common tests/test-plugin + uv run --no-sync --frozen pytest -vvv --cache-clear --show-capture=all -r A tests services: neo4j: image: neo4j:4.4.17 diff --git a/icij-common/icij_common/import_utils.py b/icij-common/icij_common/import_utils.py index ce6420a3..cf580476 100644 --- a/icij-common/icij_common/import_utils.py +++ b/icij-common/icij_common/import_utils.py @@ -1,21 +1,50 @@ import importlib +import warnings + from typing import Any -class VariableNotFound(ImportError): - pass +class VariableNotFound(ImportError): ... # pylint: disable=multiple-statements def import_variable(name: str) -> Any: + if ":" in name: + return _import_variable(name) + return _legacy_import_variable(name) + + +def _import_variable(name: str) -> Any: + module, variable_name = name.split(":") + if not module: + raise VariableNotFound(f"{name} not found in available module") + try: + module = importlib.import_module(module) + except ModuleNotFoundError as e: + raise VariableNotFound(e.msg) from e + try: + variable = getattr(module, variable_name) + except AttributeError as e: + raise VariableNotFound(e) from e + return variable + + +def _legacy_import_variable(name: str) -> Any: + msg = ( + "importing using only dot will be soon deprecated," + " use the new path.to.module:variable syntax" + ) + warnings.warn(msg, DeprecationWarning) parts = name.split(".") submodule = ".".join(parts[:-1]) + if not submodule: + raise VariableNotFound(f"{name} not found in available module") variable_name = parts[-1] try: module = importlib.import_module(submodule) except ModuleNotFoundError as e: raise VariableNotFound(e.msg) from e try: - subclass = getattr(module, variable_name) + variable = getattr(module, variable_name) except AttributeError as e: raise VariableNotFound(e) from e - return subclass + return variable diff --git a/icij-worker/docker-compose.yml b/icij-worker/docker-compose.yml index 8b37470d..ad1038f4 100644 --- a/icij-worker/docker-compose.yml +++ b/icij-worker/docker-compose.yml @@ -55,6 +55,7 @@ services: PORT: 8000 # Uncomment this and set it to your app path #TASK_MANAGER__APP_PATH: path.to.app_module.app_variable + # Uncomment this and allow the service to reach the app code healthcheck: test: curl -f http://localhost:8000/health diff --git a/icij-worker/icij_worker/__init__.py b/icij-worker/icij_worker/__init__.py index 54f2556d..1e07eeaf 100644 --- a/icij-worker/icij_worker/__init__.py +++ b/icij-worker/icij_worker/__init__.py @@ -43,3 +43,6 @@ from .backend import WorkerBackend from .event_publisher import EventPublisher + +# APP hook mean to be overridden with plugins +APP_HOOK = AsyncApp(name="app_hook") diff --git a/icij-worker/icij_worker/app.py b/icij-worker/icij_worker/app.py index 9a843176..c944c68b 100644 --- a/icij-worker/icij_worker/app.py +++ b/icij-worker/icij_worker/app.py @@ -2,13 +2,14 @@ import logging from contextlib import asynccontextmanager from copy import deepcopy +from importlib.metadata import entry_points from inspect import iscoroutinefunction, signature from typing import Callable, final from pydantic import BaseModel, field_validator, ConfigDict from typing_extensions import Self -from icij_common.import_utils import import_variable +from icij_common.import_utils import VariableNotFound, import_variable from icij_common.pydantic_utils import ICIJSettings, icij_config from icij_worker.routing_strategy import RoutingStrategy from icij_worker.typing_ import Dependency @@ -194,7 +195,21 @@ def _validate_group(self, task: RegisteredTask): @classmethod def load(cls, app_path: str, config: AsyncAppConfig | None = None) -> Self: - app = deepcopy(import_variable(app_path)) + try: + app = import_variable(app_path) + except VariableNotFound as e: + app_plugins = entry_points(group="icij_worker.APP_HOOK") + for entry_point in app_plugins: + if entry_point.name == app_path: + app = entry_point.load() + break + else: + msg = ( + f"invalid app path {app_path}, not found in available modules" + f" nor in icij_worker plugins" + ) + raise ValueError(msg) from e + app = deepcopy(app) if config is not None: app.with_config(config) return app diff --git a/icij-worker/tests/test-plugin/README.md b/icij-worker/tests/test-plugin/README.md new file mode 100644 index 00000000..e69de29b diff --git a/icij-worker/tests/test-plugin/pyproject.toml b/icij-worker/tests/test-plugin/pyproject.toml new file mode 100644 index 00000000..aede7e25 --- /dev/null +++ b/icij-worker/tests/test-plugin/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "test-plugin" +version = "0.1.0" +description = "Test plugin" +authors = [ + { name = "Clément Doumouro", email = "cdoumouro@icij.org" }, + { name = "ICIJ", email = "engineering@icij.org" }, +] +readme = "README.md" +requires-python = "~=3.10" +dependencies = [] + +[project.entry-points."icij_worker.APP_HOOK"] +plugged_app = "test_plugin:app" diff --git a/icij-worker/tests/test-plugin/test_plugin/__init__.py b/icij-worker/tests/test-plugin/test_plugin/__init__.py new file mode 100644 index 00000000..1745b282 --- /dev/null +++ b/icij-worker/tests/test-plugin/test_plugin/__init__.py @@ -0,0 +1,8 @@ +from icij_worker import AsyncApp + +app = AsyncApp(name="plugged") + + +@app.task() +def plugged_hello_world() -> str: + return "Hello Plugged World!" diff --git a/icij-worker/tests/test_app.py b/icij-worker/tests/test_app.py index a73d02dc..f067b960 100644 --- a/icij-worker/tests/test_app.py +++ b/icij-worker/tests/test_app.py @@ -31,6 +31,16 @@ def i_m_b(): return app +def test_load_from_plugin(): + # Given + # This is the name in the plugged app registry, see tests/test-plugin/pyproject.toml + path = "plugged_app" + # When + app = AsyncApp.load(path) + assert isinstance(app, AsyncApp) + assert app.name == "plugged" + + @pytest.mark.parametrize( "group,expected_keys", [("", ["i_m_a", "i_m_b"]), ("a", ["i_m_a"]), ("b", ["i_m_b"])], From dfbb18a26402997ace8ecfa2a0be0af5c910c5eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cle=CC=81ment=20Doumouro?= Date: Fri, 21 Mar 2025 13:31:04 +0100 Subject: [PATCH 2/2] fix: http service extension --- .github/workflows/tests-worker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests-worker.yml b/.github/workflows/tests-worker.yml index 2fed76ae..6dcbf133 100644 --- a/.github/workflows/tests-worker.yml +++ b/.github/workflows/tests-worker.yml @@ -27,7 +27,6 @@ jobs: uses: astral-sh/setup-uv@v5 with: version: "0.6.7" - python-version: "3.10" enable-cache: true - name: Setup Python project uses: actions/setup-python@v5 @@ -38,7 +37,7 @@ jobs: run: | cd icij-worker uv sync --dev --all-extras --frozen - uv pip install ../icij-common tests/test-plugin + uv pip install -e ../icij-common tests/test-plugin uv run --no-sync --frozen pytest -vvv --cache-clear --show-capture=all -r A tests services: neo4j: