From a1e80befdfdd17579f4085b2e78d62f39147b68f Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Thu, 14 May 2026 11:14:57 +0200 Subject: [PATCH] Expose user enabled features as pytest markers This allows guarding tests behind markers. --- docs/developer/testing.md | 26 +++++++++ tests/conftest.py | 68 +++++++++++++++--------- tests/foreman_proxy_test.py | 11 ++-- tests/iop/test_advisor.py | 2 +- tests/iop/test_advisor_frontend.py | 2 +- tests/iop/test_cvemap_downloader.py | 2 +- tests/iop/test_engine.py | 2 +- tests/iop/test_gateway.py | 2 +- tests/iop/test_ingress.py | 2 +- tests/iop/test_integration.py | 2 +- tests/iop/test_inventory.py | 2 +- tests/iop/test_inventory_frontend.py | 2 +- tests/iop/test_kafka.py | 2 +- tests/iop/test_puptoo.py | 2 +- tests/iop/test_remediation.py | 2 +- tests/iop/test_vmaas.py | 2 +- tests/iop/test_vulnerability.py | 2 +- tests/iop/test_vulnerability_frontend.py | 2 +- tests/iop/test_yuptoo.py | 2 +- 19 files changed, 87 insertions(+), 50 deletions(-) diff --git a/docs/developer/testing.md b/docs/developer/testing.md index fa6011d85..22c9122b8 100644 --- a/docs/developer/testing.md +++ b/docs/developer/testing.md @@ -146,6 +146,32 @@ def test_service_port(server): assert server.addr("localhost").port(6379).is_reachable ``` +### Feature guarding + +Some functionality can only be tested when a feature is enabled. + +You can mark an individual test to be skipped if needed: + +```python +@pytest.mark.feature("iop") +def test_ingress_service(server): + service = server.service("iop-core-ingress") + assert service.is_running and service.is_enabled +``` + +Often it's better to have an entire file dedicated to a feature and mark the entire file as guarded. + +```python +pytestmark = pytest.mark.feature("iop") + +def test_ingress_service(server): + service = server.service("iop-core-ingress") + assert service.is_running and service.is_enabled + +def test_ingress_http_endpoint(server): + # ... +``` + ### API test The `foremanapi` fixture is an [apypie](https://github.com/Apipie/apypie) `ForemanApi` client that connects to the deployed Foreman instance(authenticated as `admin`/`changeme`). It maps directly to the Foreman REST API — each method takes a resource name that corresponds to an API endpoint: diff --git a/tests/conftest.py b/tests/conftest.py index 4883e923a..c668db886 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ -import os +import subprocess import uuid +from functools import cached_property import apypie import paramiko @@ -16,11 +17,40 @@ SSH_CONFIG = './.tmp/ssh-config' +class UserParameters: + def __init__(self, config): + self._config = config + + @cached_property + def features(self): + # foremanctl outputs + # FEATURE STATE DESCRIPTION + # $feature enabled/available $description + output = subprocess.check_output(['./foremanctl', 'features'], cwd=self._config.rootdir, + universal_newlines=True) + lines = output.splitlines(keepends=False) + # feature, status, description + return [line.split(None, 2) for line in lines[1:]] + + @cached_property + def available_features(self): + return set(feature for feature, _status, _desc in self.features) + + @cached_property + def enabled_features(self): + return set(feature for feature, status, _desc in self.features if status == 'enabled') + + def pytest_addoption(parser): parser.addoption("--certificate-source", action="store", default="default", choices=('default', 'installer', 'custom_server'), help="Certificate source used during deployment") parser.addoption("--database-mode", action="store", default="internal", choices=('internal', 'external'), help="Whether the database is internal or external") +@pytest.fixture(scope="module") +def enabled_features(pytestconfig): + return pytestconfig.user_parameters.enabled_features + + @pytest.fixture(scope="module") def fixture_dir(): return py.path.local(__file__).realpath() / '..' / 'fixtures' @@ -206,35 +236,21 @@ def wait_for_metadata_generate(foremanapi): wait_for_tasks(foremanapi, 'label = Actions::Katello::Repository::MetadataGenerate') -def enabled_features(): - test_dir = os.path.dirname(os.path.abspath(__file__)) - foremanctl_dir = os.path.dirname(test_dir) - params_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'parameters.yaml') - if os.path.exists(params_file): - with open(params_file, 'r') as f: - features = yaml.safe_load(f).get('features', []) - if isinstance(features, str): - features = features.split() - return features - return [] - - -def is_iop_enabled(): - return 'iop' in enabled_features() - - def pytest_configure(config): - config.addinivalue_line("markers", "iop: tests requiring IOP to be enabled") + config.addinivalue_line("markers", "feature(name): mark a test as requiring a feature") + config.user_parameters = UserParameters(config) -def pytest_collection_modifyitems(config, items): - if is_iop_enabled(): - return - skip_iop = pytest.mark.skip(reason="IOP not enabled - skipping IOP tests ('iop' not in enabled_features)") - for item in items: - if "iop" in item.keywords: - item.add_marker(skip_iop) +def pytest_runtest_setup(item): + feature_markers = set(mark.args[0] for mark in item.iter_markers(name="feature")) + if feature_markers: + invalid_features = feature_markers - item.config.user_parameters.available_features + if invalid_features: + raise pytest.PytestConfigWarning(f"Invalid feature(s) {invalid_features!r} on {item}") + missing = feature_markers - item.config.user_parameters.enabled_features + if missing: + pytest.skip(f"test requires feature(s) {missing!r}") class ResolveAdapter(HTTPAdapter): diff --git a/tests/foreman_proxy_test.py b/tests/foreman_proxy_test.py index 847a723bb..a41c9189d 100644 --- a/tests/foreman_proxy_test.py +++ b/tests/foreman_proxy_test.py @@ -2,15 +2,10 @@ import json import pytest -from conftest import enabled_features FOREMAN_PROXY_PORT = 8443 -def is_bmc_enabled(): - return 'bmc' in enabled_features() - - def get_proxy_v2_features(server, certificates, server_fqdn): cmd = server.run( f"curl --cacert {certificates['server_ca_certificate']} " @@ -22,14 +17,14 @@ def get_proxy_v2_features(server, certificates, server_fqdn): return json.loads(cmd.stdout) -def test_foreman_proxy_features(server, certificates, server_fqdn): +def test_foreman_proxy_features(server, certificates, server_fqdn, enabled_features): cmd = server.run(f"curl --cacert {certificates['server_ca_certificate']} --silent https://{server_fqdn}:{FOREMAN_PROXY_PORT}/features") assert cmd.succeeded features = json.loads(cmd.stdout) assert "logs" in features assert "script" in features assert "dynflow" in features - if is_bmc_enabled(): + if 'bmc' in enabled_features: assert "bmc" in features else: assert "bmc" not in features @@ -60,7 +55,7 @@ def test_foreman_proxy_client_auth_to_foreman(server, certificates, server_fqdn) assert cmd.stdout == '201' -@pytest.mark.skipif("not is_bmc_enabled()") +@pytest.mark.feature('bmc') def test_bmc_capabilities(server, certificates, server_fqdn): features = get_proxy_v2_features(server, certificates, server_fqdn) assert 'bmc' in features diff --git a/tests/iop/test_advisor.py b/tests/iop/test_advisor.py index 970598fd4..06bfa776f 100644 --- a/tests/iop/test_advisor.py +++ b/tests/iop/test_advisor.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_advisor_backend_api_service(server): diff --git a/tests/iop/test_advisor_frontend.py b/tests/iop/test_advisor_frontend.py index a111b61f0..904b6b82b 100644 --- a/tests/iop/test_advisor_frontend.py +++ b/tests/iop/test_advisor_frontend.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_advisor_frontend_assets_directory(server): diff --git a/tests/iop/test_cvemap_downloader.py b/tests/iop/test_cvemap_downloader.py index b7eafc902..5f4fa12f5 100644 --- a/tests/iop/test_cvemap_downloader.py +++ b/tests/iop/test_cvemap_downloader.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_cvemap_download_script(server): diff --git a/tests/iop/test_engine.py b/tests/iop/test_engine.py index d2bf0aecb..c768ab4d2 100644 --- a/tests/iop/test_engine.py +++ b/tests/iop/test_engine.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_engine_service(server): diff --git a/tests/iop/test_gateway.py b/tests/iop/test_gateway.py index faa493851..c1ca06d82 100644 --- a/tests/iop/test_gateway.py +++ b/tests/iop/test_gateway.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_gateway_service(server): diff --git a/tests/iop/test_ingress.py b/tests/iop/test_ingress.py index 3965cbe93..84765c688 100644 --- a/tests/iop/test_ingress.py +++ b/tests/iop/test_ingress.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_ingress_service(server): diff --git a/tests/iop/test_integration.py b/tests/iop/test_integration.py index 8a8dacac2..0dd10ac6d 100644 --- a/tests/iop/test_integration.py +++ b/tests/iop/test_integration.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_iop_core_kafka_service(server): diff --git a/tests/iop/test_inventory.py b/tests/iop/test_inventory.py index d69f564c5..bf2d1260a 100644 --- a/tests/iop/test_inventory.py +++ b/tests/iop/test_inventory.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_inventory_migrate_service(server): diff --git a/tests/iop/test_inventory_frontend.py b/tests/iop/test_inventory_frontend.py index 03af2e91d..9c16fa34d 100644 --- a/tests/iop/test_inventory_frontend.py +++ b/tests/iop/test_inventory_frontend.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_inventory_frontend_assets_directory(server): diff --git a/tests/iop/test_kafka.py b/tests/iop/test_kafka.py index 18a859e90..ff94d930b 100644 --- a/tests/iop/test_kafka.py +++ b/tests/iop/test_kafka.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_kafka_service(server): diff --git a/tests/iop/test_puptoo.py b/tests/iop/test_puptoo.py index b5aedcd78..4dbc9958e 100644 --- a/tests/iop/test_puptoo.py +++ b/tests/iop/test_puptoo.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_puptoo_service(server): diff --git a/tests/iop/test_remediation.py b/tests/iop/test_remediation.py index f8b609618..f29a2afdc 100644 --- a/tests/iop/test_remediation.py +++ b/tests/iop/test_remediation.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_remediation_api_service(server): diff --git a/tests/iop/test_vmaas.py b/tests/iop/test_vmaas.py index c7078574d..e8751b265 100644 --- a/tests/iop/test_vmaas.py +++ b/tests/iop/test_vmaas.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_vmaas_reposcan_service(server): diff --git a/tests/iop/test_vulnerability.py b/tests/iop/test_vulnerability.py index 8380921fe..dffaa7e40 100644 --- a/tests/iop/test_vulnerability.py +++ b/tests/iop/test_vulnerability.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_vulnerability_manager_service(server): diff --git a/tests/iop/test_vulnerability_frontend.py b/tests/iop/test_vulnerability_frontend.py index 4b55af898..b5c7e910e 100644 --- a/tests/iop/test_vulnerability_frontend.py +++ b/tests/iop/test_vulnerability_frontend.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_vulnerability_frontend_assets_directory(server): diff --git a/tests/iop/test_yuptoo.py b/tests/iop/test_yuptoo.py index 85fcd411a..8af0ed85a 100644 --- a/tests/iop/test_yuptoo.py +++ b/tests/iop/test_yuptoo.py @@ -1,6 +1,6 @@ import pytest -pytestmark = pytest.mark.iop +pytestmark = pytest.mark.feature("iop") def test_yuptoo_service(server):