Skip tests for features not enabled and show pytest results

development/playbooks/test/test.yaml

@@ -47,3 +47,14 @@
       args:
         chdir: "{{ inventory_dir }}/../"
       changed_when: false
+      register: pytest_result
+      ignore_errors: true
+
+    - name: Display pytest results
+      ansible.builtin.debug:
+        msg: "{{ pytest_result.stdout }}"
+
+    - name: Fail if tests did not pass
+      ansible.builtin.fail:
+        msg: "Tests failed. See output above for details."
+      when: pytest_result.rc != 0

tests/client_test.py

@@ -1,3 +1,8 @@
+import pytest
+
+from conftest import has_feature
+
+
 def test_foreman_content_view(client_environment, activation_key, organization, foremanapi, client):
     client.run('dnf install -y subscription-manager')
     rcmd = foremanapi.create('registration_commands', {'organization_id': organization['id'], 'insecure': True, 'activation_keys': [activation_key['name']], 'force': True})
@@ -9,6 +14,8 @@ def test_foreman_content_view(client_environment, activation_key, organization,
     client.run('subscription-manager unregister')
     client.run('subscription-manager clean')
 
+@pytest.mark.skipif(not has_feature("remote-execution"), reason="remote-execution not enabled")
+@pytest.mark.skipif(not has_feature("foreman-proxy"), reason="foreman-proxy not enabled")
 def test_foreman_rex(client_environment, activation_key, organization, foremanapi, client, client_fqdn):
     client.run('dnf install -y subscription-manager')
     rcmd = foremanapi.create('registration_commands', {'organization_id': organization['id'], 'insecure': True, 'activation_keys': [activation_key['name']], 'force': True})

tests/conftest.py

@@ -83,11 +83,20 @@ def ssh_config(server_hostname):
 
 
 @pytest.fixture(scope="module")
-def foremanapi(ssh_config, server_fqdn):
+def foreman_admin_password():
+    test_dir = os.path.dirname(os.path.abspath(__file__))
+    foremanctl_dir = os.path.dirname(test_dir)
+    passwd_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'foreman-admin-init-passwd')
+    with open(passwd_file) as f:
+        return f.read().strip()
+
+
+@pytest.fixture(scope="module")
+def foremanapi(ssh_config, server_fqdn, foreman_admin_password):
     api = apypie.ForemanApi(
         uri=f'https://{ssh_config["hostname"]}',
         username='admin',
-        password='changeme',
+        password=foreman_admin_password,
         verify_ssl=False,
     )
     api._session.headers['Host'] = server_fqdn
@@ -174,31 +183,37 @@ def wait_for_metadata_generate(foremanapi):
     wait_for_tasks(foremanapi, 'label = Actions::Katello::Repository::MetadataGenerate')
 
 
-def is_iop_enabled():
+def get_enabled_features():
     test_dir = os.path.dirname(os.path.abspath(__file__))
     foremanctl_dir = os.path.dirname(test_dir)
     params_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'parameters.yaml')
+    defaults_file = os.path.join(foremanctl_dir, 'src', 'vars', 'defaults.yml')
 
+    params = {}
     if os.path.exists(params_file):
         with open(params_file, 'r') as f:
-            params = yaml.safe_load(f)
-            features = params.get('features', [])
-            if isinstance(features, str):
-                features = features.split()
-            return 'iop' in features
+            params = yaml.safe_load(f) or {}
+
+    defaults = {}
+    if os.path.exists(defaults_file):
+        with open(defaults_file, 'r') as f:
+            defaults = yaml.safe_load(f) or {}
 
-    return False
+    flavor = params.get('flavor', defaults.get('flavor', 'katello'))
+    flavor_file = os.path.join(foremanctl_dir, 'src', 'vars', 'flavors', f'{flavor}.yml')
 
+    flavor_features = []
+    if os.path.exists(flavor_file):
+        with open(flavor_file, 'r') as f:
+            flavor_data = yaml.safe_load(f) or {}
+            flavor_features = flavor_data.get('flavor_features', [])
 
-def pytest_configure(config):
-    config.addinivalue_line("markers", "iop: tests requiring IOP to be enabled")
+    features = params.get('features', [])
+    if isinstance(features, str):
+        features = features.split()
 
+    return set(flavor_features + features)
 
-def pytest_collection_modifyitems(config, items):
-    if is_iop_enabled():
-        return
 
-    skip_iop = pytest.mark.skip(reason="IOP not enabled - skipping IOP tests ('iop' not in enabled_features)")
-    for item in items:
-        if "iop" in item.keywords:
-            item.add_marker(skip_iop)
+def has_feature(name):
+    return name in get_enabled_features()

tests/foreman_compute_resources_test.py

@@ -1,9 +1,23 @@
 import pytest
 
+from conftest import has_feature
+
+pytestmark = pytest.mark.skipif(
+    not has_feature("hammer"),
+    reason="hammer not enabled",
+)
+
 FOREMAN_HOST = 'localhost'
 FOREMAN_PORT = 3000
 
-@pytest.mark.parametrize("compute_resource", ['AzureRm', 'EC2', 'GCE', 'Libvirt', 'Openstack', 'Vmware'])
+@pytest.mark.parametrize("compute_resource", [
+    pytest.param('AzureRm', marks=pytest.mark.skipif(not has_feature('azure-rm'), reason="azure-rm not enabled")),
+    pytest.param('EC2'),
+    pytest.param('GCE', marks=pytest.mark.skipif(not has_feature('google'), reason="google not enabled")),
+    pytest.param('Libvirt'),
+    pytest.param('Openstack'),
+    pytest.param('Vmware'),
+])
 def test_foreman_compute_resources(server, compute_resource):
     hammer = server.run("hammer compute-resource create --help | grep provider")
     assert hammer.succeeded

tests/foreman_plugins_test.py

@@ -1,6 +1,11 @@
 import pytest
 
-@pytest.mark.parametrize("foreman_plugin", ['foreman_azure_rm', 'foreman_google'])
+from conftest import has_feature
+
+@pytest.mark.parametrize("foreman_plugin", [
+    pytest.param('foreman_azure_rm', marks=pytest.mark.skipif(not has_feature('azure-rm'), reason="azure-rm not enabled")),
+    pytest.param('foreman_google', marks=pytest.mark.skipif(not has_feature('google'), reason="google not enabled")),
+])
 def test_foreman_compute_resources(foremanapi, foreman_plugin):
     plugins = [plugin['name'] for plugin in foremanapi.list('plugins')]
     assert foreman_plugin in plugins

tests/foreman_proxy_test.py

@@ -1,5 +1,14 @@
 import json
 
+import pytest
+
+from conftest import has_feature
+
+pytestmark = pytest.mark.skipif(
+    not has_feature("foreman-proxy"),
+    reason="foreman-proxy not enabled",
+)
+
 FOREMAN_PROXY_PORT = 8443
 
 def test_foreman_proxy_features(server, certificates, server_fqdn):

tests/hammer_test.py

@@ -1,3 +1,13 @@
+import pytest
+
+from conftest import has_feature
+
+pytestmark = pytest.mark.skipif(
+    not has_feature("hammer"),
+    reason="hammer not enabled",
+)
+
+
 def test_hammer_ping(server):
     hammer = server.run("hammer ping")
     assert hammer.succeeded

tests/iop/conftest.py

@@ -0,0 +1,17 @@
+import pathlib
+
+import pytest
+
+from conftest import has_feature
+
+IOP_TEST_DIR = str(pathlib.Path(__file__).parent)
+
+
+def pytest_collection_modifyitems(config, items):
+    if has_feature("iop"):
+        return
+
+    skip_iop = pytest.mark.skip(reason="iop not enabled")
+    for item in items:
+        if str(item.fspath).startswith(IOP_TEST_DIR):
+            item.add_marker(skip_iop)

tests/iop/test_advisor.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_advisor_backend_api_service(server):
     service = server.service("iop-service-advisor-backend-api")
     assert service.is_running

tests/iop/test_advisor_frontend.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_advisor_frontend_assets_directory(server):
     assets_dir = server.file("/var/www/iop/assets/apps/advisor")
     assert assets_dir.exists

tests/iop/test_cvemap_downloader.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_cvemap_download_script(server):
     script = server.file("/usr/local/bin/iop-cvemap-download.sh")
     assert script.exists

tests/iop/test_engine.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_engine_service(server):
     service = server.service("iop-core-engine")
     assert service.is_running

tests/iop/test_gateway.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_gateway_service(server):
     service = server.service("iop-core-gateway")
     assert service.is_running

tests/iop/test_ingress.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_ingress_service(server):
     service = server.service("iop-core-ingress")
     assert service.is_running

tests/iop/test_integration.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_iop_core_kafka_service(server):
     service = server.service("iop-core-kafka")
     assert service.is_running

tests/iop/test_inventory.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_inventory_migrate_service(server):
     service = server.service("iop-core-host-inventory-migrate")
     assert service.is_enabled

tests/iop/test_inventory_frontend.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_inventory_frontend_assets_directory(server):
     assets_dir = server.file("/var/www/iop/assets/apps/inventory")
     assert assets_dir.exists

tests/iop/test_kafka.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_kafka_service(server):
     service = server.service("iop-core-kafka")
     assert service.is_running

tests/iop/test_puptoo.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_puptoo_service(server):
     service = server.service("iop-core-puptoo")
     assert service.is_running

tests/iop/test_remediation.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_remediation_api_service(server):
     service = server.service("iop-service-remediations-api")
     assert service.is_running

tests/iop/test_vmaas.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_vmaas_reposcan_service(server):
     service = server.service("iop-service-vmaas-reposcan")
     assert service.is_running

tests/iop/test_vulnerability.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_vulnerability_manager_service(server):
     service = server.service("iop-service-vuln-manager")
     assert service.is_running

tests/iop/test_vulnerability_frontend.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_vulnerability_frontend_assets_directory(server):
     assets_dir = server.file("/var/www/iop/assets/apps/vulnerability")
     assert assets_dir.exists

tests/iop/test_yuptoo.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_yuptoo_service(server):
     service = server.service("iop-core-yuptoo")
     assert service.is_running