Introduce a plain foreman flavor

docs/developer/testing.md

@@ -146,6 +146,32 @@ def test_service_port(server):
     assert server.addr("localhost").port(6379).is_reachable
 ```
 
+### Feature guarding
+
+Some functionality can only be tested when a feature is enabled.
+
+You can mark an invidivual test to be skipped if needed:
+
+```python
+@pytest.mark.feature("iop")
+def test_ingress_service(server):
+    service = server.service("iop-core-ingress")
+    assert service.is_running and service.is_enabled
+```
+
+Often it's better to have an entire file dedicated to a feature and mark the entire file as guarded.
+
+```python
+pytestmark = pytest.mark.feature("iop")
+
+def test_ingress_service(server):
+    service = server.service("iop-core-ingress")
+    assert service.is_running and service.is_enabled
+
+def test_ingress_http_endpoint(server):
+    # ...
+```
+
 ### API test
 
 The `foremanapi` fixture is an [apypie](https://github.com/Apipie/apypie) `ForemanApi` client that connects to the deployed Foreman instance(authenticated as `admin`/`changeme`). It maps directly to the Foreman REST API — each method takes a resource name that corresponds to an API endpoint:

src/playbooks/deploy/deploy.yaml

@@ -13,19 +13,44 @@
     - "../../vars/foreman.yml"
     - "../../vars/base.yaml"
   pre_tasks:
-    - name: Add iop databases
+    - name: Determine databases
       when:
-        - "'iop' in enabled_features"
         - database_mode == 'internal'
       block:
-        - name: Include iop databases
-          ansible.builtin.include_vars:
-            file: "../../vars/database_iop.yml"
+        - name: Configure Candlepin database
+          ansible.builtin.set_fact:
+            postgresql_databases: "{{ postgresql_databases + postgresql_databases_candlepin }}"
+            postgresql_users: "{{ postgresql_users + postgresql_users_candlepin }}"
+          when:
+            - "'foreman' in enabled_features"
+            - "'katello' in enabled_features"
+
+        - name: Configure Foreman database
+          ansible.builtin.set_fact:
+            postgresql_databases: "{{ postgresql_databases + postgresql_databases_foreman }}"
+            postgresql_users: "{{ postgresql_users + postgresql_users_foreman }}"
+          when:
+            - "'foreman' in enabled_features"
 
-        - name: Combine lists
+        - name: Configure Pulp database
           ansible.builtin.set_fact:
-            postgresql_databases: "{{ postgresql_databases + iop_postgresql_databases }}"
-            postgresql_users: "{{ postgresql_users + iop_postgresql_users }}"
+            postgresql_databases: "{{ postgresql_databases + postgresql_databases_pulp }}"
+            postgresql_users: "{{ postgresql_users + postgresql_users_pulp }}"
+          when:
+            - "'katello' in enabled_features"
+
+        - name: Add iop databases
+          when:
+            - "'iop' in enabled_features"
+          block:
+            - name: Include iop databases
+              ansible.builtin.include_vars:
+                file: "../../vars/database_iop.yml"
+
+            - name: Combine lists
+              ansible.builtin.set_fact:
+                postgresql_databases: "{{ postgresql_databases + iop_postgresql_databases }}"
+                postgresql_users: "{{ postgresql_users + iop_postgresql_users }}"
   roles:
     - role: pre_install
     - role: checks
@@ -40,9 +65,14 @@
       when:
         - database_mode == 'internal'
     - redis
-    - candlepin
+    - role: candlepin
+      when:
+        - "'foreman' in enabled_features"
+        - "'katello' in enabled_features"
     - httpd
-    - pulp
+    - role: pulp
+      when:
+        - "'katello' in enabled_features"
     - foreman
     - role: systemd_target
     - role: iop_core

src/roles/foreman/tasks/main.yaml

@@ -313,3 +313,5 @@
     oauth1_consumer_key: "{{ foreman_oauth_consumer_key }}"
     oauth1_consumer_secret: "{{ foreman_oauth_consumer_secret }}"
     ca_path: "{{ foreman_ca_certificate }}"
+  when:
+    - "'katello' in enabled_features"

src/vars/database.yml

@@ -29,17 +29,29 @@ foreman_database_port: "{{ database_port }}"
 foreman_database_ssl_mode: "{{ database_ssl_mode }}"
 foreman_database_ssl_ca: "{{ database_ssl_ca }}"
 
-postgresql_databases:
+postgresql_databases_candlepin:
   - name: "{{ candlepin_database_name }}"
     owner: "{{ candlepin_database_user }}"
+
+postgresql_databases_foreman:
   - name: "{{ foreman_database_name }}"
     owner: "{{ foreman_database_user }}"
+
+postgresql_databases_pulp:
   - name: "{{ pulp_database_name }}"
     owner: "{{ pulp_database_user }}"
-postgresql_users:
+
+postgresql_users_candlepin:
   - name: "{{ candlepin_database_user }}"
     password: "{{ candlepin_database_password }}"
+
+postgresql_users_foreman:
   - name: "{{ foreman_database_user }}"
     password: "{{ foreman_database_password }}"
+
+postgresql_users_pulp:
   - name: "{{ pulp_database_user }}"
     password: "{{ pulp_database_password }}"
+
+postgresql_databases: []
+postgresql_users: []

src/vars/flavors/foreman.yml

@@ -0,0 +1,3 @@
+---
+flavor_features:
+  - foreman

tests/conftest.py

@@ -1,5 +1,7 @@
 import os
 import uuid
+from functools import cached_property
+from pathlib import Path
 
 import apypie
 import paramiko
@@ -16,6 +18,39 @@
 SSH_CONFIG = './.tmp/ssh-config'
 
 
+class UserParameters:
+    def __init__(self, config):
+        self._config = config
+
+    @property
+    def _obsah_state(self) -> Path:
+        # mirror what foremanctl / obsah does
+        if state := os.environ.get('OBSAH_STATE'):
+            return Path(state)
+
+        if base := os.environ.get('OBSAH_BASE'):
+            root = Path(base)
+        else:
+            root = self._config.rootpath
+        return root / '.var' / 'lib' / 'foremanctl'
+
+    def _read_parameters(self):
+        params_file = self._obsah_state / 'parameters.yaml'
+        if params_file.exists():
+            with params_file.open('r') as f:
+                return yaml.safe_load(f)
+
+        return None
+
+    @cached_property
+    def features(self):
+        # TODO: read the flavor and extract the features from there too
+        params = self._read_parameters()
+        if params:
+            return set(params.get('features', []))
+        return set()
+
+
 def pytest_addoption(parser):
     parser.addoption("--certificate-source", action="store", default="default", choices=('default', 'installer', 'custom_server'), help="Certificate source used during deployment")
     parser.addoption("--database-mode", action="store", default="internal", choices=('internal', 'external'), help="Whether the database is internal or external")
@@ -114,6 +149,10 @@
     api._session.headers['Host'] = server_fqdn
     return api
 
+@pytest.fixture
+def features(pytestconfig):
+    return pytestconfig.user_parameters.features
+
 
 @pytest.fixture
 def organization(foremanapi):
@@ -206,35 +245,32 @@
     wait_for_tasks(foremanapi, 'label = Actions::Katello::Repository::MetadataGenerate')
 
 
-def enabled_features():
-    test_dir = os.path.dirname(os.path.abspath(__file__))
-    foremanctl_dir = os.path.dirname(test_dir)
-    params_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'parameters.yaml')
-    if os.path.exists(params_file):
-        with open(params_file, 'r') as f:
-            features = yaml.safe_load(f).get('features', [])
-            if isinstance(features, str):
-                features = features.split()
-            return features
-    return []
-
-
-def is_iop_enabled():
-    return 'iop' in enabled_features()
-
-
 def pytest_configure(config):
-    config.addinivalue_line("markers", "iop: tests requiring IOP to be enabled")
+    config.addinivalue_line("markers", "feature(name): mark a test as requiring a feature")
 
+    config.user_parameters = UserParameters(config)
 
-def pytest_collection_modifyitems(config, items):
-    if is_iop_enabled():
-        return
 
-    skip_iop = pytest.mark.skip(reason="IOP not enabled - skipping IOP tests ('iop' not in enabled_features)")
+def pytest_collection_modifyitems(config, items):
+    feature_dir = config.rootdir / 'tests' / 'feature'
     for item in items:
-        if "iop" in item.keywords:
-            item.add_marker(skip_iop)
+        try:
+            rel_path = Path(item.fspath).relative_to(feature_dir)
+        except ValueError:
+            # Not in the features directory
+            pass
+        else:
+            feature = rel_path.parts[0]
+            # TODO: test if it's a valid feature?
+            item.add_marker(pytest.mark.feature(feature))
+
+
+def pytest_runtest_setup(item):
+    feature_markers = set(mark.args[0] for mark in item.iter_markers(name="feature"))
+    if feature_markers:
+        missing = feature_markers - item.config.user_parameters.features
+        if missing:
+            pytest.skip("test requires feature(s) {!r}".format(missing))
 
 
 class ResolveAdapter(HTTPAdapter):

tests/foreman_proxy_test.py → tests/feature/foreman-proxy/base_test.py

@@ -2,15 +2,10 @@
 import json
 
 import pytest
-from conftest import enabled_features
 
 FOREMAN_PROXY_PORT = 8443
 
 
-def is_bmc_enabled():
-    return 'bmc' in enabled_features()
-
-
 def get_proxy_v2_features(server, certificates, server_fqdn):
     cmd = server.run(
         f"curl --cacert {certificates['server_ca_certificate']} "
@@ -22,14 +17,14 @@ def get_proxy_v2_features(server, certificates, server_fqdn):
     return json.loads(cmd.stdout)
 
 
-def test_foreman_proxy_features(server, certificates, server_fqdn):
+def test_foreman_proxy_features(server, certificates, server_fqdn, pytestconfig):
     cmd = server.run(f"curl --cacert {certificates['server_ca_certificate']} --silent https://{server_fqdn}:{FOREMAN_PROXY_PORT}/features")
     assert cmd.succeeded
     features = json.loads(cmd.stdout)
     assert "logs" in features
     assert "script" in features
     assert "dynflow" in features
-    if is_bmc_enabled():
+    if 'bmc' in pytestconfig.user_parameters.features:
         assert "bmc" in features
     else:
         assert "bmc" not in features
@@ -60,7 +55,7 @@ def test_foreman_proxy_client_auth_to_foreman(server, certificates, server_fqdn)
     assert cmd.stdout == '201'
 
 
-@pytest.mark.skipif("not is_bmc_enabled()")
+@pytest.mark.feature('bmc')
 def test_bmc_capabilities(server, certificates, server_fqdn):
     features = get_proxy_v2_features(server, certificates, server_fqdn)
     assert 'bmc' in features

tests/feature/foreman/api_test.py

@@ -0,0 +1,10 @@
+def test_foreman_organization(organization):
+    assert organization
+
+
+def test_foreman_initial_organization(foremanapi):
+    assert foremanapi.list('organizations', search='name="Foreman CI"')
+
+
+def test_foreman_initial_location(foremanapi):
+    assert foremanapi.list('locations', search='name="Internet"')

tests/foreman_test.py → tests/feature/foreman/base_test.py

@@ -47,6 +47,7 @@ def test_foreman_status_cache(foreman_status):
     assert foreman_status['results']['foreman']['cache']['servers'][0]['status'] == 'ok'
 
 
+@pytest.mark.feature('katello')
 @pytest.mark.parametrize("katello_service", ['candlepin', 'candlepin_auth', 'foreman_tasks', 'katello_events', 'pulp3', 'pulp3_content'])
 def test_katello_services_status(foreman_status, katello_service):
     assert foreman_status['results']['katello']['services'][katello_service]['status'] == 'ok'

tests/feature/foreman/compute_resources_test.py

@@ -0,0 +1,25 @@
+import pytest
+
+
+# TODO: Foreman really should have a dedicated API endpoint to expose this info
+@pytest.fixture
+def provider_description(foremanapi):
+    create = foremanapi.resource('compute_resources').action('create')
+    compute_resource_param = [param for param in create.params if param.name == 'compute_resource'][0]
+    provider = [param for param in compute_resource_param.params if param.name == 'provider'][0]
+    return provider.description
+
+
+@pytest.mark.parametrize("compute_resource", ['EC2', 'Libvirt', 'Openstack', 'Vmware'])
+def test_foreman_compute_resources_built_in(provider_description, compute_resource):
+    assert compute_resource in provider_description
+
+
+@pytest.mark.feature('azure-rm')
+def test_foreman_compute_resources_azure_rm(provider_description):
+    assert 'AzureRm' in provider_description
+
+
+@pytest.mark.feature('google')
+def test_foreman_compute_resources_google(provider_description):
+    assert 'GCE' in provider_description

tests/foreman_api_test.py → tests/feature/foreman/katello_api_test.py

@@ -1,5 +1,6 @@
-def test_foreman_organization(organization):
-    assert organization
+import pytest
+
+pytestmark = pytest.mark.feature('katello')
 
 
 def test_foreman_product(product):
@@ -49,11 +50,3 @@ def test_foreman_manifest(organization, foremanapi, fixture_dir):
         files = {'content': (str(manifest_path), manifest_file, 'application/zip')}
         params = {'organization_id': organization['id']}
         foremanapi.resource_action('subscriptions', 'upload', params, files=files)
-
-
-def test_foreman_initial_organization(foremanapi):
-    assert foremanapi.list('organizations', search='name="Foreman CI"')
-
-
-def test_foreman_initial_location(foremanapi):
-    assert foremanapi.list('locations', search='name="Internet"')

tests/feature/foreman/plugins_test.py

@@ -0,0 +1,16 @@
+import pytest
+
+
+@pytest.fixture
+def foreman_plugins(foremanapi):
+    return [plugin['name'] for plugin in foremanapi.list('plugins')]
+
+
+@pytest.mark.feature('azure-rm')
+def test_foreman_compute_resources_azure_rm(foreman_plugins):
+    assert 'foreman_azure_rm' in foreman_plugins
+
+
+@pytest.mark.feature('google')
+def test_foreman_compute_resources_google(foreman_plugins):
+    assert 'foreman_google' in foreman_plugins

tests/iop/test_advisor.py → tests/feature/iop/test_advisor.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_advisor_backend_api_service(server):
     service = server.service("iop-service-advisor-backend-api")
     assert service.is_running