diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e4bc75019..3221c8999 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,7 +145,9 @@ jobs: --initial-admin-password=changeme \ --initial-organization "Foreman CI" \ --initial-location "Internet" \ - --tuning development + --tuning development \ + --content-import-path /custom/import \ + --content-export-path /custom/export - name: Deploy features run: | ./foremanctl deploy \ diff --git a/development/playbooks/_pulp/metadata.obsah.yaml b/development/playbooks/_pulp/metadata.obsah.yaml new file mode 100644 index 000000000..6b6a4a301 --- /dev/null +++ b/development/playbooks/_pulp/metadata.obsah.yaml @@ -0,0 +1,12 @@ +--- +variables: + pulp_import_paths: + help: Extra file path that Pulp can use for content imports. Argument may be used more than once. + action: append_unique + type: AbsolutePath + parameter: --content-import-path + pulp_export_paths: + help: Extra file path that Pulp can use for content exports. Argument may be used more than once. + action: append_unique + type: AbsolutePath + parameter: --content-export-path diff --git a/development/playbooks/deploy-dev/metadata.obsah.yaml b/development/playbooks/deploy-dev/metadata.obsah.yaml index 6ad314b3b..964d6bf5c 100644 --- a/development/playbooks/deploy-dev/metadata.obsah.yaml +++ b/development/playbooks/deploy-dev/metadata.obsah.yaml @@ -32,3 +32,4 @@ variables: include: - _flavor_features + - _pulp diff --git a/docs/user/parameters.md b/docs/user/parameters.md index 85d9c6619..9b9ccba3d 100644 --- a/docs/user/parameters.md +++ b/docs/user/parameters.md @@ -54,6 +54,8 @@ There are multiple use cases from the users perspective that dictate what parame | `--foreman-puma-workers` | Number of workers for Puma | `--foreman-foreman-service-puma-workers` | | `--pulp-worker-count` | Number of pulp workers | `--foreman-proxy-content-pulpcore-worker-count` | | `--tuning` | Sets the tuning profile | `--tuning` | +| `--content-import-path` | Extra file path that Pulp can use for content imports | `--foreman-proxy-content-pulpcore-additional-import-paths` | +| `--content-export-path` | Extra file path that Pulp can use for content exports | | | `--external-authentication={ipa,ipa_with_api}` | Enable configuration for external authentication via IPA for web UI (or webUI and API for `ipa_with_api`), expects the target machine to [be enrolled into FreeIPA/IDM](https://docs.theforeman.org/3.16/Configuring_User_Authentication/index-katello.html#enrolling-foreman-server-in-freeipa-domain) | `--foreman-ipa-authentication`
`--foreman-ipa-authentication-api` | | `--external-authentication-pam-service` | PAM service used for host-based access control in IPA | `--foreman-pam-service` | @@ -155,7 +157,6 @@ There are multiple use cases from the users perspective that dictate what parame | `--foreman-proxy-bmc` | | | | | `--foreman-proxy-bmc-default-provider` | | | | | `--foreman-proxy-content-enable-ostree` | | | | -| `--foreman-proxy-content-pulpcore-additional-import-paths` | | | | | `--foreman-proxy-http` | | | | | `--foreman-proxy-log` | | | | | `--foreman-proxy-log-level` | | | | diff --git a/src/playbooks/_pulp/metadata.obsah.yaml b/src/playbooks/_pulp/metadata.obsah.yaml new file mode 100644 index 000000000..6b6a4a301 --- /dev/null +++ b/src/playbooks/_pulp/metadata.obsah.yaml @@ -0,0 +1,12 @@ +--- +variables: + pulp_import_paths: + help: Extra file path that Pulp can use for content imports. Argument may be used more than once. + action: append_unique + type: AbsolutePath + parameter: --content-import-path + pulp_export_paths: + help: Extra file path that Pulp can use for content exports. Argument may be used more than once. + action: append_unique + type: AbsolutePath + parameter: --content-export-path diff --git a/src/playbooks/deploy/metadata.obsah.yaml b/src/playbooks/deploy/metadata.obsah.yaml index e572ff46d..fda5062d1 100644 --- a/src/playbooks/deploy/metadata.obsah.yaml +++ b/src/playbooks/deploy/metadata.obsah.yaml @@ -53,6 +53,7 @@ constraints: forbidden_if: - [certificates_source, installer, [certificates_custom_server_certificate, certificates_custom_server_key, certificates_custom_server_ca_certificate]] + include: - _certificate_source - _certificate_validity @@ -60,3 +61,4 @@ include: - _database_connection - _tuning - _flavor_features + - _pulp diff --git a/src/roles/pulp/README.md b/src/roles/pulp/README.md new file mode 100644 index 000000000..76f2f8cfc --- /dev/null +++ b/src/roles/pulp/README.md @@ -0,0 +1,32 @@ +Pulp Role +========= + +Deploys and manages [Pulp](https://pulpproject.org/) as Podman quadlets. + +Variables +--------- +- `pulp_container_image`: Container image to use for Pulp (default: `quay.io/foreman/pulp`) +- `pulp_container_tag`: Container image tag (default: `"3.73"`) +- `pulp_registry_auth_file`: Path to the registry authentication file (default: `/etc/foreman/registry-auth.json`) +- `pulp_worker_count`: Number of Pulp workers (default: min of 8 or CPU count) +- `pulp_content_origin`: URL for the Pulp content service (default: `http://{{ fqdn }}:24816`) +- `pulp_pulp_url`: URL for the Pulp API service (default: `http://{{ fqdn }}:24817`) +- `pulp_volumes`: Volume mounts for Pulp containers (default: `/var/lib/pulp:/var/lib/pulp`) +- `pulp_enable_analytics`: Enable Pulp analytics (default: `false`) +- `pulp_import_paths`: Paths Pulp can use for content imports (default: `[/var/lib/pulp/sync_imports, /var/lib/pulp/imports]`) +- `pulp_export_paths`: Paths Pulp can use for content exports (default: `[/var/lib/pulp/exports]`) +- `pulp_plugins`: Additional Pulp plugins to enable (default: `[pulp_container, pulp_rpm]`) +- `pulp_database_name`: Name of the Pulp database (default: `pulp`) +- `pulp_database_user`: Database user (default: `pulp`) +- `pulp_database_host`: Database host (default: `localhost`) +- `pulp_database_port`: Database port (default: `5432`) +- `pulp_database_password`: Database password (required, no default) +- `pulp_database_ssl_mode`: Database SSL mode (default: `disabled`) +- `pulp_database_ssl_ca`: Path to the database SSL CA certificate on the control node (default: empty) + +Usage Inside foremanctl +----------------------- +When used as part of `foremanctl`, the variables are setup as the following +- `pulp_worker_count`: `--pulp-worker-count` +- `pulp_import_paths`: `--content-import-path` (may be specified multiple times) +- `pulp_export_paths`: `--content-export-path` (may be specified multiple times) diff --git a/src/roles/pulp/defaults/main.yaml b/src/roles/pulp/defaults/main.yaml index a4b9fa44a..2d33986b7 100644 --- a/src/roles/pulp/defaults/main.yaml +++ b/src/roles/pulp/defaults/main.yaml @@ -10,8 +10,12 @@ pulp_worker_count: "{{ [8, ansible_facts['processor_nproc']] | min }}" pulp_content_service_worker_count: "{{ (2 * ([8, ansible_facts['processor_nproc']] | min)) + 1 }}" pulp_api_service_worker_count: "{{ ([4, ansible_facts['processor_nproc']] | min) + 1 }}" -pulp_volumes: - - /var/lib/pulp:/var/lib/pulp +pulp_volumes: >- + {{ + ['/var/lib/pulp:/var/lib/pulp'] + + (pulp_import_paths | map('regex_replace', '^(.+)$', '\1:\1') | list) + + (pulp_export_paths | map('regex_replace', '^(.+)$', '\1:\1') | list) + }} pulp_api_container_name: pulp-api pulp_content_container_name: pulp-content @@ -22,6 +26,11 @@ pulp_pulp_url: "http://{{ ansible_facts['fqdn'] }}:24817" pulp_enable_analytics: false +pulp_default_import_paths: ["/var/lib/pulp/sync_imports", "/var/lib/pulp/imports"] +pulp_default_export_paths: ["/var/lib/pulp/exports"] +pulp_import_paths: [] +pulp_export_paths: [] + pulp_default_plugins: - pulp_certguard - pulp_file @@ -66,6 +75,10 @@ pulp_settings_other_env: PULP_CONTENT_WORKERS: "{{ pulp_content_service_worker_count }}" PULP_TOKEN_AUTH_DISABLED: "true" PULP_FLATPAK_INDEX: "true" + PULP_ALLOWED_IMPORT_PATHS: >- + {{ pulp_default_import_paths + pulp_import_paths }} + PULP_ALLOWED_EXPORT_PATHS: >- + {{ pulp_default_export_paths + pulp_export_paths }} PULP_ALLOWED_CONTENT_CHECKSUMS: >- ['sha1', 'sha224', 'sha256', 'sha384', 'sha512'] diff --git a/src/roles/pulp/tasks/main.yaml b/src/roles/pulp/tasks/main.yaml index 66dcad042..a861db531 100644 --- a/src/roles/pulp/tasks/main.yaml +++ b/src/roles/pulp/tasks/main.yaml @@ -37,6 +37,20 @@ - assets - media +- name: Create import directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: "{{ pulp_default_import_paths + pulp_import_paths }}" + +- name: Create export directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: "{{ pulp_default_export_paths + pulp_export_paths }}" + - name: Create DB password secret containers.podman.podman_secret: state: present diff --git a/tests/pulp_test.py b/tests/pulp_test.py index 9829fb3b4..8e1f7fc46 100644 --- a/tests/pulp_test.py +++ b/tests/pulp_test.py @@ -1,12 +1,28 @@ import json +import os import pytest +import yaml PULP_HOST = 'localhost' PULP_API_PORT = 24817 PULP_CONTENT_PORT = 24816 +def load_pulp_paths_from_parameters(): + test_dir = os.path.dirname(os.path.abspath(__file__)) + foremanctl_dir = os.path.dirname(test_dir) + params_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'parameters.yaml') + + if os.path.exists(params_file): + with open(params_file, 'r') as f: + params = yaml.safe_load(f) + import_paths = params.get('pulp_import_paths', []) + export_paths = params.get('pulp_export_paths', []) + + return import_paths, export_paths + + @pytest.fixture(scope="module") def pulp_status_curl(server): return server.run(f"curl -k -s -w '%{{stderr}}%{{http_code}}' http://{PULP_HOST}:{PULP_API_PORT}/pulp/api/v3/status/") @@ -17,6 +33,11 @@ def pulp_status(pulp_status_curl): return json.loads(pulp_status_curl.stdout) +@pytest.fixture(scope="module") +def pulp_import_export_paths(): + return load_pulp_paths_from_parameters() + + def test_pulp_api_service(server): pulp_api = server.service("pulp-api") assert pulp_api.is_running @@ -85,3 +106,40 @@ def test_pulp_worker_target(server): def test_pulp_manager_check(server): result = server.run("podman exec -ti pulp-api pulpcore-manager check --deploy") assert result.succeeded + + +def test_pulp_import_export_settings(server, pulp_import_export_paths): + expected_import_paths, expected_export_paths = pulp_import_export_paths + py = 'from django.conf import settings; import json; print(json.dumps({"import": list(settings.ALLOWED_IMPORT_PATHS), "export": list(settings.ALLOWED_EXPORT_PATHS)}))' + result = server.run(f"podman exec pulp-api pulpcore-manager shell -c '{py}'") + assert result.succeeded + data = json.loads(result.stdout) + for path in expected_import_paths: + assert path in data['import'], f"expected {path} in Pulp ALLOWED_IMPORT_PATHS" + for path in expected_export_paths: + assert path in data['export'], f"expected {path} in Pulp ALLOWED_EXPORT_PATHS" + + +def test_pulp_import_directories(server, pulp_import_export_paths): + import_paths, _ = pulp_import_export_paths + for path in import_paths: + assert server.file(path).is_directory + + +def test_pulp_export_directories(server, pulp_import_export_paths): + _, export_paths = pulp_import_export_paths + for path in export_paths: + assert server.file(path).is_directory + + +@pytest.mark.parametrize("container", ["pulp-api", "pulp-content", "pulp-worker-1"]) +def test_pulp_import_export_volume_mounts(server, container, pulp_import_export_paths): + import_paths, export_paths = pulp_import_export_paths + result = server.run(f"podman inspect {container} --format '{{{{json .Mounts}}}}'") + assert result.succeeded + mounts = json.loads(result.stdout) + destinations = [mount['Destination'] for mount in mounts] + + for path in import_paths + export_paths: + mounted = path in destinations or any(path.startswith(d + '/') for d in destinations) + assert mounted, f"expected {path} to be mounted as a volume in {container}"