Pulp import export support

.github/workflows/test.yml

@@ -145,7 +145,9 @@ jobs:
             --initial-admin-password=changeme \
             --initial-organization "Foreman CI" \
             --initial-location "Internet" \
-            --tuning development
+            --tuning development \
+            --content-import-path /custom/import \
+            --content-export-path /custom/export
       - name: Deploy features
         run: |
           ./foremanctl deploy \

development/playbooks/_pulp/metadata.obsah.yaml

@@ -0,0 +1,12 @@
+---
+variables:
+  pulp_import_paths:
+    help: Extra file path that Pulp can use for content imports. Argument may be used more than once.
+    action: append_unique
+    type: AbsolutePath
+    parameter: --content-import-path
+  pulp_export_paths:
+    help: Extra file path that Pulp can use for content exports. Argument may be used more than once.
+    action: append_unique
+    type: AbsolutePath
+    parameter: --content-export-path

development/playbooks/deploy-dev/metadata.obsah.yaml

@@ -32,3 +32,4 @@ variables:
 
 include:
   - _flavor_features
+  - _pulp

docs/user/parameters.md

@@ -54,6 +54,8 @@ There are multiple use cases from the users perspective that dictate what parame
 | `--foreman-puma-workers` | Number of workers for Puma | `--foreman-foreman-service-puma-workers` |
 | `--pulp-worker-count` | Number of pulp workers | `--foreman-proxy-content-pulpcore-worker-count` |
 | `--tuning` | Sets the tuning profile | `--tuning` |
+| `--content-import-path` | Extra file path that Pulp can use for content imports | `--foreman-proxy-content-pulpcore-additional-import-paths` |
+| `--content-export-path` | Extra file path that Pulp can use for content exports | |
 | `--external-authentication={ipa,ipa_with_api}` | Enable configuration for external authentication via IPA for web UI (or webUI and API for `ipa_with_api`), expects the target machine to [be enrolled into FreeIPA/IDM](https://docs.theforeman.org/3.16/Configuring_User_Authentication/index-katello.html#enrolling-foreman-server-in-freeipa-domain) | `--foreman-ipa-authentication`<br/> `--foreman-ipa-authentication-api` |
 | `--external-authentication-pam-service` | PAM service used for host-based access control in IPA | `--foreman-pam-service` |
 
@@ -155,7 +157,6 @@ There are multiple use cases from the users perspective that dictate what parame
 | `--foreman-proxy-bmc` | | | |
 | `--foreman-proxy-bmc-default-provider` | | | |
 | `--foreman-proxy-content-enable-ostree` | | | |
-| `--foreman-proxy-content-pulpcore-additional-import-paths` | | | |
 | `--foreman-proxy-http` | | | |
 | `--foreman-proxy-log` | | | |
 | `--foreman-proxy-log-level` | | | |

src/playbooks/_pulp/metadata.obsah.yaml

@@ -0,0 +1,12 @@
+---
+variables:
+  pulp_import_paths:
+    help: Extra file path that Pulp can use for content imports. Argument may be used more than once.
+    action: append_unique
+    type: AbsolutePath
+    parameter: --content-import-path
+  pulp_export_paths:
+    help: Extra file path that Pulp can use for content exports. Argument may be used more than once.
+    action: append_unique
+    type: AbsolutePath
+    parameter: --content-export-path

src/playbooks/deploy/metadata.obsah.yaml

@@ -53,10 +53,12 @@ constraints:
   forbidden_if:
     - [certificates_source, installer, [certificates_custom_server_certificate, certificates_custom_server_key, certificates_custom_server_ca_certificate]]
 
+
 include:
   - _certificate_source
   - _certificate_validity
   - _database_mode
   - _database_connection
   - _tuning
   - _flavor_features
+  - _pulp

src/roles/pulp/README.md

@@ -0,0 +1,32 @@
+Pulp Role
+=========
+
+Deploys and manages [Pulp](https://pulpproject.org/) as Podman quadlets.
+
+Variables
+---------
+- `pulp_container_image`: Container image to use for Pulp (default: `quay.io/foreman/pulp`)
+- `pulp_container_tag`: Container image tag (default: `"3.73"`)
+- `pulp_registry_auth_file`: Path to the registry authentication file (default: `/etc/foreman/registry-auth.json`)
+- `pulp_worker_count`: Number of Pulp workers (default: min of 8 or CPU count)
+- `pulp_content_origin`: URL for the Pulp content service (default: `http://{{ fqdn }}:24816`)
+- `pulp_pulp_url`: URL for the Pulp API service (default: `http://{{ fqdn }}:24817`)
+- `pulp_volumes`: Volume mounts for Pulp containers (default: `/var/lib/pulp:/var/lib/pulp`)
+- `pulp_enable_analytics`: Enable Pulp analytics (default: `false`)
+- `pulp_import_paths`: Paths Pulp can use for content imports (default: `[/var/lib/pulp/sync_imports, /var/lib/pulp/imports]`)
+- `pulp_export_paths`: Paths Pulp can use for content exports (default: `[/var/lib/pulp/exports]`)
+- `pulp_plugins`: Additional Pulp plugins to enable (default: `[pulp_container, pulp_rpm]`)
+- `pulp_database_name`: Name of the Pulp database (default: `pulp`)
+- `pulp_database_user`: Database user (default: `pulp`)
+- `pulp_database_host`: Database host (default: `localhost`)
+- `pulp_database_port`: Database port (default: `5432`)
+- `pulp_database_password`: Database password (required, no default)
+- `pulp_database_ssl_mode`: Database SSL mode (default: `disabled`)
+- `pulp_database_ssl_ca`: Path to the database SSL CA certificate on the control node (default: empty)
+
+Usage Inside foremanctl
+-----------------------
+When used as part of `foremanctl`, the variables are setup as the following
+- `pulp_worker_count`: `--pulp-worker-count`
+- `pulp_import_paths`: `--content-import-path` (may be specified multiple times)
+- `pulp_export_paths`: `--content-export-path` (may be specified multiple times)

src/roles/pulp/defaults/main.yaml

@@ -10,8 +10,12 @@ pulp_worker_count: "{{ [8, ansible_facts['processor_nproc']] | min }}"
 pulp_content_service_worker_count: "{{ (2 * ([8, ansible_facts['processor_nproc']] | min)) + 1 }}"
 pulp_api_service_worker_count: "{{ ([4, ansible_facts['processor_nproc']] | min) + 1 }}"
 
-pulp_volumes:
-  - /var/lib/pulp:/var/lib/pulp
+pulp_volumes: >-
+  {{
+    ['/var/lib/pulp:/var/lib/pulp'] +
+    (pulp_import_paths | map('regex_replace', '^(.+)$', '\1:\1') | list) +
+    (pulp_export_paths | map('regex_replace', '^(.+)$', '\1:\1') | list)
+  }}
 
 pulp_api_container_name: pulp-api
 pulp_content_container_name: pulp-content
@@ -22,6 +26,11 @@ pulp_pulp_url: "http://{{ ansible_facts['fqdn'] }}:24817"
 
 pulp_enable_analytics: false
 
+pulp_default_import_paths: ["/var/lib/pulp/sync_imports", "/var/lib/pulp/imports"]
+pulp_default_export_paths: ["/var/lib/pulp/exports"]
+pulp_import_paths: []
+pulp_export_paths: []
+
 pulp_default_plugins:
   - pulp_certguard
   - pulp_file
@@ -66,6 +75,10 @@ pulp_settings_other_env:
   PULP_CONTENT_WORKERS: "{{ pulp_content_service_worker_count }}"
   PULP_TOKEN_AUTH_DISABLED: "true"
   PULP_FLATPAK_INDEX: "true"
+  PULP_ALLOWED_IMPORT_PATHS: >-
+    {{ pulp_default_import_paths + pulp_import_paths }}
+  PULP_ALLOWED_EXPORT_PATHS: >-
+    {{ pulp_default_export_paths + pulp_export_paths }}
   PULP_ALLOWED_CONTENT_CHECKSUMS: >-
     ['sha1', 'sha224', 'sha256', 'sha384', 'sha512']
 

src/roles/pulp/tasks/main.yaml

@@ -37,6 +37,20 @@
     - assets
     - media
 
+- name: Create import directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  loop: "{{ pulp_default_import_paths + pulp_import_paths }}"
+
+- name: Create export directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  loop: "{{ pulp_default_export_paths + pulp_export_paths }}"
+
 - name: Create DB password secret
   containers.podman.podman_secret:
     state: present

tests/pulp_test.py

@@ -1,12 +1,28 @@
 import json
+import os
 
 import pytest
+import yaml
 
 PULP_HOST = 'localhost'
 PULP_API_PORT = 24817
 PULP_CONTENT_PORT = 24816
 
 
+def load_pulp_paths_from_parameters():
+    test_dir = os.path.dirname(os.path.abspath(__file__))
+    foremanctl_dir = os.path.dirname(test_dir)
+    params_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'parameters.yaml')
+
+    if os.path.exists(params_file):
+        with open(params_file, 'r') as f:
+            params = yaml.safe_load(f)
+            import_paths = params.get('pulp_import_paths', [])
+            export_paths = params.get('pulp_export_paths', [])
+
+    return import_paths, export_paths
+
+
 @pytest.fixture(scope="module")
 def pulp_status_curl(server):
     return server.run(f"curl -k -s -w '%{{stderr}}%{{http_code}}' http://{PULP_HOST}:{PULP_API_PORT}/pulp/api/v3/status/")
@@ -17,6 +33,11 @@ def pulp_status(pulp_status_curl):
     return json.loads(pulp_status_curl.stdout)
 
 
+@pytest.fixture(scope="module")
+def pulp_import_export_paths():
+    return load_pulp_paths_from_parameters()
+
+
 def test_pulp_api_service(server):
     pulp_api = server.service("pulp-api")
     assert pulp_api.is_running
@@ -85,3 +106,40 @@ def test_pulp_worker_target(server):
 def test_pulp_manager_check(server):
     result = server.run("podman exec -ti pulp-api pulpcore-manager check --deploy")
     assert result.succeeded
+
+
+def test_pulp_import_export_settings(server, pulp_import_export_paths):
+    expected_import_paths, expected_export_paths = pulp_import_export_paths
+    py = 'from django.conf import settings; import json; print(json.dumps({"import": list(settings.ALLOWED_IMPORT_PATHS), "export": list(settings.ALLOWED_EXPORT_PATHS)}))'
+    result = server.run(f"podman exec pulp-api pulpcore-manager shell -c '{py}'")
+    assert result.succeeded
+    data = json.loads(result.stdout)
+    for path in expected_import_paths:
+        assert path in data['import'], f"expected {path} in Pulp ALLOWED_IMPORT_PATHS"
+    for path in expected_export_paths:
+        assert path in data['export'], f"expected {path} in Pulp ALLOWED_EXPORT_PATHS"
+
+
+def test_pulp_import_directories(server, pulp_import_export_paths):
+    import_paths, _ = pulp_import_export_paths
+    for path in import_paths:
+        assert server.file(path).is_directory
+
+
+def test_pulp_export_directories(server, pulp_import_export_paths):
+    _, export_paths = pulp_import_export_paths
+    for path in export_paths:
+        assert server.file(path).is_directory
+
+
+@pytest.mark.parametrize("container", ["pulp-api", "pulp-content", "pulp-worker-1"])
+def test_pulp_import_export_volume_mounts(server, container, pulp_import_export_paths):
+    import_paths, export_paths = pulp_import_export_paths
+    result = server.run(f"podman inspect {container} --format '{{{{json .Mounts}}}}'")
+    assert result.succeeded
+    mounts = json.loads(result.stdout)
+    destinations = [mount['Destination'] for mount in mounts]
+
+    for path in import_paths + export_paths:
+        mounted = path in destinations or any(path.startswith(d + '/') for d in destinations)
+        assert mounted, f"expected {path} to be mounted as a volume in {container}"