fix: harden package name resolution + add pyp2spec dep tooling

.github/workflows/generate-package-list.yml

@@ -80,11 +80,31 @@ jobs:
         run: git annex init
       - name: Update Packages
         run: ./update_packages.sh  ${{ matrix.package_name }} ${{ matrix.new_version }}
+      - name: Generate dependency diff
+        id: dep_diff
+        run: |
+          import subprocess, os, sys
+          pkg = "${{ matrix.package_name }}"
+          version = "${{ matrix.new_version }}"
+          # package_name here is already the resolved dir suffix from find_package.py
+          pkg_dir = f"python-{pkg}"
+          spec_file = f"packages/{pkg_dir}/{pkg_dir}.spec"
+          diff = subprocess.run(
+              ["python3", "automation/dep_diff.py", spec_file, pkg, version],
+              capture_output=True, text=True
+          ).stdout
+          delimiter = "DEPDIFF_EOF"
+          with open(os.environ["GITHUB_OUTPUT"], "a") as gh:
+              gh.write(f"dep_diff<<{delimiter}\n{diff}\n{delimiter}\n")
+        shell: python
       - name: Open a PR
         uses: peter-evans/create-pull-request@v8
         with:
           commit-message: "Update ${{ matrix.package_name }} to ${{ matrix.new_version }}"
           branch: "bump_rpm/${{ matrix.package_name }}"
           title: "Update ${{ matrix.package_name }} to ${{ matrix.new_version }}"
-          body: ''
+          body: |
+            ## Dependency changes
+
+            ${{ steps.dep_diff.outputs.dep_diff }}
           delete-branch: true

.github/workflows/update-pulp-packages.yml

@@ -58,11 +58,38 @@ jobs:
       run: git annex init
     - name: Update Packages
       run: ./update_packages.sh  ${{ matrix.package_name }} ${{ matrix.new_version }}
+    - name: Generate dependency diff
+      id: dep_diff
+      run: |
+        import subprocess, os, sys
+        pkg = "${{ matrix.package_name }}"
+        version = "${{ matrix.new_version }}"
+        result = subprocess.run(
+            ["python3", "find_package.py", "--resolve-dir", pkg],
+            capture_output=True, text=True
+        )
+        if result.returncode != 0:
+            with open(os.environ["GITHUB_OUTPUT"], "a") as gh:
+                gh.write("dep_diff=_Could not resolve package directory._\n")
+            sys.exit(0)
+        pkg_dir = result.stdout.strip()
+        spec_file = f"packages/{pkg_dir}/{pkg_dir}.spec"
+        diff = subprocess.run(
+            ["python3", "automation/dep_diff.py", spec_file, pkg, version],
+            capture_output=True, text=True
+        ).stdout
+        delimiter = "DEPDIFF_EOF"
+        with open(os.environ["GITHUB_OUTPUT"], "a") as gh:
+            gh.write(f"dep_diff<<{delimiter}\n{diff}\n{delimiter}\n")
+      shell: python
     - name: Open a PR
       uses: peter-evans/create-pull-request@v8
       with:
         commit-message: "Update ${{ matrix.package_name }} to ${{ matrix.new_version }}"
         branch: "bump_rpm/${{ matrix.package_name }}"
         title: "Update ${{ matrix.package_name }} to ${{ matrix.new_version }}"
-        body: ''
+        body: |
+          ## Dependency changes
+
+          ${{ steps.dep_diff.outputs.dep_diff }}
         delete-branch: true

automation/dep_diff.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""Compare declared PyPI runtime deps against an existing spec file's Requires.
+
+Usage:
+    python3 automation/dep_diff.py <spec_file> <pypi_name> <version>
+
+Outputs a markdown table of added/removed/unchanged mandatory Requires entries.
+Uses the PyPI JSON API — no external tools required.
+"""
+
+import json
+import re
+import sys
+import urllib.request
+
+PYTHON_VER = "3.12"
+_SEP_RE = re.compile(r"[-_.]+")
+
+# RPM macro patterns to normalize before comparison
+_RPM_MACRO_RE = re.compile(r"%\{python3_pkgversion\}|%\{python3_abi\}")
+
+
+def canonicalize(name):
+    return _SEP_RE.sub("-", name).lower()
+
+
+def pypi_mandatory_deps(pypi_name, version):
+    """Return set of python3.12-* RPM names for mandatory (non-extra) runtime deps."""
+    import random, time
+    url = f"https://pypi.org/pypi/{pypi_name}/{version}/json"
+    # Jitter 0-20s so concurrent matrix jobs don't hit PyPI simultaneously
+    time.sleep(random.uniform(0, 20))
+    for attempt in range(3):
+        try:
+            with urllib.request.urlopen(url, timeout=15) as r:
+                data = json.load(r)
+            break
+        except Exception as e:
+            if attempt == 2:
+                print(f"WARNING: PyPI API error for {pypi_name}=={version}: {e}", file=sys.stderr)
+                return set()
+            time.sleep(2 ** attempt + random.uniform(0, 3))
+
+    requires_dist = data["info"].get("requires_dist") or []
+    rpm_names = set()
+    for dep in requires_dist:
+        # Skip optional extras
+        if "extra ==" in dep or "extra==" in dep:
+            continue
+        # Extract package name (before any version specifier or env marker)
+        name = re.split(r"[><=!;\s\(]", dep)[0].strip()
+        if not name:
+            continue
+        canonical = canonicalize(name)
+        if canonical.startswith("python-"):
+            canonical = canonical[len("python-"):]
+        rpm_names.add(f"python{PYTHON_VER}-{canonical}")
+    return rpm_names
+
+
+def parse_spec_requires(spec_file):
+    """Return set of python3.12-* Requires entries from spec (toolchain excluded)."""
+    requires = set()
+    with open(spec_file) as f:
+        for line in f:
+            stripped = line.strip()
+            if not stripped.startswith("Requires:"):
+                continue
+            rest = stripped[len("Requires:"):].strip()
+            # Expand RPM macros
+            rest = _RPM_MACRO_RE.sub(PYTHON_VER, rest)
+            for token in rest.split():
+                # Skip version comparators and non-python tokens
+                if re.match(r"^[><=!]", token):
+                    continue
+                if token.startswith(f"python{PYTHON_VER}-"):
+                    requires.add(token)
+    return requires
+
+
+def diff_table(spec_requires, pypi_requires):
+    added = sorted(pypi_requires - spec_requires)
+    removed = sorted(spec_requires - pypi_requires)
+    unchanged = sorted(spec_requires & pypi_requires)
+
+    if not added and not removed:
+        return "_No mandatory dependency changes detected._\n"
+
+    lines = [
+        "| Status | Package |",
+        "|--------|---------|",
+    ]
+    for pkg in added:
+        lines.append(f"| :green_circle: added | `{pkg}` |")
+    for pkg in removed:
+        lines.append(f"| :red_circle: removed | `{pkg}` |")
+    for pkg in unchanged:
+        lines.append(f"| unchanged | `{pkg}` |")
+    return "\n".join(lines) + "\n"
+
+
+def main():
+    if len(sys.argv) != 4:
+        print(f"Usage: {sys.argv[0]} <spec_file> <pypi_name> <version>", file=sys.stderr)
+        sys.exit(1)
+
+    spec_file, pypi_name, version = sys.argv[1], sys.argv[2], sys.argv[3]
+    spec_requires = parse_spec_requires(spec_file)
+    pypi_requires = pypi_mandatory_deps(pypi_name, version)
+    print(diff_table(spec_requires, pypi_requires))
+
+
+if __name__ == "__main__":
+    main()

automation/update_deps.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""Regenerate Requires in an existing spec file using PyPI dependency metadata.
+
+Usage:
+    python3 automation/update_deps.py <spec_file> <pypi_name> <version>
+
+Replaces mandatory (non-extra) Requires: python3.12-* entries in the spec
+with the new version's declared runtime dependencies from the PyPI JSON API.
+Toolchain BuildRequires (python3.12-devel, pyproject-rpm-macros, etc.) are untouched.
+"""
+
+import json
+import re
+import sys
+import urllib.request
+
+PYTHON_VER = "3.12"
+_SEP_RE = re.compile(r"[-_.]+")
+_RPM_MACRO_RE = re.compile(r"%\{python3_pkgversion\}|%\{python3_abi\}")
+
+# Token patterns that are build toolchain, not Python library deps
+_TOOLCHAIN_RE = re.compile(
+    rf"python{re.escape(PYTHON_VER)}-(devel|wheel|pip|setuptools)|"
+    r"pyproject-rpm-macros|gcc|make|cmake|perl|ruby"
+)
+
+
+def canonicalize(name):
+    return _SEP_RE.sub("-", name).lower()
+
+
+def pypi_mandatory_deps(pypi_name, version):
+    """Return sorted list of python3.12-* RPM names for mandatory runtime deps."""
+    import random, time
+    url = f"https://pypi.org/pypi/{pypi_name}/{version}/json"
+    # Jitter 0-20s so concurrent matrix jobs don't hit PyPI simultaneously
+    time.sleep(random.uniform(0, 20))
+    for attempt in range(3):
+        try:
+            with urllib.request.urlopen(url, timeout=15) as r:
+                data = json.load(r)
+            break
+        except Exception as e:
+            if attempt == 2:
+                print(f"WARNING: PyPI API error for {pypi_name}=={version}: {e}", file=sys.stderr)
+                return []
+            time.sleep(2 ** attempt + random.uniform(0, 3))
+
+    requires_dist = data["info"].get("requires_dist") or []
+    rpm_names = set()
+    for dep in requires_dist:
+        if "extra ==" in dep or "extra==" in dep:
+            continue
+        name = re.split(r"[><=!;\s\(]", dep)[0].strip()
+        if not name:
+            continue
+        canonical = canonicalize(name)
+        if canonical.startswith("python-"):
+            canonical = canonical[len("python-"):]
+        rpm_names.add(f"python{PYTHON_VER}-{canonical}")
+    return sorted(rpm_names)
+
+
+def is_library_name(name):
+    """Return True if name (after macro expansion) is a python3.12-* library dep."""
+    expanded = _RPM_MACRO_RE.sub(PYTHON_VER, name)
+    return (
+        expanded.startswith(f"python{PYTHON_VER}-")
+        and not _TOOLCHAIN_RE.search(expanded)
+    )
+
+
+def parse_requires_entries(rest):
+    """Parse an RPM Requires value into list of (name, version_constraint) tuples.
+
+    Handles: 'foo', 'foo >= 1.0', 'foo >= 1.0 bar baz >= 2.0'
+    """
+    entries = []
+    tokens = rest.split()
+    i = 0
+    while i < len(tokens):
+        name = tokens[i]
+        constraint = ""
+        # Consume optional version operator + value
+        if i + 1 < len(tokens) and re.match(r"^[><=!]", tokens[i + 1]):
+            op = tokens[i + 1]
+            i += 2
+            if i < len(tokens) and not re.match(r"^[><=!%a-z]", tokens[i]):
+                constraint = f"{op} {tokens[i]}"
+                i += 1
+            else:
+                constraint = op
+        else:
+            i += 1
+        entries.append((name, constraint))
+    return entries
+
+
+def rewrite_requires(spec_file, new_requires):
+    """Replace python3.12-* library Requires: lines in spec with new_requires."""
+    with open(spec_file) as f:
+        lines = f.readlines()
+
+    out = []
+    library_block_written = False
+
+    for line in lines:
+        stripped = line.rstrip()
+        if not stripped.startswith("Requires:"):
+            out.append(line)
+            continue
+
+        rest = stripped[len("Requires:"):].strip()
+        entries = parse_requires_entries(rest)
+        lib_entries = [(n, c) for n, c in entries if is_library_name(n)]
+        preserved_entries = [(n, c) for n, c in entries if not is_library_name(n)]
+
+        if not lib_entries:
+            # No library entries on this line — keep it unchanged
+            out.append(line)
+            continue
+
+        if library_block_written:
+            # Additional library-only Requires lines are consolidated above
+            continue
+
+        indent = "Requires:       "
+        # Write preserved non-library entries (with their version constraints)
+        for name, constraint in preserved_entries:
+            token = f"{name} {constraint}".strip()
+            out.append(f"{indent}{token}\n")
+        # Write the new library Requires
+        for req in new_requires:
+            out.append(f"{indent}{req}\n")
+        library_block_written = True
+
+    with open(spec_file, "w") as f:
+        f.writelines(out)
+
+
+def main():
+    if len(sys.argv) != 4:
+        print(f"Usage: {sys.argv[0]} <spec_file> <pypi_name> <version>", file=sys.stderr)
+        sys.exit(1)
+
+    spec_file, pypi_name, version = sys.argv[1], sys.argv[2], sys.argv[3]
+    new_requires = pypi_mandatory_deps(pypi_name, version)
+
+    if not new_requires:
+        print(f"INFO: no mandatory runtime deps for {pypi_name}=={version}; "
+              "removing existing library Requires entries.")
+
+    rewrite_requires(spec_file, new_requires)
+
+    if new_requires:
+        print(f"Updated Requires in {spec_file}: {', '.join(new_requires)}")
+    else:
+        print(f"Cleared library Requires in {spec_file} (no mandatory deps declared).")
+
+
+if __name__ == "__main__":
+    main()

build_matrix.py

@@ -4,12 +4,23 @@
 import os
 import sys
 
-from find_package import parse_package_list
+from find_package import canonicalize, parse_package_list
+
+# This script intentionally does NOT route through resolve_package_dir().
+# update-pulp-packages.yml is scoped only to packages in automation/requirements.txt
+# (pulpcore + official pulp plugins) — it must not trigger broad package discovery.
+# update_packages.sh calls find_package.py --resolve-dir at runtime to resolve each
+# raw PyPI name to its on-disk directory suffix, handling separator/prefix mismatches.
 
 
 def main():
     packages = list(parse_package_list(sys.stdin.readlines()))
 
+    # Normalize package names so separator mismatches (ruamel.yaml → ruamel-yaml) don't
+    # produce malformed branch names or spec paths downstream.
+    for p in packages:
+        p["package_name"] = canonicalize(p["package_name"])
+
     if 'GITHUB_OUTPUT' in os.environ:
         with open(os.environ['GITHUB_OUTPUT'], 'a') as github_output:
             print(f'matrix={json.dumps(packages)}', file=github_output)