Skip to content

Resolve package spec directories dynamically instead of hardcoded maps #2987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Odilhao merged 3 commits into from
Aug 27, 2026
Merged

Resolve package spec directories dynamically instead of hardcoded maps #2987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
954ab71
Resolve package spec directories dynamically instead of hardcoded maps
Odilhao Aug 27, 2026
fa2928a
Build the directory index once instead of per-package
Odilhao Aug 27, 2026
46072fb
Address review: prefix-order bug, exact-match priority, hardening
Odilhao Aug 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions build_matrix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,7 @@
import os
import sys


def parse_package_list(lines):
for line in lines:
line = line.strip()
if line:
name, version = line.split('==')
yield {'package_name': name, 'new_version': version}
from find_package import parse_package_list


def main():
Expand Down
242 changes: 105 additions & 137 deletions find_package.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,128 +2,97 @@

import subprocess
import sys
import json
import os
import re
import glob

PACKAGES_DIR = "packages"
PYTHON_PREFIX = "python-"


def canonicalize(name):
"""PEP 503 style canonicalization: lowercase, collapse runs of -_. into a single '-'.

Used to match PyPI project names (which may use '-', '_' or '.' inconsistently,
e.g. 'ruamel.yaml', 'poetry_core', 'opentelemetry-api') against the on-disk spec
directory suffix for the same package, regardless of which separator style either
side happens to use. Applied before any prefix stripping, since pip freeze can emit
either 'python-socks' or 'python_socks' for the same project.
"""
return re.sub(r"[-_.]+", "-", name).lower()


def build_directory_index(packages_dir=PACKAGES_DIR):
"""Map canonical package name -> actual on-disk spec directory suffix.

Every RPM-packaged python library lives at packages/python-<suffix>/python-<suffix>.spec.
<suffix> doesn't always match the PyPI project name's separator style (e.g. the directory
suffix is 'poetry_core' while the PyPI project is 'poetry-core', or 'et-xmlfile' while the
PyPI project is 'et_xmlfile'). Building this index from the actual directories on disk means
every current and future package resolves correctly without a hand-maintained mapping table.

Raises if two directories canonicalize to the same key: that would make resolution
filesystem-order-dependent, and this repo has renamed directories between separator
styles before (galaxy_importer -> galaxy-importer, importlib_resources -> importlib-resources).
"""
index = {}
for spec_path in sorted(glob.glob(f"{packages_dir}/python-*/python-*.spec")):
dir_name = spec_path.split("/")[-2]
suffix = dir_name[len(PYTHON_PREFIX):]
key = canonicalize(suffix)
if key in index and index[key] != suffix:
raise ValueError(
f"Ambiguous package directory: '{index[key]}' and '{suffix}' both canonicalize to '{key}'"
)
index[key] = suffix
return index


def resolve_package_dir(pkg, directory_index=None):
"""Resolve a PyPI package name to its on-disk spec directory suffix, if packaged.

Tries an exact canonical match first, since a handful of PyPI projects (e.g. 'gnupg'
and 'python-gnupg') are genuinely distinct packages that happen to differ only by a
'python-' prefix -- stripping the prefix unconditionally would collapse them onto the
same directory. Only if there's no exact match do we fall back to stripping a leading
'python-' from the canonical form, for projects (python-socks, python-dateutil, ...)
that bundle this repo's own directory-naming prefix into their PyPI name.
"""
if directory_index is None:
directory_index = build_directory_index()

canonical_pkg = canonicalize(pkg)

if canonical_pkg in directory_index:
return directory_index[canonical_pkg]

if canonical_pkg.startswith(PYTHON_PREFIX):
stripped = canonical_pkg[len(PYTHON_PREFIX):]
if stripped in directory_index:
return directory_index[stripped]

return None


def parse_package_list(lines):
# Define transformation rules for better maintainability

# Packages that need prefix removal
prefix_removals = {
'python-': ''
}

# Specific package name mappings (exact matches)
package_mappings = {
'typing_extensions': 'typing-extensions',
'galaxy_importer': 'galaxy-importer',
'psycopg-c': 'psycopg_c',
'importlib_resources': 'importlib-resources',
'ruamel.yaml': 'ruamel-yaml',
'ruamel.yaml.clib': 'ruamel-yaml-clib',
'jaraco.classes': 'jaraco-classes',
'et_xmlfile': 'et-xmlfile',
'aiohttp_socks': 'aiohttp-socks',
'pyasn1_modules': 'pyasn1-modules',
'pydantic_core': 'pydantic-core',
'flit_core': 'flit-core',
'poetry_core': 'poetry-core',
'poetry_plugin_export': 'poetry-plugin-export',
}

# Packages that need to be lowercased
lowercase_packages = {
'PyYAML': 'pyyaml',
'GitPython': 'gitpython',
'Deprecated': 'deprecated',
'CacheControl': 'cachecontrol',
'Django': 'django',
'Jinja2': 'jinja2',
'MarkupPy': 'markuppy',
'MarkupSafe': 'markupsafe',
'Parsley': 'parsley',
'PyGObject': 'pygobject',
'Pygments': 'pygments',
'PyJWT': 'pyjwt',
'RapidFuzz': 'rapidfuzz',
'SecretStorage': 'secretstorage',
}

# Pattern-based transformations
def apply_pattern_transformations(name):
# OpenTelemetry packages: replace - with _
if name.startswith('opentelemetry'):
return name.replace('-', '_')
# Poetry packages: only transform the main poetry package and core packages with underscores
elif name == 'poetry' or (name.startswith('poetry') and '_' in name):
return name.replace('-', '_')
# Flit packages: replace _ with -
elif name.startswith('flit') and '_' in name:
return name.replace('_', '-')
# ET packages: replace _ with -
elif name.startswith('et') and '_' in name:
return name.replace('_', '-')
# AioHTTP packages: replace _ with -
elif name.startswith('aiohttp') and '_' in name:
return name.replace('_', '-')
# PyASN1 packages: replace _ with -
elif name.startswith('pyasn1') and '_' in name:
return name.replace('_', '-')
# Jaraco packages: replace . with -
elif name.startswith('jaraco') and '.' in name:
return name.replace('.', '-')
# Pydantic packages: replace _ with -
elif name.startswith('pydantic') and '_' in name:
return name.replace('_', '-')
# Ruamel packages: replace . with -
elif name.startswith('ruamel') and '.' in name:
return name.replace('.', '-')
# Default: no transformation
return name

for line in lines:
line = line.strip()
if line:
name, version = line.split('==')

# Apply prefix removals first
for prefix, replacement in prefix_removals.items():
if name.startswith(prefix):
name = name.replace(prefix, replacement, 1)
break

# Apply specific package mappings
if name in package_mappings:
name = package_mappings[name]
# Apply lowercase mappings
elif name in lowercase_packages:
name = lowercase_packages[name]
# Apply pattern-based transformations
else:
name = apply_pattern_transformations(name)

yield {'package_name': name, 'new_version': version}

def find_packages(pkg, new_version):
# Create reverse mapping to find the original package name for directory lookup
reverse_mappings = {
'poetry-core': 'poetry_core',
'poetry-plugin-export': 'poetry_plugin_export',
'galaxy-importer': 'galaxy_importer',
'importlib-resources': 'importlib_resources',
'ruamel-yaml': 'ruamel.yaml',
'ruamel-yaml-clib': 'ruamel.yaml.clib',
'jaraco-classes': 'jaraco.classes',
'et-xmlfile': 'et_xmlfile',
'aiohttp-socks': 'aiohttp_socks',
'pyasn1-modules': 'pyasn1_modules',
}

# Use original package name for directory lookup if it exists in reverse mapping
dir_pkg_name = reverse_mappings.get(pkg, pkg)

# Set paths and file names
spec_file = f"packages/python-{dir_pkg_name}/python-{dir_pkg_name}.spec"
if not line:
continue
if "==" not in line:
print(f"Skipping unparseable requirements line: {line}")
continue
name, version = line.split("==")
yield {"package_name": name, "new_version": version}


def find_packages(pkg, new_version, directory_index=None):
dir_pkg_name = resolve_package_dir(pkg, directory_index)

if dir_pkg_name is None:
print(f"Spec file not found for package {pkg} (no packages/python-* directory matches)")
return

spec_file = f"{PACKAGES_DIR}/python-{dir_pkg_name}/python-{dir_pkg_name}.spec"

# Retrieve the current RPM version from the spec file
try:
Expand All @@ -137,33 +106,32 @@ def find_packages(pkg, new_version):
vercmp_cmd = ["rpmdev-vercmp", rpm_version, new_version]
exit_code = subprocess.run(vercmp_cmd).returncode

# The resolved directory suffix (not the raw PyPI name) is what downstream tooling
# (update_packages.sh, PR title/branch name) expects, since it builds spec paths the
# same naive "packages/python-$pkg/python-$pkg.spec" way without re-resolving names.
if exit_code == 12:
print(f"RPM for Package {pkg} needs to be updated from {rpm_version} to {new_version}")
print(f"RPM for Package {dir_pkg_name} needs to be updated from {rpm_version} to {new_version}")
with open("packages-to-update.txt", "a") as file:
file.write(f"{pkg} {new_version}\n")
file.write(f"{dir_pkg_name} {new_version}\n")
elif exit_code == 0:
print(f"Package {pkg} version is the same as the packaged RPM")
print(f"Package {dir_pkg_name} version is the same as the packaged RPM")
elif exit_code == 11:
print(f"Packaged {pkg} RPM is newer than the version in requirements")

def build_package_list(file_handle):
for line in file_handle:
pkg_info = line.strip().split()
if len(pkg_info) != 2:
print(f"Invalid entry in list: {line.strip()}")
continue

pkg, new_version = pkg_info
find_packages(pkg, new_version)

print(f"Packaged {dir_pkg_name} RPM is newer than the version in requirements")


def main():
packages = list(parse_package_list(sys.stdin.readlines()))

directory_index = build_directory_index()

if not directory_index:
sys.exit(
f"No package directories found under '{PACKAGES_DIR}/python-*/'. "
"Run this script from the repository root."
)

for package in packages:
find_packages(package['package_name'], package['new_version'])
find_packages(package["package_name"], package["new_version"], directory_index)


if __name__ == '__main__':
if __name__ == "__main__":
main()
Loading