diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7b3ebce7..de53b856 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -20,34 +20,42 @@ jobs: fetch-depth: 1 persist-credentials: false - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - - name: Test release build-output materialization + - name: Test release catalog materialization run: | - ./tests/release_build_output_descriptors_test.sh - ./tests/release_build_output_prepare_test.sh - ./tests/release_build_output_test.sh - - name: Prepare release build-output dispatch smoke test + ./tests/release_catalog_config_test.sh + ./tests/release_catalog_discovery_test.sh + ./tests/release_catalog_package_parsing_test.sh + ./tests/release_catalog_test.sh + - name: Prepare release catalog dispatch smoke test run: | - mkdir -p release-build-output-smoke - printf '%s\n' smoke >release-build-output-smoke/package.tar.gz - - name: Run release build-output dispatch smoke test - uses: ./release-build-output-dispatch + mkdir -p release-catalog-smoke + printf '%s\n' smoke >release-catalog-smoke/package.tar.gz + jq -n \ + '{ecosystem: "archive", name: "smoke", version: "1.0"}' \ + >release-catalog-smoke/package-identity.json + - name: Run release catalog dispatch smoke test + uses: ./release-catalog-dispatch env: + RAPIDS_SHA: ${{ github.sha }} SHARED_ACTIONS_REPO: ${{ github.event.pull_request.head.repo.full_name }} SHARED_ACTIONS_REF: ${{ github.event.pull_request.head.sha }} with: - artifact-type: custom - output-directory: release-build-output-smoke - release-artifacts: '[{"path":"package.tar.gz"}]' - release-package: '{"ecosystem":"archive","name":"smoke","version":"1.0"}' - release-unit: archive:smoke - source-artifact-name: release-build-output-dispatch-smoke - source-sha: ${{ github.event.pull_request.head.sha }} - - name: Verify release build-output dispatch smoke test + config: >- + { + "release_catalog_key": "archive:smoke", + "artifact_directory": "release-catalog-smoke", + "artifacts": [{ + "path": "package.tar.gz", + "package_identity_file": "package-identity.json" + }] + } + source-artifact-name: release-catalog-dispatch-smoke + - name: Verify release catalog dispatch smoke test run: | jq -e ' - .artifacts[0].unit_id == "archive:smoke" - and .artifacts[0].path == "package.tar.gz" - ' release-build-output-smoke/release-build-output.json >/dev/null - jq -e ' - .metadata.artifacts == [{path: "package.tar.gz", sbom_kind: "generated-identity"}] - ' release-build-output-smoke/release-build-metadata.json >/dev/null + .producer == "shared-workflows" + and .source.artifact == "release-catalog-dispatch-smoke" + and .entries[0].release_catalog_key == "archive:smoke" + and .entries[0].path == "package.tar.gz" + and .entries[0].sbom_kind == "generated-identity" + ' release-catalog-smoke/release-catalog-entries.json >/dev/null diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 799d7320..6cf849f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,13 @@ ci: autoupdate_schedule: quarterly - skip: [actionlint-docker] + # These hooks require tools unavailable in pre-commit.ci: + # + # * actionlint-docker requires Docker + # * release-catalog-config requires jq + # + # Both run in other CI jobs with controlled runtime dependencies. + skip: [actionlint-docker, release-catalog-config] repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -11,6 +17,7 @@ repos: hooks: - id: trailing-whitespace - id: check-added-large-files + - id: check-json - id: check-yaml - id: end-of-file-fixer - repo: https://github.com/zizmorcore/zizmor-pre-commit @@ -52,3 +59,18 @@ repos: hooks: - id: yamllint additional_dependencies: [pyyaml] + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.37.2 + hooks: + - id: check-jsonschema + name: validate release catalog config schema fixtures + args: [--schemafile, release-catalog/config.schema.json] + files: ^tests/release-catalog-config/.*\.json$ + - repo: local + hooks: + - id: release-catalog-config + name: validate release catalog configuration + entry: ./tests/release_catalog_config_test.sh + language: system + pass_filenames: false + files: ^(release-catalog/(config\.schema\.json|validate-config\.sh)|tests/release_catalog_config_test\.sh)$ diff --git a/README.md b/README.md index 4ea1b0ca..40f29a86 100644 --- a/README.md +++ b/README.md @@ -9,35 +9,15 @@ A dispatch action is one that: * clones the shared-actions repository (repo/ref changeable using env vars) * runs (dispatches to) another action within the clone, using a relative path -## Release build-output companions +## Release catalog companions -`release-build-output-dispatch` validates a producer's local build artifact +`release-catalog-dispatch` validates a producer's local build artifact directory and uploads a companion artifact named -`release-build-output-`. The companion contains -`release-build-output.json`, `release-build-metadata.json`, provenance, and an -SBOM record for every primary artifact. +`release-catalog-`. -Conda and wheel jobs can set `artifact-type` to `conda` or `wheel` and omit -`release-artifacts`; the implementation reads package metadata from the built -files. Custom bundles provide explicit artifact descriptors and either inline -package identity or a producer-created package JSON file. - -```yaml -- name: Create release build-output companion - uses: rapidsai/shared-actions/release-build-output-dispatch@main - with: - artifact-type: wheel - output-directory: ${{ steps.package-name.outputs.WHEEL_OUTPUT_DIR }} - release-unit: wheel:example - source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} - source-sha: ${{ github.sha }} -``` - -A descriptor-selected producer SBOM is classified as `producer-dependency`. -When no SBOM is supplied, the action generates an SPDX artifact-identity -envelope and classifies it as `generated-identity`. The generated envelope -contains the primary artifact's identity and SHA-256 but no dependency -inventory; it must not be treated as dependency coverage. +See the [release catalog documentation](release-catalog/README.md) +for configuration, examples, source-revision handling, companion contents, and +generated identity evidence semantics. The dispatch wrapper honors `SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF`. When neither is set, it checks out the same repository and ref used to invoke diff --git a/release-build-output-dispatch/action.yml b/release-build-output-dispatch/action.yml deleted file mode 100644 index 154ba3d7..00000000 --- a/release-build-output-dispatch/action.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Dispatch release build output -description: Check out the selected shared-actions revision and create a release build-output companion. - -inputs: - artifact-type: - description: One of conda, wheel, or custom. - required: true - release-unit: - description: Release-platform unit ID for every primary artifact in this bundle. - required: true - release-package: - description: JSON package fields shared by the bundle. - required: false - release-package-file: - description: Relative path to producer-created package JSON inside output-directory. - required: false - release-artifacts: - description: JSON artifact and evidence descriptors relative to output-directory. - required: false - output-directory: - description: Directory containing the primary files and any producer-supplied evidence. - required: true - manifest-name: - description: Filename to write inside output-directory. - required: false - default: release-build-output.json - metadata-name: - description: Filename for the build metadata envelope. - required: false - default: release-build-metadata.json - source-artifact-name: - description: Name of the GitHub Actions artifact bundle containing this output. - required: true - source-sha: - description: Source revision built by the producing job. - required: false - -outputs: - manifest-path: - description: Absolute path to the generated manifest. - value: ${{ steps.release-build-output.outputs.manifest-path }} - metadata-path: - description: Absolute path to the build metadata envelope. - value: ${{ steps.release-build-output.outputs.metadata-path }} - manifest-artifact-name: - description: Name of the uploaded GitHub Actions companion artifact. - value: ${{ steps.release-build-output.outputs.manifest-artifact-name }} - -runs: - using: composite - steps: - - name: Check out shared-actions implementation - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ env.SHARED_ACTIONS_REPO || github.action_repository || 'rapidsai/shared-actions' }} - ref: ${{ env.SHARED_ACTIONS_REF || github.action_ref || 'main' }} - path: ./shared-actions - persist-credentials: false - - id: release-build-output - name: Create release build-output companion - uses: ./shared-actions/release-build-output - with: - artifact-type: ${{ inputs.artifact-type }} - release-unit: ${{ inputs.release-unit }} - release-package: ${{ inputs.release-package }} - release-package-file: ${{ inputs.release-package-file }} - release-artifacts: ${{ inputs.release-artifacts }} - output-directory: ${{ inputs.output-directory }} - manifest-name: ${{ inputs.manifest-name }} - metadata-name: ${{ inputs.metadata-name }} - source-artifact-name: ${{ inputs.source-artifact-name }} - source-sha: ${{ inputs.source-sha }} diff --git a/release-build-output/action.yml b/release-build-output/action.yml deleted file mode 100644 index 7c6965e3..00000000 --- a/release-build-output/action.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Create release build output companion -description: Validate a build artifact bundle and upload its release-platform manifest and evidence companion. - -inputs: - artifact-type: - description: One of conda, wheel, or custom. Conda and wheel descriptors are derived when release-artifacts is omitted. - required: true - release-unit: - description: Release-platform unit ID for every primary artifact in this bundle. - required: true - release-package: - description: JSON package fields shared by the bundle. Provide this or release-package-file for custom artifacts. - required: false - release-package-file: - description: Relative path to producer-created package JSON inside output-directory. - required: false - release-artifacts: - description: JSON artifact and evidence descriptors relative to output-directory. Required for custom artifacts. - required: false - output-directory: - description: Directory containing the primary files and any producer-supplied evidence. - required: true - manifest-name: - description: Filename to write inside output-directory. - required: false - default: release-build-output.json - metadata-name: - description: Filename for the build-environment and SBOM-classification envelope. - required: false - default: release-build-metadata.json - source-artifact-name: - description: Name of the GitHub Actions artifact bundle containing this output. - required: true - source-sha: - description: Source revision built by the producing job. Defaults to the current workflow SHA. - required: false - -outputs: - manifest-path: - description: Absolute path to the generated manifest. - value: ${{ steps.materialize.outputs.manifest-path }} - metadata-path: - description: Absolute path to the build metadata envelope. - value: ${{ steps.materialize.outputs.metadata-path }} - manifest-artifact-name: - description: Name of the uploaded GitHub Actions companion artifact. - value: ${{ steps.companion-name.outputs.name }} - -runs: - using: composite - steps: - - id: prepare - name: Describe release artifacts - shell: bash - env: - RELEASE_ARTIFACTS: ${{ inputs.release-artifacts }} - RELEASE_ARTIFACT_TYPE: ${{ inputs.artifact-type }} - RELEASE_OUTPUT_DIRECTORY: ${{ inputs.output-directory }} - RELEASE_PACKAGE: ${{ inputs.release-package }} - RELEASE_PACKAGE_FILE: ${{ inputs.release-package-file }} - run: ./shared-actions/release-build-output/prepare.sh - - id: materialize - name: Materialize release build-output records - shell: bash - env: - RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} - RELEASE_MANIFEST_NAME: ${{ inputs.manifest-name }} - RELEASE_METADATA_NAME: ${{ inputs.metadata-name }} - RELEASE_OUTPUT_DIRECTORY: ${{ inputs.output-directory }} - RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} - RELEASE_PACKAGE_FILE: ${{ inputs.release-package-file }} - RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} - RELEASE_UNIT: ${{ inputs.release-unit }} - run: ./shared-actions/release-build-output/materialize.sh - - id: companion-name - name: Set companion artifact name - shell: bash - env: - SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - run: echo "name=release-build-output-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" - - name: Upload release build-output companion - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: error - name: ${{ steps.companion-name.outputs.name }} - path: | - ${{ inputs.output-directory }}/${{ inputs.manifest-name }} - ${{ inputs.output-directory }}/${{ inputs.metadata-name }} - ${{ inputs.output-directory }}/release-evidence/** diff --git a/release-build-output/describe-conda.sh b/release-build-output/describe-conda.sh deleted file mode 100755 index 6449addd..00000000 --- a/release-build-output/describe-conda.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -if [[ "$#" -ne 1 || ! -d "$1" ]]; then - echo "usage: $0 CONDA_OUTPUT_DIRECTORY" >&2 - exit 1 -fi - -output_directory="$(realpath "$1")" -descriptors='[]' -package_count=0 - -while IFS= read -r package_path; do - package_path="$(realpath "${package_path}")" - if [[ "${package_path}" != "${output_directory}"/* ]]; then - echo "Conda package must resolve inside output directory: ${package_path}" >&2 - exit 1 - fi - relative_path="${package_path#"${output_directory}/"}" - - case "${package_path}" in - *.conda) - info_members=() - while IFS= read -r info_member; do - info_members+=("${info_member}") - done < <(unzip -Z1 "${package_path}" | awk '/^info-.*\.tar\.zst$/') - if [[ "${#info_members[@]}" -ne 1 ]]; then - echo ".conda package must contain exactly one info-*.tar.zst member: ${relative_path}" >&2 - exit 1 - fi - index_json="$(unzip -p "${package_path}" "${info_members[0]}" | zstd -dc | tar -xOf - info/index.json)" - ;; - *.tar.bz2) - index_json="$(tar -xOjf "${package_path}" info/index.json)" - ;; - *) - echo "unsupported Conda package extension: ${relative_path}" >&2 - exit 1 - ;; - esac - - if ! jq -e ' - type == "object" - and (.name | type == "string" and length > 0) - and (.version | type == "string" and length > 0) - and (.build | type == "string" and length > 0) - and (.subdir | type == "string" and length > 0) - ' <<<"${index_json}" >/dev/null; then - echo "Conda info/index.json must contain exact name, version, build, and subdir fields: ${relative_path}" >&2 - exit 1 - fi - - package="$(jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}")" - descriptors="$(jq -cn \ - --arg path "${relative_path}" \ - --argjson package "${package}" \ - --argjson current "${descriptors}" \ - '$current + [{path: $path, package: $package}]')" - package_count=$((package_count + 1)) -done < <(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) - -if [[ "${package_count}" -eq 0 ]]; then - echo "Conda output directory contains no .conda or .tar.bz2 packages: ${output_directory}" >&2 - exit 1 -fi - -printf '%s\n' "${descriptors}" diff --git a/release-build-output/describe-wheels.sh b/release-build-output/describe-wheels.sh deleted file mode 100755 index bc6b925e..00000000 --- a/release-build-output/describe-wheels.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -if [[ "$#" -ne 1 || ! -d "$1" ]]; then - echo "usage: $0 WHEEL_OUTPUT_DIRECTORY" >&2 - exit 1 -fi - -output_directory="$(realpath "$1")" -descriptors='[]' -wheel_count=0 - -while IFS= read -r wheel_path; do - wheel_path="$(realpath "${wheel_path}")" - if [[ "${wheel_path}" != "${output_directory}"/* ]]; then - echo "wheel must resolve inside output directory: ${wheel_path}" >&2 - exit 1 - fi - relative_path="${wheel_path#"${output_directory}/"}" - - metadata_members=() - while IFS= read -r metadata_member; do - metadata_members+=("${metadata_member}") - done < <(unzip -Z1 "${wheel_path}" | awk '/\.dist-info\/METADATA$/') - if [[ "${#metadata_members[@]}" -ne 1 ]]; then - echo "wheel must contain exactly one .dist-info/METADATA file: ${relative_path}" >&2 - exit 1 - fi - - metadata="$(unzip -p "${wheel_path}" "${metadata_members[0]}")" - package_name="$(awk 'tolower($0) ~ /^name:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" - package_version="$(awk 'tolower($0) ~ /^version:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" - if [[ -z "${package_name}" || -z "${package_version}" ]]; then - echo "wheel metadata must contain non-empty Name and Version fields: ${relative_path}" >&2 - exit 1 - fi - - descriptors="$(jq -cn \ - --arg path "${relative_path}" \ - --arg name "${package_name}" \ - --arg version "${package_version}" \ - --argjson current "${descriptors}" \ - '$current + [{path: $path, package: {ecosystem: "wheel", name: $name, version: $version}}]')" - wheel_count=$((wheel_count + 1)) -done < <(find "${output_directory}" -type f -name '*.whl' -print | sort) - -if [[ "${wheel_count}" -eq 0 ]]; then - echo "wheel output directory contains no .whl files: ${output_directory}" >&2 - exit 1 -fi - -printf '%s\n' "${descriptors}" diff --git a/release-build-output/materialize.sh b/release-build-output/materialize.sh deleted file mode 100755 index 673dd6ba..00000000 --- a/release-build-output/materialize.sh +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -require_nonempty() { - local name="$1" - local value="$2" - if [[ -z "${value}" ]]; then - echo "${name} must be a non-empty string" >&2 - exit 1 - fi -} - -require_nonempty "RELEASE_UNIT" "${RELEASE_UNIT:-}" -require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" -require_nonempty "RELEASE_MANIFEST_NAME" "${RELEASE_MANIFEST_NAME:-}" -require_nonempty "RELEASE_METADATA_NAME" "${RELEASE_METADATA_NAME:-}" -require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" -require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" - -if [[ -n "${RELEASE_PACKAGE:-}" && -n "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "release-package and release-package-file are mutually exclusive" >&2 - exit 1 -fi -if [[ -z "${RELEASE_PACKAGE:-}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "one of release-package or release-package-file is required" >&2 - exit 1 -fi - -source_sha="${RELEASE_SOURCE_SHA:-${GITHUB_SHA:-}}" -require_nonempty "RELEASE_SOURCE_SHA or GITHUB_SHA" "${source_sha}" - -require_plain_filename() { - local label="$1" - local filename="$2" - if [[ "${filename}" == */* || "${filename}" == .* || "${filename}" == *".."* ]]; then - echo "${label} must be a plain filename" >&2 - exit 1 - fi -} - -require_plain_filename "manifest-name" "${RELEASE_MANIFEST_NAME}" -require_plain_filename "metadata-name" "${RELEASE_METADATA_NAME}" -if [[ "${RELEASE_MANIFEST_NAME}" == "${RELEASE_METADATA_NAME}" ]]; then - echo "manifest-name and metadata-name must differ" >&2 - exit 1 -fi - -if [[ ! -d "${RELEASE_OUTPUT_DIRECTORY}" ]]; then - echo "output-directory does not exist or is not a directory: ${RELEASE_OUTPUT_DIRECTORY}" >&2 - exit 1 -fi - -ensure_relative_pattern() { - local field="$1" - local pattern="$2" - if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then - echo "${field} must be a relative path inside output-directory: ${pattern}" >&2 - exit 1 - fi -} - -output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" -if [[ -n "${RELEASE_PACKAGE_FILE:-}" ]]; then - ensure_relative_pattern "release-package-file" "${RELEASE_PACKAGE_FILE}" - package_file_path="$(realpath "${output_directory}/${RELEASE_PACKAGE_FILE}")" - if [[ "${package_file_path}" != "${output_directory}"/* || ! -f "${package_file_path}" ]]; then - echo "release-package-file must resolve to one file inside output-directory: ${RELEASE_PACKAGE_FILE}" >&2 - exit 1 - fi - RELEASE_PACKAGE="$(jq -c . "${package_file_path}")" -fi - -if ! jq -e ' - type == "object" - and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) - and (.ecosystem | type == "string" and length > 0) - and (.name | type == "string" and length > 0) - and ((.version // "") | type == "string") - and ((.build // "") | type == "string") - and ((.platform // "") | type == "string") -' <<<"${RELEASE_PACKAGE}" >/dev/null; then - echo "release-package must be a package object with ecosystem and name; version may be supplied or derived per artifact" >&2 - exit 1 -fi - -if ! jq -e 'type == "array" and length > 0' <<<"${RELEASE_ARTIFACTS}" >/dev/null; then - echo "release-artifacts must be a non-empty JSON array" >&2 - exit 1 -fi - -manifest_path="${output_directory}/${RELEASE_MANIFEST_NAME}" -metadata_path="${output_directory}/${RELEASE_METADATA_NAME}" -temporary_manifest="$(mktemp "${output_directory}/.release-build-output.XXXXXX")" -trap 'rm -f "${temporary_manifest}"' EXIT - -printf '%s\n' '{"schema_version":1,"producer":"release-platform","artifacts":[]}' >"${temporary_manifest}" - -resolve_one_file() { - local field="$1" - local pattern="$2" - local -a matches=() - - ensure_relative_pattern "${field}" "${pattern}" - while IFS= read -r match; do - matches+=("${match}") - done < <(compgen -G "${output_directory}/${pattern}" || true) - if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then - echo "${field} pattern must resolve to exactly one file: ${pattern}" >&2 - exit 1 - fi - - local resolved - resolved="$(realpath "${matches[0]}")" - if [[ "${resolved}" != "${output_directory}"/* ]]; then - echo "${field} must resolve inside output-directory: ${pattern}" >&2 - exit 1 - fi - printf '%s\n' "${resolved#"${output_directory}/"}" -} - -derive_package_version() { - local ecosystem="$1" - local package_name="$2" - local artifact_path="$3" - local filename - filename="$(basename "${artifact_path}")" - - case "${ecosystem}" in - conda) - local conda_prefix="${package_name}-" - if [[ "${filename}" != "${conda_prefix}"* ]]; then - echo "Conda artifact filename does not start with package name '${package_name}': ${filename}" >&2 - exit 1 - fi - local conda_remainder="${filename#"${conda_prefix}"}" - local conda_version="${conda_remainder%%-*}" - if [[ -z "${conda_version}" || "${conda_version}" == "${conda_remainder}" ]]; then - echo "cannot derive Conda package version from artifact filename: ${filename}" >&2 - exit 1 - fi - printf '%s\n' "${conda_version}" - ;; - wheel) - local wheel_prefix="${package_name//-/_}-" - if [[ "${filename}" != "${wheel_prefix}"* ]]; then - echo "wheel artifact filename does not start with normalized package name '${package_name}': ${filename}" >&2 - exit 1 - fi - local wheel_remainder="${filename#"${wheel_prefix}"}" - local wheel_version="${wheel_remainder%%-*}" - if [[ -z "${wheel_version}" || "${wheel_version}" == "${wheel_remainder}" ]]; then - echo "cannot derive wheel package version from artifact filename: ${filename}" >&2 - exit 1 - fi - printf '%s\n' "${wheel_version}" - ;; - *) - echo "release-package version is required for ${ecosystem} artifacts" >&2 - exit 1 - ;; - esac -} - -generated_evidence_path() { - local primary_path="$1" - local kind="$2" - local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - local artifact_label="${primary_path//\//_}" - printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" -} - -copy_supplied_evidence() { - local primary_path="$1" - local supplied_path="$2" - local kind="$3" - local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - local artifact_label="${primary_path//\//_}" - local destination - destination="release-evidence/${artifact_label}.${artifact_digest}/${kind}-$(basename "${supplied_path}")" - mkdir -p "$(dirname "${output_directory}/${destination}")" - cp "${output_directory}/${supplied_path}" "${output_directory}/${destination}" - printf '%s\n' "${destination}" -} - -write_generated_sbom() { - local primary_path="$1" - local package="$2" - local destination="$3" - local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - mkdir -p "$(dirname "${output_directory}/${destination}")" - jq -n -S \ - --arg artifact_digest "${artifact_digest}" \ - --arg artifact_path "${primary_path}" \ - --argjson package "${package}" \ - '{ - spdxVersion: "SPDX-2.3", - dataLicense: "CC0-1.0", - SPDXID: "SPDXRef-DOCUMENT", - name: ("RAPIDS release artifact " + $artifact_path), - documentNamespace: ("https://rapids.ai/release-platform/spdx/" + $artifact_digest), - creationInfo: { - creators: ["Tool: rapidsai/shared-workflows release-build-output"], - created: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) - }, - documentDescribes: ["SPDXRef-Artifact"], - packages: [{ - SPDXID: "SPDXRef-Artifact", - name: $package.name, - versionInfo: $package.version, - downloadLocation: "NOASSERTION", - filesAnalyzed: false, - checksums: [{algorithm: "SHA256", checksumValue: $artifact_digest}] - }], - relationships: [{ - spdxElementId: "SPDXRef-DOCUMENT", - relationshipType: "DESCRIBES", - relatedSpdxElement: "SPDXRef-Artifact" - }], - comment: "Artifact-identity SBOM envelope. A producer-supplied dependency SBOM may replace this record." - }' >"${output_directory}/${destination}" -} - -write_generated_provenance() { - local primary_path="$1" - local package="$2" - local destination="$3" - local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - mkdir -p "$(dirname "${output_directory}/${destination}")" - jq -n -S \ - --arg artifact_digest "${artifact_digest}" \ - --arg artifact_path "${primary_path}" \ - --arg repository "${GITHUB_REPOSITORY:-}" \ - --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ - --arg run_id "${GITHUB_RUN_ID:-}" \ - --arg source_sha "${source_sha}" \ - --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ - --argjson package "${package}" \ - '{ - _type: "https://in-toto.io/Statement/v1", - subject: [{name: $artifact_path, digest: {sha256: $artifact_digest}}], - predicateType: "https://slsa.dev/provenance/v1", - predicate: { - buildDefinition: { - buildType: "https://rapids.ai/release-platform/build-output/v1", - externalParameters: {release_unit: env.RELEASE_UNIT, package: $package}, - resolvedDependencies: [{ - uri: ("git+https://github.com/" + $repository + "@" + $source_sha), - digest: {gitCommit: $source_sha} - }] - }, - runDetails: { - builder: {id: ("https://github.com/" + $workflow_ref)}, - metadata: {invocationId: ("https://github.com/" + $repository + "/actions/runs/" + $run_id + "/attempts/" + $run_attempt)} - } - } - }' >"${output_directory}/${destination}" -} - -shopt -s globstar nullglob -artifact_metadata='[]' -while IFS= read -r descriptor; do - if ! jq -e ' - type == "object" - and (keys - ["path", "sbom", "provenance", "signature", "package"] | length == 0) - and (.path | type == "string" and length > 0) - and ((.sbom // "") | type == "string") - and ((.provenance // "") | type == "string") - and ((.signature // "") | type == "string") - and ((.package // {}) | type == "object") - and ((.package // {} | keys - ["ecosystem", "name", "version", "build", "platform"]) | length == 0) - and ((.package // {} | to_entries | map(.value | type == "string" and length > 0) | all)) - ' <<<"${descriptor}" >/dev/null; then - echo "every release-artifacts entry must contain path and optional SBOM/provenance/signature/package overrides" >&2 - exit 1 - fi - - primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" - sbom_pattern="$(jq -r '.sbom // empty' <<<"${descriptor}")" - provenance_pattern="$(jq -r '.provenance // empty' <<<"${descriptor}")" - signature_pattern="$(jq -r '.signature // empty' <<<"${descriptor}")" - package_override="$(jq -c '.package // {}' <<<"${descriptor}")" - - package="$(jq -cn --argjson base "${RELEASE_PACKAGE}" --argjson override "${package_override}" '$base + $override')" - package_version="$(jq -r '.version // empty' <<<"${package}")" - if [[ -z "${package_version}" ]]; then - package_version="$(derive_package_version "$(jq -r '.ecosystem' <<<"${package}")" "$(jq -r '.name' <<<"${package}")" "${primary_path}")" - package="$(jq -c --arg version "${package_version}" '. + {version: $version}' <<<"${package}")" - fi - - if [[ -n "${sbom_pattern}" ]]; then - supplied_sbom_path="$(resolve_one_file sbom "${sbom_pattern}")" - sbom_path="$(copy_supplied_evidence "${primary_path}" "${supplied_sbom_path}" "sbom")" - sbom_kind="producer-dependency" - else - sbom_path="$(generated_evidence_path "${primary_path}" "spdx")" - write_generated_sbom "${primary_path}" "${package}" "${sbom_path}" - sbom_kind="generated-identity" - fi - if [[ -n "${provenance_pattern}" ]]; then - supplied_provenance_path="$(resolve_one_file provenance "${provenance_pattern}")" - provenance_path="$(copy_supplied_evidence "${primary_path}" "${supplied_provenance_path}" "provenance")" - else - provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" - write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" - fi - artifact="$(jq -cn \ - --arg unit_id "${RELEASE_UNIT}" \ - --arg path "${primary_path}" \ - --arg sbom "${sbom_path}" \ - --arg provenance "${provenance_path}" \ - --argjson package "${package}" \ - '{unit_id: $unit_id, path: $path, sbom: $sbom, provenance: $provenance, package: $package}')" - if [[ -n "${signature_pattern}" ]]; then - supplied_signature_path="$(resolve_one_file signature "${signature_pattern}")" - signature_path="$(copy_supplied_evidence "${primary_path}" "${supplied_signature_path}" "signature")" - artifact="$(jq -c --arg signature "${signature_path}" '. + {signature: $signature}' <<<"${artifact}")" - fi - - artifact_metadata="$(jq -cn \ - --arg path "${primary_path}" \ - --arg sbom_kind "${sbom_kind}" \ - --argjson artifacts "${artifact_metadata}" \ - '$artifacts + [{path: $path, sbom_kind: $sbom_kind}]')" - - jq --argjson artifact "${artifact}" '.artifacts += [$artifact]' "${temporary_manifest}" >"${temporary_manifest}.next" - mv "${temporary_manifest}.next" "${temporary_manifest}" -done < <(jq -c '.[]' <<<"${RELEASE_ARTIFACTS}") - -if ! jq -e '.artifacts as $items | ($items | map([.unit_id, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then - echo "release-artifacts contains duplicate unit/path entries" >&2 - exit 1 -fi - -jq -S . "${temporary_manifest}" >"${manifest_path}" -jq -n -S \ - --arg artifact_name "${RELEASE_SOURCE_ARTIFACT_NAME}" \ - --arg manifest_name "${RELEASE_MANIFEST_NAME}" \ - --arg repository "${GITHUB_REPOSITORY:-}" \ - --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ - --arg run_id "${GITHUB_RUN_ID:-}" \ - --arg sha "${source_sha}" \ - --arg unit_id "${RELEASE_UNIT}" \ - --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ - --argjson artifact_metadata "${artifact_metadata}" \ - '{ - schema_version: 1, - producer: "shared-workflows", - release_unit: $unit_id, - source_artifact: $artifact_name, - build_output_manifest: $manifest_name, - build_environment: { - repository: $repository, - sha: $sha, - workflow_ref: $workflow_ref, - run_id: $run_id, - run_attempt: $run_attempt - }, - metadata: {artifacts: $artifact_metadata} - }' >"${metadata_path}" -echo "manifest-path=${manifest_path}" >>"${GITHUB_OUTPUT}" -echo "metadata-path=${metadata_path}" >>"${GITHUB_OUTPUT}" diff --git a/release-build-output/prepare.sh b/release-build-output/prepare.sh deleted file mode 100755 index f3aa97f4..00000000 --- a/release-build-output/prepare.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -require_nonempty() { - local name="$1" - local value="$2" - if [[ -z "${value}" ]]; then - echo "${name} must be a non-empty string" >&2 - exit 1 - fi -} - -require_nonempty "RELEASE_ARTIFACT_TYPE" "${RELEASE_ARTIFACT_TYPE:-}" -require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" - -script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -artifacts="${RELEASE_ARTIFACTS:-}" -package="${RELEASE_PACKAGE:-}" - -case "${RELEASE_ARTIFACT_TYPE}" in - conda) - if [[ -z "${artifacts}" ]]; then - artifacts="$("${script_directory}/describe-conda.sh" "${RELEASE_OUTPUT_DIRECTORY}")" - fi - if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - package='{"ecosystem":"conda","name":"bundle"}' - fi - ;; - wheel) - if [[ -z "${artifacts}" ]]; then - artifacts="$("${script_directory}/describe-wheels.sh" "${RELEASE_OUTPUT_DIRECTORY}")" - fi - if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - package='{"ecosystem":"wheel","name":"bundle"}' - fi - ;; - custom) - require_nonempty "RELEASE_ARTIFACTS" "${artifacts}" - ;; - *) - echo "artifact-type must be one of: conda, custom, wheel" >&2 - exit 1 - ;; -esac - -if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "one of release-package or release-package-file is required for custom artifacts" >&2 - exit 1 -fi - -printf 'artifacts=%s\n' "${artifacts}" >>"${GITHUB_OUTPUT}" -printf 'package=%s\n' "${package}" >>"${GITHUB_OUTPUT}" diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml new file mode 100644 index 00000000..77758741 --- /dev/null +++ b/release-catalog-dispatch/action.yml @@ -0,0 +1,59 @@ +name: Dispatch release catalog +description: Create a release catalog companion from the selected shared-actions revision. + +inputs: + config: + description: >- + JSON string with configuration for this action. Schema and field documentation: + https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json + required: true + source-artifact-name: + description: Name of the existing GitHub Actions artifact whose files are being cataloged. + required: true +runs: + using: composite + steps: + - id: implementation + name: Resolve release catalog implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + implementation_path="$(dirname "${action_path}")/release-catalog" + if [[ ! -d "${implementation_path}" ]]; then + echo "release-catalog implementation not found at ${implementation_path}" >&2 + exit 1 + fi + echo "path=${implementation_path}" >>"${GITHUB_OUTPUT}" + - id: configuration + name: Validate release catalog configuration + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_CATALOG_CONFIG: ${{ inputs.config }} + run: "${IMPLEMENTATION_PATH}/validate-config.sh" + - name: Materialize release catalog entries + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} + RELEASE_ENTRIES_NAME: release-catalog-entries.json + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} + RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} + run: "${IMPLEMENTATION_PATH}/materialize.sh" + - name: Upload release catalog companion + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: error + name: release-catalog-${{ inputs.source-artifact-name }} + path: | + ${{ steps.configuration.outputs.artifact_directory }}/release-catalog-entries.json + ${{ steps.configuration.outputs.artifact_directory }}/release-evidence/** diff --git a/release-catalog/README.md b/release-catalog/README.md new file mode 100644 index 00000000..c37275df --- /dev/null +++ b/release-catalog/README.md @@ -0,0 +1,156 @@ +# Release catalog companions + +`release-catalog-dispatch` records the exact files produced by a package +build and uploads a companion GitHub Actions artifact named +`release-catalog-`. Release tooling uses the +companion to associate primary artifacts with their build identity and generated +identity evidence without reconstructing the build later. This evidence supports: + +- [assembling releases that can be tested and verified before formal tagging](https://github.com/rapidsai/release-scripts/issues/102) +- using build-time metadata to triage CVE scan records more quickly (internal + GitLab project, `nspect-manager`) + +The companion contains: + +```text +. +├── release-catalog-entries.json +└── release-evidence + ├── .provenance.json + └── .spdx.json +``` + +`release-catalog-entries.json` is one atomic job-level envelope. Its `source` +object records the source artifact and build context, while its `entries` array +contains the release catalog entries produced by that job. The release platform +validates and aggregates entry arrays from selected builds into the release +catalog; there is no separate metadata document to keep synchronized. + +Each matrix job produces its own companion artifact. The release platform merges +their entries into one composite release catalog. + +## Configuration + +Every build job that calls this action passes one `config` JSON object. Its +canonical schema and field documentation are in +[`config.schema.json`](config.schema.json). + +The action validates the configuration before inspecting build outputs. It then +verifies properties that depend on produced files, including the package +identity contents and whether primary-artifact paths resolve unambiguously. + +### `release_catalog_key` + +The `release_catalog_key` is used to group artifacts in the catalog. Every file +from every matrix variant in the same publishable artifact set uses the same +key. The release platform aggregates multiple `release-catalog-entries.json` +files and coalesces entries with the same `release_catalog_key`. Standard RAPIDS +conda and wheel workflows construct the key as `:`, +such as `conda:cudf`. Custom producers select a key, such as `maven:cuvs-java`, +that represents their release policy. + +Use a distinct `release_catalog_key` only when the outputs intentionally have +different release policies. Reasons include differences in: + +- versioning schemes +- validation requirements +- dependency ordering +- publication destinations +- promotion strategy + +Multiple package names from one repository do not by themselves justify separate +keys: for example, `cudf` and `dask-cudf` Conda packages remain part of +`conda:cudf` when they share one release policy. + +### `artifacts` + +This is a list of objects, where each object corresponds to exactly one artifact +(filename). Standard Conda and wheel workflows do not explicitly specify +`artifacts`. Instead, the action discovers every Conda and wheel output in +`artifact_directory` and creates corresponding artifact entries. + +```yaml +- name: Create wheel release catalog companion + uses: rapidsai/shared-actions/release-catalog-dispatch@main + with: + config: >- + { + "release_catalog_key": "wheel:example", + "artifact_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} + } + source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} +``` + +For artifacts other than conda and wheels, `path` and `package_identity_file` must +be specified: + +```yaml +- name: Create custom release catalog companion + uses: rapidsai/shared-actions/release-catalog-dispatch@main + with: + config: >- + { + "release_catalog_key": "maven:cuvs-java", + "artifact_directory": "java/cuvs-java/target", + "artifacts": [{ + "path": "cuvs-java-*-x86_64-cuda*.jar", + "package_identity_file": "cuvs-java.release-package-identity.json" + }] + } + source-artifact-name: cuvs-java +``` + +Wildcards are allowed to allow for variance in filenames, but each wildcard must +resolve to only one file. In other words, each list object corresponds to +exactly one artifact, and any ambiguity is an error. + +#### Package identity file + +Package identity is high-level information about an artifact outside of its +filename. It requires `ecosystem`, `name`, and `version` and may include `build` +and `platform`. The action implicitly parses this information from conda and +wheel artifacts, but other artifact formats require the producer to provide it +explicitly using the `package_identity_file` parameter. Any referenced package +identity file must be created before this action runs. An artifact descriptor's +`package_identity_file` is the path to that file relative to +`artifact_directory`. Example contents of a package_identity_file: + +```json +{ + "ecosystem": "maven", + "name": "ai.rapids:cuvs-java", + "version": "26.08.0" +} +``` + +Multiple artifact descriptors may reference the same identity file when those +artifacts have the same package identity. + +## Prerequisite state + +The action requires `RAPIDS_SHA` in the job environment. It must identify the +repository revision actually checked out and built: + +- Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately + after checkout. +- Wheel and custom workflows use `rapids-github-info`; it uses `inputs.sha` + when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. + +This distinction matters when a reusable workflow checks out a repository or +revision different from the workflow event. Direct callers must likewise set +`RAPIDS_SHA` to the checked-out commit instead of assuming `${{ github.sha }}` +names the built source. Variables written to `GITHUB_ENV` are available to the +action when it runs as a subsequent job step. + +## Generated identity evidence + +For every artifact, the action generates an SPDX artifact-identity envelope +classified as `generated-identity` and a build-context provenance statement. +They record the artifact SHA-256, package identity, source revision, and workflow +context, but contain no dependency or source-license inventory. They must not be +reported as producer-supplied dependency coverage. + +Producer-supplied dependency SBOMs and richer build provenance, such as build +dependencies and compiler flags, would make package contents easier to +understand without downloading them. Associating that input-side evidence with +artifacts is left for future work. diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json new file mode 100644 index 00000000..11165ee5 --- /dev/null +++ b/release-catalog/config.schema.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rapids.ai/schemas/release-catalog-config-v1.json", + "title": "Release catalog configuration", + "description": "Producer configuration for one release catalog companion.", + "type": "object", + "additionalProperties": false, + "required": [ + "release_catalog_key" + ], + "properties": { + "release_catalog_key": { + "description": "Release-catalog entry name that owns these artifacts. Default for conda & wheels is :, such as conda:cudf. Custom jobs specify their own, such as maven:cuvs-java.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "artifact_directory": { + "description": "Base directory for artifact paths. Defaults to the job working directory.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$", + "default": "." + }, + "artifacts": { + "description": "Optional explicit primary release files. When omitted, Conda and wheel artifacts are discovered in artifact_directory.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/artifact" + } + } + }, + "$defs": { + "artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "path" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "package_identity_file": { + "description": "Path, relative to artifact_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#package-identity-file", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + } + } + } + } +} diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh new file mode 100755 index 00000000..1bde11ee --- /dev/null +++ b/release-catalog/materialize.sh @@ -0,0 +1,414 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +require_nonempty() { + local name="$1" + local value="$2" + if [[ -z "${value}" ]]; then + echo "${name} must be a non-empty string" >&2 + exit 1 + fi +} + +require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" +require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" +require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" +require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" + +source_sha="${RAPIDS_SHA:-}" +require_nonempty "RAPIDS_SHA" "${source_sha}" + +require_plain_filename() { + local label="$1" + local filename="$2" + if [[ "${filename}" == */* || "${filename}" == .* || "${filename}" == *".."* ]]; then + echo "${label} must be a plain filename" >&2 + exit 1 + fi +} + +require_plain_filename "entries-name" "${RELEASE_ENTRIES_NAME}" + +if [[ ! -d "${RELEASE_ARTIFACT_DIRECTORY}" ]]; then + echo "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" >&2 + exit 1 +fi + +ensure_relative_pattern() { + local field="$1" + local pattern="$2" + if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then + echo "${field} must be a relative path inside artifact-directory: ${pattern}" >&2 + exit 1 + fi +} + +artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" + +resolve_one_file() { + local field="$1" + local pattern="$2" + local -a matches=() + + ensure_relative_pattern "${field}" "${pattern}" + while IFS= read -r match; do + matches+=("${match}") + done < <(compgen -G "${artifact_directory}/${pattern}" || true) + if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then + echo "${field} pattern must resolve to exactly one file: ${pattern}" >&2 + exit 1 + fi + + local resolved + resolved="$(realpath "${matches[0]}")" + if [[ "${resolved}" != "${artifact_directory}"/* ]]; then + echo "${field} must resolve inside artifact-directory: ${pattern}" >&2 + exit 1 + fi + printf '%s\n' "${resolved#"${artifact_directory}/"}" +} + +describe_wheel_package() { + local wheel_path="$1" + local relative_path="${wheel_path#"${artifact_directory}/"}" + local -a metadata_members=() + local metadata_member + while IFS= read -r metadata_member; do + metadata_members+=("${metadata_member}") + done < <(unzip -Z1 "${wheel_path}" | awk '/\.dist-info\/METADATA$/') + if [[ "${#metadata_members[@]}" -ne 1 ]]; then + echo "wheel must contain exactly one .dist-info/METADATA file: ${relative_path}" >&2 + exit 1 + fi + + local metadata package_name package_version + metadata="$(unzip -p "${wheel_path}" "${metadata_members[0]}")" + package_name="$(awk 'tolower($0) ~ /^name:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + package_version="$(awk 'tolower($0) ~ /^version:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + if [[ -z "${package_name}" || -z "${package_version}" ]]; then + echo "wheel metadata must contain non-empty Name and Version fields: ${relative_path}" >&2 + exit 1 + fi + + jq -cn --arg name "${package_name}" --arg version "${package_version}" \ + '{ecosystem: "wheel", name: $name, version: $version}' +} + +describe_conda_package() { + local package_path="$1" + local relative_path="${package_path#"${artifact_directory}/"}" + local index_json + case "${package_path}" in + *.conda) + local -a info_members=() + local info_member + while IFS= read -r info_member; do + info_members+=("${info_member}") + done < <(unzip -Z1 "${package_path}" | awk '/^info-.*\.tar\.zst$/') + if [[ "${#info_members[@]}" -ne 1 ]]; then + echo ".conda package must contain exactly one info-*.tar.zst member: ${relative_path}" >&2 + exit 1 + fi + index_json="$(unzip -p "${package_path}" "${info_members[0]}" | zstd -dc | tar -xOf - info/index.json)" + ;; + *.tar.bz2) + index_json="$(tar -xOjf "${package_path}" info/index.json)" + ;; + *) + echo "unsupported Conda package extension: ${relative_path}" >&2 + exit 1 + ;; + esac + + if ! jq -e ' + type == "object" + and (.name | type == "string" and length > 0) + and (.version | type == "string" and length > 0) + and (.build | type == "string" and length > 0) + and (.subdir | type == "string" and length > 0) + ' <<<"${index_json}" >/dev/null; then + echo "Conda info/index.json must contain exact name, version, build, and subdir fields: ${relative_path}" >&2 + exit 1 + fi + + jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}" +} + +prepare_artifacts() { + local configured_artifacts="${RELEASE_ARTIFACTS:-}" + local prepared_artifacts='[]' + local conda_package descriptor package primary_path primary_file identity_file + + if [[ -z "${configured_artifacts}" ]]; then + local -a detected_files=() + while IFS= read -r primary_file; do + detected_files+=("${primary_file}") + done < <(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' -o -name '*.whl' \) -print | sort) + + for primary_file in "${detected_files[@]}"; do + primary_path="${primary_file#"${artifact_directory}/"}" + case "${primary_file}" in + *.whl) + if ! package="$(describe_wheel_package "${primary_file}")"; then + return 1 + fi + ;; + *) + if ! package="$(describe_conda_package "${primary_file}")"; then + return 1 + fi + ;; + esac + descriptor="$(jq -cn --arg path "${primary_path}" --argjson package "${package}" '{path: $path, package: $package}')" + prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" + done + else + if ! jq -e 'type == "array" and length > 0' <<<"${configured_artifacts}" >/dev/null; then + echo "release-artifacts must be a non-empty JSON array when supplied" >&2 + exit 1 + fi + while IFS= read -r descriptor; do + if jq -e 'has("package")' <<<"${descriptor}" >/dev/null; then + echo "package identity must not be supplied inline: ${descriptor}" >&2 + return 1 + fi + primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" + primary_file="${artifact_directory}/${primary_path}" + identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" + package='' + + case "${primary_file}" in + *.whl) + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + if ! package="$(describe_wheel_package "${primary_file}")"; then + return 1 + fi + ;; + *.conda) + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + if ! package="$(describe_conda_package "${primary_file}")"; then + return 1 + fi + ;; + *.tar.bz2) + if conda_package="$(describe_conda_package "${primary_file}" 2>/dev/null)"; then + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + package="${conda_package}" + elif [[ -z "${identity_file}" ]]; then + echo "artifact is not a valid Conda package and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; + *) + if [[ -z "${identity_file}" ]]; then + echo "artifact identity cannot be extracted; package_identity_file is required: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; + esac + + if [[ -n "${package}" ]]; then + descriptor="$(jq -c --argjson package "${package}" '. + {package: $package}' <<<"${descriptor}")" + fi + prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" + done < <(jq -c '.[]' <<<"${configured_artifacts}") + fi + + if [[ "$(jq 'length' <<<"${prepared_artifacts}")" -eq 0 ]]; then + echo "artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 + exit 1 + fi + printf '%s\n' "${prepared_artifacts}" +} + +artifacts="$(prepare_artifacts)" +entries_path="${artifact_directory}/${RELEASE_ENTRIES_NAME}" +temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" +trap 'rm -f "${temporary_manifest}"' EXIT + +printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" + +validate_package_identity() { + jq -e ' + type == "object" + and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (.ecosystem | type == "string" and length > 0) + and (.name | type == "string" and length > 0) + and (.version | type == "string" and length > 0) + and ((has("build") | not) or (.build | type == "string" and length > 0)) + and ((has("platform") | not) or (.platform | type == "string" and length > 0)) + ' >/dev/null +} + +generated_evidence_path() { + local primary_path="$1" + local kind="$2" + local artifact_digest + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + local artifact_label="${primary_path//\//_}" + printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" +} + +write_generated_sbom() { + local primary_path="$1" + local package="$2" + local destination="$3" + local artifact_digest + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" + jq -n -S \ + --arg artifact_digest "${artifact_digest}" \ + --arg artifact_path "${primary_path}" \ + --argjson package "${package}" \ + '{ + spdxVersion: "SPDX-2.3", + dataLicense: "CC0-1.0", + SPDXID: "SPDXRef-DOCUMENT", + name: ("RAPIDS release artifact " + $artifact_path), + # This is a unique identifier, not a retrieval URL; SPDX requires an + # absolute URI for the document namespace but does not require it to resolve. + documentNamespace: ("https://rapids.ai/release-platform/spdx/" + $artifact_digest), + creationInfo: { + creators: ["Tool: rapidsai/shared-workflows release catalog"], + created: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) + }, + documentDescribes: ["SPDXRef-Artifact"], + packages: [{ + SPDXID: "SPDXRef-Artifact", + name: $package.name, + versionInfo: $package.version, + downloadLocation: "NOASSERTION", + filesAnalyzed: false, + checksums: [{algorithm: "SHA256", checksumValue: $artifact_digest}] + }], + relationships: [{ + spdxElementId: "SPDXRef-DOCUMENT", + relationshipType: "DESCRIBES", + relatedSpdxElement: "SPDXRef-Artifact" + }], + comment: "Artifact-identity SBOM envelope. It does not contain a dependency inventory." + }' >"${artifact_directory}/${destination}" +} + +write_generated_provenance() { + local primary_path="$1" + local package="$2" + local destination="$3" + local artifact_digest + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" + jq -n -S \ + --arg artifact_digest "${artifact_digest}" \ + --arg artifact_path "${primary_path}" \ + --arg repository "${GITHUB_REPOSITORY:-}" \ + --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ + --arg run_id "${GITHUB_RUN_ID:-}" \ + --arg source_sha "${source_sha}" \ + --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ + --argjson package "${package}" \ + '{ + _type: "https://in-toto.io/Statement/v1", + subject: [{name: $artifact_path, digest: {sha256: $artifact_digest}}], + predicateType: "https://slsa.dev/provenance/v1", + predicate: { + buildDefinition: { + buildType: "https://rapids.ai/release-platform/catalog-record/v1", + externalParameters: {release_catalog_key: env.RELEASE_CATALOG_KEY, package: $package}, + resolvedDependencies: [{ + uri: ("git+https://github.com/" + $repository + "@" + $source_sha), + digest: {gitCommit: $source_sha} + }] + }, + runDetails: { + builder: {id: ("https://github.com/" + $workflow_ref)}, + metadata: {invocationId: ("https://github.com/" + $repository + "/actions/runs/" + $run_id + "/attempts/" + $run_attempt)} + } + } + }' >"${artifact_directory}/${destination}" +} + +shopt -s globstar nullglob +while IFS= read -r descriptor; do + if ! jq -e ' + type == "object" + and (keys - ["path", "package_identity_file", "package"] | length == 0) + and (.path | type == "string" and length > 0) + and ((.package_identity_file // "") | type == "string") + and ([has("package"), has("package_identity_file")] | map(select(.)) | length == 1) + ' <<<"${descriptor}" >/dev/null; then + echo "release artifact descriptor must contain path and exactly one package identity source: ${descriptor}" >&2 + exit 1 + fi + + primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" + package_identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" + if [[ -n "${package_identity_file}" ]]; then + package_identity_path="$(resolve_one_file package_identity_file "${package_identity_file}")" + if ! package="$(jq -ce . "${artifact_directory}/${package_identity_path}" 2>/dev/null)"; then + echo "package_identity_file must contain valid JSON: ${package_identity_file}" >&2 + exit 1 + fi + else + package="$(jq -c '.package' <<<"${descriptor}")" + fi + if ! validate_package_identity <<<"${package}"; then + echo "package identity for ${primary_path} must contain non-empty ecosystem, name, and version strings and only optional build or platform strings" >&2 + exit 1 + fi + + sbom_path="$(generated_evidence_path "${primary_path}" "spdx")" + write_generated_sbom "${primary_path}" "${package}" "${sbom_path}" + provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" + write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" + + # TODO: Accept producer-supplied SBOM, provenance, and signature evidence after + # the release catalog defines a generic artifact-to-evidence association. + entry="$(jq -cn \ + --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ + --arg path "${primary_path}" \ + --arg sbom "${sbom_path}" \ + --arg provenance "${provenance_path}" \ + --argjson package "${package}" \ + '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, sbom_kind: "generated-identity", provenance: $provenance, package: $package}')" + + jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" + mv "${temporary_manifest}.next" "${temporary_manifest}" +done < <(jq -c '.[]' <<<"${artifacts}") + +if ! jq -e '.entries as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then + echo "release-artifacts contains duplicate release_catalog_key/path entries" >&2 + exit 1 +fi + +jq -n -S \ + --arg artifact_name "${RELEASE_SOURCE_ARTIFACT_NAME}" \ + --arg repository "${GITHUB_REPOSITORY:-}" \ + --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ + --arg run_id "${GITHUB_RUN_ID:-}" \ + --arg sha "${source_sha}" \ + --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ + --argjson entries "$(jq -c '.entries' "${temporary_manifest}")" \ + '{ + schema_version: 1, + producer: "shared-workflows", + source: { + artifact: $artifact_name, + repository: $repository, + sha: $sha, + workflow_ref: $workflow_ref, + run_id: $run_id, + run_attempt: $run_attempt + }, + entries: $entries + }' >"${entries_path}" diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh new file mode 100755 index 00000000..5ed529e9 --- /dev/null +++ b/release-catalog/validate-config.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +emit_error() { + local message="$1" + printf '::error title=Invalid release catalog configuration::%s\n' "${message}" >&2 +} + +config="${RELEASE_CATALOG_CONFIG:-}" +if [[ -z "${config}" ]]; then + emit_error "release catalog configuration must be a non-empty JSON object" + exit 1 +fi + +if ! compact_config="$(jq -ce . <<<"${config}" 2>/dev/null)"; then + parse_error="$(jq -ce . <<<"${config}" 2>&1 || true)" + emit_error "release catalog configuration must be valid JSON: ${parse_error}" + exit 1 +fi + +validation_errors="$(jq -r ' + def single_line_string: + type == "string" and length > 0 and (test("[\\r\\n]") | not); + if type != "object" then + ["release catalog configuration must be a JSON object"] + else + (keys - ["release_catalog_key", "artifact_directory", "artifacts"]) as $unknown + | [ + if ($unknown | length) > 0 then + "unknown field(s): " + ($unknown | join(", ")) + else empty end, + if (.release_catalog_key | single_line_string) then empty + else "release_catalog_key must be a non-empty, single-line string" end, + if (has("artifact_directory") | not) or (.artifact_directory | single_line_string) then empty + else "artifact_directory must be a non-empty, single-line string when supplied" end + ] + + if has("artifacts") then + [ + if (.artifacts | type == "array" and length > 0) then empty + else "artifacts must be a non-empty array when supplied" end + ] + else [] end + + if (.artifacts | type) == "array" then + [ + .artifacts | to_entries[] + | .key as $index + | .value as $artifact + | if ($artifact | type) != "object" then + "artifacts[\($index)] must be an object" + else + ($artifact | keys - ["path", "package_identity_file"]) as $artifact_unknown + | if ($artifact_unknown | length) > 0 then + "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) + else empty end, + if ($artifact.path | single_line_string) then empty + else "artifacts[\($index)].path must be a non-empty, single-line string" end, + ($artifact | to_entries[] + | select(.key == "package_identity_file") + | select((.value | single_line_string) | not) + | "artifacts[\($index)].\(.key) must be a non-empty, single-line string") + end + ] + else [] end + end + | .[] +' <<<"${compact_config}")" + +if [[ -n "${validation_errors}" ]]; then + while IFS= read -r validation_error; do + emit_error "${validation_error}" + done <<<"${validation_errors}" + exit 1 +fi + +if [[ -z "${GITHUB_OUTPUT:-}" ]]; then + emit_error "GITHUB_OUTPUT is required" + exit 1 +fi + +{ + printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" + printf 'artifact_directory=%s\n' "$(jq -r '.artifact_directory // "."' <<<"${compact_config}")" + printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" +} >>"${GITHUB_OUTPUT}" diff --git a/tests/release-catalog-config/package-identity-file.json b/tests/release-catalog-config/package-identity-file.json new file mode 100644 index 00000000..c3b55a05 --- /dev/null +++ b/tests/release-catalog-config/package-identity-file.json @@ -0,0 +1,10 @@ +{ + "release_catalog_key": "maven:cuvs-java", + "artifact_directory": "java/cuvs-java/target", + "artifacts": [ + { + "path": "cuvs-java-*-x86_64-cuda*.jar", + "package_identity_file": "cuvs-java.release-package-identity.json" + } + ] +} diff --git a/tests/release_build_output_prepare_test.sh b/tests/release_build_output_prepare_test.sh deleted file mode 100755 index 3598e1e5..00000000 --- a/tests/release_build_output_prepare_test.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -temporary_directory="$(mktemp -d)" -trap 'rm -rf "${temporary_directory}"' EXIT - -mkdir -p "${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info" -printf '%s\n' \ - 'Metadata-Version: 2.1' \ - 'Name: libkvikio-cu12' \ - 'Version: 26.8.0' \ - >"${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info/METADATA" -( - cd "${temporary_directory}/wheel" - zip -qr "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" . -) - -GITHUB_OUTPUT="${temporary_directory}/wheel-output" -RELEASE_ARTIFACTS='' -RELEASE_ARTIFACT_TYPE=wheel -RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE='' -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE - -"${repository_root}/release-build-output/prepare.sh" - -grep -Fx 'package={"ecosystem":"wheel","name":"bundle"}' "${GITHUB_OUTPUT}" -artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" -jq -e ' - . == [{ - path: "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl", - package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} - }] -' <<<"${artifacts}" >/dev/null - -GITHUB_OUTPUT="${temporary_directory}/custom-output" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", sbom: "bundle.spdx.json"}]')" -RELEASE_ARTIFACT_TYPE=custom -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE=release-package.json -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_PACKAGE RELEASE_PACKAGE_FILE - -"${repository_root}/release-build-output/prepare.sh" - -grep -Fx 'artifacts=[{"path":"bundle.tar.gz","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" -grep -Fx 'package=' "${GITHUB_OUTPUT}" - -GITHUB_OUTPUT="${temporary_directory}/invalid-output" -RELEASE_ARTIFACT_TYPE=archive -export GITHUB_OUTPUT RELEASE_ARTIFACT_TYPE -if "${repository_root}/release-build-output/prepare.sh" 2>"${temporary_directory}/invalid-error"; then - echo "prepare.sh unexpectedly accepted an invalid artifact type" >&2 - exit 1 -fi -grep -Fx 'artifact-type must be one of: conda, custom, wheel' "${temporary_directory}/invalid-error" diff --git a/tests/release_build_output_test.sh b/tests/release_build_output_test.sh deleted file mode 100755 index 2a5dc6e1..00000000 --- a/tests/release_build_output_test.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -temporary_directory="$(mktemp -d)" -trap 'rm -rf "${temporary_directory}"' EXIT - -bundle_directory="${temporary_directory}/bundle" -mkdir -p "${bundle_directory}" -printf '%s\n' jar >"${bundle_directory}/cuvs-java-26.08.0.jar" -printf '%s\n' sbom >"${bundle_directory}/cuvs-java-26.08.0.spdx.json" -printf '%s\n' provenance >"${bundle_directory}/cuvs-java-26.08.0.provenance.jsonl" -printf '%s\n' signature >"${bundle_directory}/cuvs-java-26.08.0.jar.asc" -jq -n \ - '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ - >"${bundle_directory}/cuvs-java-package.json" - -GITHUB_OUTPUT="${temporary_directory}/github-output" -GITHUB_REPOSITORY="rapidsai/cuvs" -GITHUB_RUN_ATTEMPT="1" -GITHUB_RUN_ID="1234" -GITHUB_SHA="0123456789012345678901234567890123456789" -GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" -export GITHUB_OUTPUT -RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" -RELEASE_MANIFEST_NAME="release-build-output.json" -RELEASE_METADATA_NAME="release-build-metadata.json" -RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE="cuvs-java-package.json" -RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" -RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -RELEASE_UNIT="maven:cuvs-java" -export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_MANIFEST_NAME RELEASE_METADATA_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE -export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT - -"${repository_root}/release-build-output/materialize.sh" - -canonical_bundle_directory="$(realpath "${bundle_directory}")" -manifest_path="${canonical_bundle_directory}/release-build-output.json" -metadata_path="${canonical_bundle_directory}/release-build-metadata.json" -jq -e ' - .schema_version == 1 - and .producer == "release-platform" - and (.artifacts | length == 1) - and .artifacts[0].unit_id == "maven:cuvs-java" - and .artifacts[0].path == "cuvs-java-26.08.0.jar" - and .artifacts[0].package.name == "ai.rapids:cuvs-java" -' "${manifest_path}" >/dev/null -jq -e ' - .schema_version == 1 - and .producer == "shared-workflows" - and .release_unit == "maven:cuvs-java" - and .source_artifact == "cuvs-java-cuda12.9.1" - and .build_output_manifest == "release-build-output.json" - and .build_environment.repository == "rapidsai/cuvs" - and .build_environment.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - and .metadata.artifacts == [{path: "cuvs-java-26.08.0.jar", sbom_kind: "producer-dependency"}] -' "${metadata_path}" >/dev/null -grep -Fx "manifest-path=${manifest_path}" "${GITHUB_OUTPUT}" -grep -Fx "metadata-path=${metadata_path}" "${GITHUB_OUTPUT}" - -supplied_sbom_path="$(jq -r '.artifacts[0].sbom' "${manifest_path}")" -supplied_provenance_path="$(jq -r '.artifacts[0].provenance' "${manifest_path}")" -supplied_signature_path="$(jq -r '.artifacts[0].signature' "${manifest_path}")" -[[ "${supplied_sbom_path}" == release-evidence/*/sbom-* ]] -[[ "${supplied_provenance_path}" == release-evidence/*/provenance-* ]] -[[ "${supplied_signature_path}" == release-evidence/*/signature-* ]] -grep -Fx sbom "${canonical_bundle_directory}/${supplied_sbom_path}" -grep -Fx provenance "${canonical_bundle_directory}/${supplied_provenance_path}" -grep -Fx signature "${canonical_bundle_directory}/${supplied_signature_path}" - -isolated_companion_directory="${temporary_directory}/isolated-companion" -mkdir -p "${isolated_companion_directory}" -cp "${manifest_path}" "${metadata_path}" "${isolated_companion_directory}/" -cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" -while IFS= read -r evidence_path; do - test -f "${isolated_companion_directory}/${evidence_path}" -done < <(jq -r '.artifacts[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-build-output.json") - -generated_directory="${temporary_directory}/generated-bundle" -mkdir -p "${generated_directory}/linux-64" -printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" - -RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda"}]')" -RELEASE_OUTPUT_DIRECTORY="${generated_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "conda", name: "kvikio"}')" -RELEASE_PACKAGE_FILE='' -RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" -RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" -RELEASE_UNIT="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT - -"${repository_root}/release-build-output/materialize.sh" - -generated_manifest_path="${generated_directory}/release-build-output.json" -generated_metadata_path="${generated_directory}/release-build-metadata.json" -generated_sbom_path="$(jq -r '.artifacts[0].sbom' "${generated_manifest_path}")" -generated_provenance_path="$(jq -r '.artifacts[0].provenance' "${generated_manifest_path}")" -jq -e ' - .artifacts[0].unit_id == "conda:kvikio" - and .artifacts[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" - and .artifacts[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} -' "${generated_manifest_path}" >/dev/null -jq -e ' - .spdxVersion == "SPDX-2.3" - and .packages[0].name == "kvikio" - and .packages[0].versionInfo == "26.08.00a32" -' "${generated_directory}/${generated_sbom_path}" >/dev/null -jq -e ' - .predicateType == "https://slsa.dev/provenance/v1" - and .predicate.buildDefinition.externalParameters.release_unit == "conda:kvikio" - and .predicate.buildDefinition.resolvedDependencies[0].digest.gitCommit == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" -' "${generated_directory}/${generated_provenance_path}" >/dev/null -jq -e ' - .release_unit == "conda:kvikio" - and .source_artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" - and .build_environment.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - and .metadata.artifacts == [{path: "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda", sbom_kind: "generated-identity"}] -' "${generated_metadata_path}" >/dev/null - -wheel_directory="${temporary_directory}/wheel-bundle" -mkdir -p "${wheel_directory}" -printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" - -RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl"}]')" -RELEASE_OUTPUT_DIRECTORY="${wheel_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "wheel", name: "libkvikio-cu12"}')" -RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" -RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" -RELEASE_UNIT="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT - -"${repository_root}/release-build-output/materialize.sh" - -jq -e ' - .artifacts[0].unit_id == "wheel:kvikio" - and .artifacts[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" - and .artifacts[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} -' "${wheel_directory}/release-build-output.json" >/dev/null -jq -e ' - .metadata.artifacts == [{path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", sbom_kind: "generated-identity"}] -' "${wheel_directory}/release-build-metadata.json" >/dev/null - -missing_version_directory="${temporary_directory}/missing-version-bundle" -mkdir -p "${missing_version_directory}" -printf '%s\n' archive >"${missing_version_directory}/bundle.tar.gz" - -RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" -RELEASE_OUTPUT_DIRECTORY="${missing_version_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "archive", name: "bundle"}')" -RELEASE_PACKAGE_FILE='' -RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" -RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" -RELEASE_UNIT="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT - -if "${repository_root}/release-build-output/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then - echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 - exit 1 -fi -grep -Fx 'release-package version is required for archive artifacts' "${temporary_directory}/missing-version-error" diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh new file mode 100755 index 00000000..cf615f90 --- /dev/null +++ b/tests/release_catalog_config_test.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +validator="${repository_root}/release-catalog/validate-config.sh" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +assert_invalid() { + local name="$1" + local config="$2" + local expected="$3" + local error_file="${temporary_directory}/${name}.error" + + if RELEASE_CATALOG_CONFIG="${config}" GITHUB_OUTPUT="${temporary_directory}/${name}.output" \ + "${validator}" 2>"${error_file}"; then + echo "validate-config.sh unexpectedly accepted ${name}" >&2 + exit 1 + fi + if ! grep -F -- "${expected}" "${error_file}" >/dev/null; then + echo "validate-config.sh did not explain ${name}; output was:" >&2 + sed 's/^/ /' "${error_file}" >&2 + exit 1 + fi +} + +valid_output="${temporary_directory}/valid.output" +RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/package-identity-file.json")" \ + GITHUB_OUTPUT="${valid_output}" "${validator}" + +grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" +grep -Fx 'artifact_directory=java/cuvs-java/target' "${valid_output}" +grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","package_identity_file":"cuvs-java.release-package-identity.json"}]' "${valid_output}" + +assert_invalid \ + malformed-json \ + '{"release_catalog_key":' \ + 'release catalog configuration must be valid JSON' + +assert_invalid \ + not-an-object \ + '[]' \ + 'release catalog configuration must be a JSON object' + +assert_invalid \ + missing-fields \ + '{}' \ + 'release_catalog_key must be a non-empty, single-line string' + +assert_invalid \ + top-level-package-identity-file \ + '{"release_catalog_key":"archive:smoke","package_identity_file":"package.json"}' \ + 'unknown field(s): package_identity_file' + +assert_invalid \ + unknown-field \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}],"component_id":"archive:smoke"}' \ + 'unknown field(s): component_id' + +assert_invalid \ + artifact-type \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}]}' \ + 'unknown field(s): artifact_type' + +assert_invalid \ + inline-package \ + '{"release_catalog_key":"archive:smoke","package":{"ecosystem":"archive","name":"smoke","version":"1.0"},"artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}]}' \ + 'unknown field(s): package' + +assert_invalid \ + per-artifact-package \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package":{"ecosystem":"archive","name":"smoke","version":"1.0"}}]}' \ + 'artifacts[0] has unknown field(s): package' + +assert_invalid \ + malformed-artifact \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"file":"smoke.tar.gz","package_identity_file":false,"sbom":false}]}' \ + 'artifacts[0] has unknown field(s): file, sbom' +grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null +grep -F 'artifacts[0].package_identity_file must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null + +default_output="${temporary_directory}/default.output" +RELEASE_CATALOG_CONFIG='{"release_catalog_key":"conda:smoke"}' \ + GITHUB_OUTPUT="${default_output}" "${validator}" +grep -Fx 'release_catalog_key=conda:smoke' "${default_output}" +grep -Fx 'artifact_directory=.' "${default_output}" +grep -Fx 'artifacts=' "${default_output}" + +standard_output="${temporary_directory}/standard.output" +RELEASE_CATALOG_CONFIG='{ + "release_catalog_key": "wheel:kvikio", + "artifact_directory": "dist" +}' GITHUB_OUTPUT="${standard_output}" "${validator}" + +grep -Fx 'release_catalog_key=wheel:kvikio' "${standard_output}" +grep -Fx 'artifact_directory=dist' "${standard_output}" +grep -Fx 'artifacts=' "${standard_output}" diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh new file mode 100755 index 00000000..fce7a1e6 --- /dev/null +++ b/tests/release_catalog_discovery_test.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +RELEASE_CATALOG_KEY="test:discovery" +RELEASE_ENTRIES_NAME="release-catalog-entries.json" +RELEASE_SOURCE_ARTIFACT_NAME="discovery-test" +RAPIDS_SHA="0123456789012345678901234567890123456789" +export RAPIDS_SHA RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME + +mkdir -p "${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info" +printf '%s\n' \ + 'Metadata-Version: 2.1' \ + 'Name: libkvikio-cu12' \ + 'Version: 26.8.0' \ + >"${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info/METADATA" +( + cd "${temporary_directory}/wheel" + zip -qr "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" . +) + +RELEASE_ARTIFACTS='' +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY + +"${repository_root}/release-catalog/materialize.sh" + +jq -e ' + [.entries[] | {path, package}] == [{ + path: "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl", + package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + }] +' "${temporary_directory}/release-catalog-entries.json" >/dev/null + +conda_output_directory="${temporary_directory}/conda" +conda_staging_directory="${temporary_directory}/conda-staging" +mkdir -p "${conda_output_directory}/noarch" "${conda_staging_directory}/info" +jq -n \ + '{name: "rapids-dask-dependency", version: "26.08.0", build: "py_0", subdir: "noarch"}' \ + >"${conda_staging_directory}/info/index.json" +tar -cjf \ + "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" \ + -C "${conda_staging_directory}" \ + info/index.json + +RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" +export RELEASE_ARTIFACT_DIRECTORY + +"${repository_root}/release-catalog/materialize.sh" + +jq -e ' + [.entries[] | {path, package}] == [{ + path: "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2", + package: { + ecosystem: "conda", + name: "rapids-dask-dependency", + version: "26.08.0", + build: "py_0", + platform: "noarch" + } + }] +' "${conda_output_directory}/release-catalog-entries.json" >/dev/null + +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" +jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ + >"${temporary_directory}/release-package-identity.json" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json"}]')" +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY + +"${repository_root}/release-catalog/materialize.sh" + +jq -e ' + .entries[0].path == "bundle.tar.gz" + and .entries[0].package == {ecosystem: "archive", name: "bundle", version: "1.0"} + and .entries[0].sbom_kind == "generated-identity" +' "${temporary_directory}/release-catalog-entries.json" >/dev/null + +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl"}]')" +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + .entries[0].path == "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" + and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + and .entries[0].sbom_kind == "generated-identity" +' "${temporary_directory}/release-catalog-entries.json" >/dev/null + +mixed_directory="${temporary_directory}/mixed" +mkdir -p "${mixed_directory}" +cp "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" "${mixed_directory}/" +cp "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" "${mixed_directory}/" +RELEASE_ARTIFACTS='' +RELEASE_ARTIFACT_DIRECTORY="${mixed_directory}" +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + (.entries | length == 2) + and ([.entries[].package.ecosystem] | sort) == ["conda", "wheel"] +' "${mixed_directory}/release-catalog-entries.json" >/dev/null + +empty_directory="${temporary_directory}/empty" +mkdir -p "${empty_directory}" +RELEASE_ARTIFACT_DIRECTORY="${empty_directory}" +export RELEASE_ARTIFACT_DIRECTORY +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/empty-error"; then + echo "materialize.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 + exit 1 +fi +grep -Fx 'artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" + +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/unsupported-error"; then + echo "materialize.sh unexpectedly accepted an unsupported artifact without identity" >&2 + exit 1 +fi +grep -Fx 'artifact identity cannot be extracted; package_identity_file is required: bundle.tar.gz' "${temporary_directory}/unsupported-error" + +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package_identity_file: "release-package-identity.json"}]')" +export RELEASE_ARTIFACTS +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/conflicting-identity-error"; then + echo "materialize.sh unexpectedly accepted an identity file for a parseable artifact" >&2 + exit 1 +fi +grep -Fx 'package_identity_file is not allowed when wheel identity can be extracted: libkvikio_cu12-*.whl' "${temporary_directory}/conflicting-identity-error" diff --git a/tests/release_build_output_descriptors_test.sh b/tests/release_catalog_package_parsing_test.sh similarity index 64% rename from tests/release_build_output_descriptors_test.sh rename to tests/release_catalog_package_parsing_test.sh index c6b6c46c..3b037c4d 100755 --- a/tests/release_build_output_descriptors_test.sh +++ b/tests/release_catalog_package_parsing_test.sh @@ -7,6 +7,13 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT +RELEASE_ARTIFACTS='' +RELEASE_CATALOG_KEY="test:package-parsing" +RELEASE_ENTRIES_NAME="release-catalog-entries.json" +RELEASE_SOURCE_ARTIFACT_NAME="package-parsing-test" +RAPIDS_SHA="0123456789012345678901234567890123456789" +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME + wheel_output_directory="${temporary_directory}/wheels" wheel_staging_directory="${temporary_directory}/wheel-staging" mkdir -p "${wheel_output_directory}" "${wheel_staging_directory}/libkvikio_cu12-26.8.0.dist-info" @@ -20,13 +27,15 @@ printf '%s\n' \ zip -qr "${wheel_output_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" . ) -wheel_descriptors="$("${repository_root}/release-build-output/describe-wheels.sh" "${wheel_output_directory}")" +RELEASE_ARTIFACT_DIRECTORY="${wheel_output_directory}" +export RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" jq -e ' - . == [{ + [.entries[] | {path, package}] == [{ path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} }] -' <<<"${wheel_descriptors}" >/dev/null +' "${wheel_output_directory}/release-catalog-entries.json" >/dev/null conda_output_directory="${temporary_directory}/conda" tar_bz2_staging_directory="${temporary_directory}/tar-bz2-staging" @@ -57,10 +66,12 @@ zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/i info-librmm.tar.zst ) -conda_descriptors="$("${repository_root}/release-build-output/describe-conda.sh" "${conda_output_directory}")" +RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" +export RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" jq -e ' - length == 2 - and any(.[]; + (.entries | length == 2) + and any(.entries[]; .path == "linux-64/librmm-26.08.00a32-cuda12_260714_2f567060.conda" and .package == { ecosystem: "conda", @@ -70,7 +81,7 @@ jq -e ' platform: "linux-64" } ) - and any(.[]; + and any(.entries[]; .path == "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" and .package == { ecosystem: "conda", @@ -80,4 +91,16 @@ jq -e ' platform: "noarch" } ) -' <<<"${conda_descriptors}" >/dev/null +' "${conda_output_directory}/release-catalog-entries.json" >/dev/null + +invalid_wheel_directory="${temporary_directory}/invalid-wheel" +mkdir -p "${invalid_wheel_directory}" +printf '%s\n' invalid >"${invalid_wheel_directory}/invalid.whl" +RELEASE_ARTIFACT_DIRECTORY="${invalid_wheel_directory}" +export RELEASE_ARTIFACT_DIRECTORY +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-wheel-error"; then + echo "materialize.sh unexpectedly accepted an invalid wheel" >&2 + exit 1 +fi +grep -F 'wheel must contain exactly one .dist-info/METADATA file: invalid.whl' \ + "${temporary_directory}/invalid-wheel-error" >/dev/null diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh new file mode 100755 index 00000000..86ce8c89 --- /dev/null +++ b/tests/release_catalog_test.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +bundle_directory="${temporary_directory}/bundle" +mkdir -p "${bundle_directory}" +printf '%s\n' jar >"${bundle_directory}/cuvs-java-26.08.0.jar" +jq -n \ + '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ + >"${bundle_directory}/cuvs-java-package-identity.json" + +GITHUB_REPOSITORY="rapidsai/cuvs" +GITHUB_RUN_ATTEMPT="1" +GITHUB_RUN_ID="1234" +GITHUB_SHA="0123456789012345678901234567890123456789" +GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json"}]')" +RELEASE_ENTRIES_NAME="release-catalog-entries.json" +RELEASE_ARTIFACT_DIRECTORY="${bundle_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" +RAPIDS_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +RELEASE_CATALOG_KEY="maven:cuvs-java" +export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF +export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_ARTIFACT_DIRECTORY +export RAPIDS_SHA RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY + +"${repository_root}/release-catalog/materialize.sh" + +canonical_bundle_directory="$(realpath "${bundle_directory}")" +entries_path="${canonical_bundle_directory}/release-catalog-entries.json" +test ! -e "${canonical_bundle_directory}/release-catalog-metadata.json" +jq -e ' + .schema_version == 1 + and .producer == "shared-workflows" + and .source.artifact == "cuvs-java-cuda12.9.1" + and .source.repository == "rapidsai/cuvs" + and .source.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + and (.entries | length == 1) + and .entries[0].release_catalog_key == "maven:cuvs-java" + and .entries[0].path == "cuvs-java-26.08.0.jar" + and .entries[0].package.name == "ai.rapids:cuvs-java" + and .entries[0].sbom_kind == "generated-identity" +' "${entries_path}" >/dev/null + +isolated_companion_directory="${temporary_directory}/isolated-companion" +mkdir -p "${isolated_companion_directory}" +cp "${entries_path}" "${isolated_companion_directory}/" +cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" +while IFS= read -r evidence_path; do + test -f "${isolated_companion_directory}/${evidence_path}" +done < <(jq -r '.entries[] | .sbom, .provenance' "${isolated_companion_directory}/release-catalog-entries.json") + +multiple_identity_directory="${temporary_directory}/multiple-identities" +mkdir -p "${multiple_identity_directory}" +printf '%s\n' first >"${multiple_identity_directory}/first.tar.gz" +printf '%s\n' second >"${multiple_identity_directory}/second.jar" +jq -n '{ecosystem: "archive", name: "first", version: "1.0"}' \ + >"${multiple_identity_directory}/first.identity.json" +jq -n '{ecosystem: "maven", name: "example:second", version: "2.0"}' \ + >"${multiple_identity_directory}/second.identity.json" +RELEASE_ARTIFACTS="$(jq -cn '[ + {path: "first.tar.gz", package_identity_file: "first.identity.json"}, + {path: "second.jar", package_identity_file: "second.identity.json"} +]')" +RELEASE_ARTIFACT_DIRECTORY="${multiple_identity_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="multiple-identities" +RAPIDS_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" +RELEASE_CATALOG_KEY="archive:multiple" +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + (.entries | length == 2) + and .entries[0].package == {ecosystem: "archive", name: "first", version: "1.0"} + and .entries[1].package == {ecosystem: "maven", name: "example:second", version: "2.0"} +' "${multiple_identity_directory}/release-catalog-entries.json" >/dev/null + +generated_directory="${temporary_directory}/generated-bundle" +mkdir -p "${generated_directory}/linux-64" +printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.bin" +jq -n '{ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}' \ + >"${generated_directory}/kvikio.identity.json" + +RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.bin", package_identity_file: "kvikio.identity.json"}]')" +RELEASE_ARTIFACT_DIRECTORY="${generated_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" +RAPIDS_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +RELEASE_CATALOG_KEY="conda:kvikio" +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY + +"${repository_root}/release-catalog/materialize.sh" + +generated_entries_path="${generated_directory}/release-catalog-entries.json" +generated_sbom_path="$(jq -r '.entries[0].sbom' "${generated_entries_path}")" +generated_provenance_path="$(jq -r '.entries[0].provenance' "${generated_entries_path}")" +jq -e ' + .source.artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" + and .source.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + and .entries[0].release_catalog_key == "conda:kvikio" + and .entries[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.bin" + and .entries[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} + and .entries[0].sbom_kind == "generated-identity" +' "${generated_entries_path}" >/dev/null +jq -e ' + .spdxVersion == "SPDX-2.3" + and .packages[0].name == "kvikio" + and .packages[0].versionInfo == "26.08.00a32" +' "${generated_directory}/${generated_sbom_path}" >/dev/null +jq -e ' + .predicateType == "https://slsa.dev/provenance/v1" + and .predicate.buildDefinition.externalParameters.release_catalog_key == "conda:kvikio" + and .predicate.buildDefinition.resolvedDependencies[0].digest.gitCommit == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +' "${generated_directory}/${generated_provenance_path}" >/dev/null +wheel_directory="${temporary_directory}/wheel-bundle" +mkdir -p "${wheel_directory}" +printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32.bin" +jq -n '{ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}' \ + >"${wheel_directory}/libkvikio.identity.json" + +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.bin", package_identity_file: "libkvikio.identity.json"}]')" +RELEASE_ARTIFACT_DIRECTORY="${wheel_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" +RAPIDS_SHA="cccccccccccccccccccccccccccccccccccccccc" +RELEASE_CATALOG_KEY="wheel:kvikio" +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY + +"${repository_root}/release-catalog/materialize.sh" + +jq -e ' + .entries[0].release_catalog_key == "wheel:kvikio" + and .entries[0].path == "libkvikio_cu12-26.8.0a32.bin" + and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} + and .entries[0].sbom_kind == "generated-identity" +' "${wheel_directory}/release-catalog-entries.json" >/dev/null + +missing_version_directory="${temporary_directory}/missing-version-bundle" +mkdir -p "${missing_version_directory}" +printf '%s\n' archive >"${missing_version_directory}/bundle.tar.gz" +jq -n \ + '{ecosystem: "archive", name: "bundle"}' \ + >"${missing_version_directory}/bundle-package-identity.json" + +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "bundle-package-identity.json"}]')" +RELEASE_ARTIFACT_DIRECTORY="${missing_version_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" +RAPIDS_SHA="dddddddddddddddddddddddddddddddddddddddd" +RELEASE_CATALOG_KEY="archive:bundle" +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY + +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then + echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 + exit 1 +fi +grep -Fx 'package identity for bundle.tar.gz must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" + +if (unset RAPIDS_SHA; "${repository_root}/release-catalog/materialize.sh") 2>"${temporary_directory}/missing-rapids-sha-error"; then + echo "materialize.sh unexpectedly accepted a missing RAPIDS_SHA" >&2 + exit 1 +fi +grep -Fx 'RAPIDS_SHA must be a non-empty string' "${temporary_directory}/missing-rapids-sha-error"