chore: automate release process with GitHub Actions

.github/workflows/build-and-push-images.yaml

@@ -8,6 +8,7 @@ on:
     tags:
       - "v*"
   pull_request:
+  workflow_dispatch:
 
 jobs:
   build-and-publish:
@@ -16,7 +17,7 @@ jobs:
     runs-on: oracle-vm-16cpu-64gb-x86-64
 
     env:
-      SHOULD_PUBLISH: ${{ github.event_name == 'push' }}
+      SHOULD_PUBLISH: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
 
     strategy:
       fail-fast: false

.github/workflows/check-pr-title.yaml

@@ -42,3 +42,4 @@ jobs:
           ignoreLabels: |
             do-not-merge/work-in-progress
             dependencies
+            area/release

.github/workflows/check-release.yaml

@@ -0,0 +1,129 @@
+name: Check Release
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - VERSION
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  SEMVER_PATTERN: '^(v)?([0-9]+)\.([0-9]+)\.([0-9]+)(-rc\.([0-9]+))?$'
+  CHART_FILE: charts/kubeflow-trainer/Chart.yaml
+  PY_API_VERSION_FILE: api/python_api/kubeflow_trainer_api/__init__.py
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Parse version and export env vars
+        run: |
+          RAW_VERSION=$(cat VERSION | tr -d ' \n\r')
+          VERSION=${RAW_VERSION#v}
+          if [[ ${RAW_VERSION} =~ ${{ env.SEMVER_PATTERN }} ]]; then
+            echo "Version '${RAW_VERSION}' matches semver pattern."
+          else
+            echo "Version '${RAW_VERSION}' does not match semver pattern."
+            exit 1
+          fi
+          TAG="v${VERSION}"
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
+          echo "TAG=${TAG}" >> $GITHUB_ENV
+
+      - name: Check if tag exists
+        run: |
+          git fetch --tags
+          if git tag -l | grep -q "^${TAG}$"; then
+            echo "Tag '${TAG}' already exists."
+            exit 1
+          else
+            echo "Tag '${TAG}' does not exist."
+          fi
+
+      - name: Check if manifests image tag matches version
+        run: |
+          MANIFEST_TAGS=$(grep -r 'newTag:' manifests | sed 's/.*newTag:[[:space:]]*//' | tr -d '"' | tr -d "'" | sort | uniq)
+          if [ -z "$MANIFEST_TAGS" ]; then
+            echo "No newTag found in manifests."
+            exit 1
+          fi
+          for t in $MANIFEST_TAGS; do
+            if [ "$t" != "$TAG" ]; then
+              echo "Image tag in manifests ($t) does not match version tag ($TAG)."
+              exit 1
+            fi
+          done
+          echo "All image tags in manifests match version tag $TAG."
+
+      - name: Check Helm chart version
+        run: |
+          CHART_VERSION=$(grep -E '^version:' "$CHART_FILE" | head -n1 | awk '{print $2}')
+          if [ -z "$CHART_VERSION" ]; then
+            echo "Chart version not found in $CHART_FILE"
+            exit 1
+          fi
+          if [ "$CHART_VERSION" != "$VERSION" ]; then
+            echo "Chart version ($CHART_VERSION) does not match VERSION ($VERSION)."
+            exit 1
+          fi
+          echo "Chart version matches VERSION ($VERSION)."
+
+      - name: Check Python API version
+        run: |
+          PY_VER=$(python - <<'PY'
+          import os
+          import re
+          import sys
+          from pathlib import Path
+          path = Path(os.environ["PY_API_VERSION_FILE"])
+          text = path.read_text()
+          match = re.search(r"__version__\s*=\s*['\"]([^'\"]+)['\"]", text)
+          if not match:
+              print("__version__ not found", file=sys.stderr)
+              sys.exit(1)
+          print(match.group(1))
+          PY
+          )
+          if [ "$PY_VER" != "$VERSION" ]; then
+            echo "Python API version ($PY_VER) does not match VERSION ($VERSION)."
+            exit 1
+          fi
+          echo "Python API version matches VERSION ($VERSION)."
+
+      - name: Check configmap version in manager overlay
+        run: |
+          MANAGER_KUSTOMIZE="manifests/overlays/manager/kustomization.yaml"
+          if [ ! -f "$MANAGER_KUSTOMIZE" ]; then
+            echo "Manager kustomization not found: $MANAGER_KUSTOMIZE"
+            exit 1
+          fi
+          CM_VERSION=$(grep 'kubeflow_trainer_version=' "$MANAGER_KUSTOMIZE" | sed 's/.*kubeflow_trainer_version=//' | tr -d ' \t')
+          if [ -z "$CM_VERSION" ]; then
+            echo "kubeflow_trainer_version not found in $MANAGER_KUSTOMIZE."
+            exit 1
+          fi
+          if [ "$CM_VERSION" != "$TAG" ]; then
+            echo "Configmap version ($CM_VERSION) does not match version tag ($TAG)."
+            exit 1
+          fi
+          echo "Configmap version matches version tag $TAG."
+
+      - name: Check data-cache image is pinned
+        run: |
+          UNPINNED=$(grep -rn 'ghcr\.io/kubeflow/trainer/[A-Za-z0-9._/-]*:latest' manifests || true)
+          if [ -n "$UNPINNED" ]; then
+            echo "Found unpinned :latest image references in manifests:"
+            echo "$UNPINNED"
+            exit 1
+          fi
+          echo "All inline image references in manifests are pinned (no :latest)."

.github/workflows/publish-helm-charts.yaml

@@ -6,6 +6,7 @@ on:
       - master
     tags:
       - "v*"
+  workflow_dispatch:
 
 env:
   CHART_PATH: charts/kubeflow-trainer

.github/workflows/release.yaml

@@ -0,0 +1,178 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - VERSION
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+env:
+  SEMVER_PATTERN: '^(v)?([0-9]+)\.([0-9]+)\.([0-9]+)(-rc\.([0-9]+))?$'
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.vars.outputs.version }}
+      tag: ${{ steps.vars.outputs.tag }}
+      branch: ${{ steps.vars.outputs.branch }}
+      is-prerelease: ${{ steps.vars.outputs.is-prerelease }}
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Validate version and set outputs
+        id: vars
+        run: |
+          RAW_VERSION=$(cat VERSION | tr -d ' \n\r')
+          VERSION=${RAW_VERSION#v}
+          if [[ ! ${RAW_VERSION} =~ ${{ env.SEMVER_PATTERN }} ]]; then
+            echo "Version '${RAW_VERSION}' does not match semver pattern."
+            exit 1
+          fi
+
+          MAJOR_MINOR=$(echo "$VERSION" | cut -d. -f1,2)
+          BRANCH="release-${MAJOR_MINOR}"
+          TAG="v${VERSION}"
+          IS_PRERELEASE=false
+          if [[ ${VERSION} == *"-rc."* ]]; then
+            IS_PRERELEASE=true
+          fi
+
+          echo "Version '${RAW_VERSION}' matches semver pattern."
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+          echo "branch=${BRANCH}" >> $GITHUB_OUTPUT
+          echo "tag=${TAG}" >> $GITHUB_OUTPUT
+          echo "is-prerelease=${IS_PRERELEASE}" >> $GITHUB_OUTPUT
+
+      - name: Ensure tag does not exist
+        run: |
+          git fetch --tags
+          if git tag -l | grep -q "^${{ steps.vars.outputs.tag }}$"; then
+            echo "Tag '${{ steps.vars.outputs.tag }}' already exists."
+            exit 1
+          fi
+
+      - name: Check manifests image tag matches version
+        run: |
+          TAG="${{ steps.vars.outputs.tag }}"
+          MANIFEST_TAGS=$(grep -r 'newTag:' manifests | sed 's/.*newTag:[[:space:]]*//' | tr -d '"' | tr -d "'" | sort | uniq)
+          if [ -z "$MANIFEST_TAGS" ]; then
+            echo "No newTag found in manifests."
+            exit 1
+          fi
+          for t in $MANIFEST_TAGS; do
+            if [ "$t" != "$TAG" ]; then
+              echo "Image tag in manifests ($t) does not match version tag ($TAG)."
+              exit 1
+            fi
+          done
+          echo "All image tags in manifests match version tag $TAG."
+
+  create_branch_and_tag:
+    needs:
+      - prepare
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "GitHub Actions"
+          git config user.email "actions@github.com"
+
+      - name: Create release branch
+        run: |
+          BRANCH="${{ needs.prepare.outputs.branch }}"
+          git fetch origin
+          if git ls-remote --heads origin "$BRANCH" | grep -q "$BRANCH"; then
+            echo "Release branch $BRANCH already exists."
+          else
+            echo "Creating release branch $BRANCH from $GITHUB_SHA"
+            git checkout -b "$BRANCH" "$GITHUB_SHA"
+            git push origin "$BRANCH"
+          fi
+
+      - name: Create and push tag
+        run: |
+          git tag -a "${{ needs.prepare.outputs.tag }}" "$GITHUB_SHA" -m "Kubeflow Trainer ${{ needs.prepare.outputs.tag }}"
+          git push origin "${{ needs.prepare.outputs.tag }}"
+
+  trigger_builds:
+    needs:
+      - prepare
+      - create_branch_and_tag
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+
+    steps:
+      - name: Trigger image build for release tag
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'build-and-push-images.yaml',
+              ref: '${{ needs.prepare.outputs.tag }}',
+            })
+
+      - name: Trigger Helm chart publish for release tag
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'publish-helm-charts.yaml',
+              ref: '${{ needs.prepare.outputs.tag }}',
+            })
+
+  github_release:
+    needs:
+      - prepare
+      - trigger_builds
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Generate changelog
+        id: changelog
+        uses: orhun/git-cliff-action@v4
+        with:
+          config: cliff.toml
+          args: --latest --tag ${{ needs.prepare.outputs.tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: "Kubeflow Trainer ${{ needs.prepare.outputs.tag }}"
+          tag_name: ${{ needs.prepare.outputs.tag }}
+          target_commitish: ${{ github.sha }}
+          prerelease: ${{ needs.prepare.outputs.is-prerelease == 'true' }}
+          draft: false
+          body: ${{ steps.changelog.outputs.content }}

.github/workflows/template-publish-image/action.yaml

@@ -82,6 +82,7 @@ runs:
         images: ${{ inputs.image }}
         tags: |
           type=ref,event=tag
+          type=raw,value=${{ github.ref_name }},enable=${{ github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/tags/') }}
           type=raw,value=latest,enable={{is_default_branch}}
           type=sha
 

Makefile

@@ -241,3 +241,39 @@ helm-lint: ## Run Helm chart lint test.
 .PHONY: helm-docs
 helm-docs: helm-docs-plugin ## Generates markdown documentation for helm charts from requirements and values files.
 	$(HELM_DOCS) --sort-values-order=file
+
+##@ Release
+
+# Release version (X.Y.Z or X.Y.Z-rc.N)
+VERSION ?=
+GITHUB_TOKEN ?=
+
+.PHONY: release
+release: ## Create a new release and generate changelog.
+	@if [ -z "$(VERSION)" ] || ! echo "$(VERSION)" | grep -E -q '^[0-9]+\.[0-9]+\.[0-9]+$$'; then \
+		echo "ERROR: VERSION must be set in X.Y.Z format. Usage: make release VERSION=X.Y.Z"; \
+		exit 1; \
+	fi
+
+	@if [ -z "$(GITHUB_TOKEN)" ]; then \
+		echo "ERROR: GITHUB_TOKEN is required. Usage: make release VERSION=X.Y.Z GITHUB_TOKEN=<token>"; \
+		exit 1; \
+	fi
+
+	@echo "Fetching upstream tags..."
+	@git fetch --tags https://github.com/kubeflow/trainer.git
+
+	@echo "Generating changelog for v$(VERSION)..."
+	@if ! git-cliff --unreleased --tag "v$(VERSION)" --prepend CHANGELOG.md; then \
+		echo ""; \
+		echo "ERROR: git-cliff failed."; \
+		echo "Ensure git-cliff is installed and GITHUB_TOKEN is valid."; \
+		exit 1; \
+	fi
+
+	@echo "Running release script..."
+	@export GITHUB_TOKEN=$(GITHUB_TOKEN); \
+	./hack/release.sh $(VERSION)
+
+	@echo ""
+	@echo "✅ Release v$(VERSION) prepared successfully."