diff --git a/.github/workflows/build-and-push-images.yaml b/.github/workflows/build-and-push-images.yaml
index a9526f85d1..d09e01b22f 100644
--- a/.github/workflows/build-and-push-images.yaml
+++ b/.github/workflows/build-and-push-images.yaml
@@ -8,6 +8,7 @@ on:
     tags:
       - "v*"
   pull_request:
+  workflow_dispatch:
 
 jobs:
   build-and-publish:
@@ -16,7 +17,7 @@ jobs:
     runs-on: oracle-vm-16cpu-64gb-x86-64
 
     env:
-      SHOULD_PUBLISH: ${{ github.event_name == 'push' }}
+      SHOULD_PUBLISH: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
 
     strategy:
       fail-fast: false
diff --git a/.github/workflows/check-pr-title.yaml b/.github/workflows/check-pr-title.yaml
index 5cb8380c94..7b504ef19c 100644
--- a/.github/workflows/check-pr-title.yaml
+++ b/.github/workflows/check-pr-title.yaml
@@ -42,3 +42,4 @@ jobs:
           ignoreLabels: |
             do-not-merge/work-in-progress
             dependencies
+            area/release
diff --git a/.github/workflows/check-release.yaml b/.github/workflows/check-release.yaml
new file mode 100644
index 0000000000..dfd7148bf3
--- /dev/null
+++ b/.github/workflows/check-release.yaml
@@ -0,0 +1,129 @@
+name: Check Release
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - VERSION
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  SEMVER_PATTERN: '^(v)?([0-9]+)\.([0-9]+)\.([0-9]+)(-rc\.([0-9]+))?$'
+  CHART_FILE: charts/kubeflow-trainer/Chart.yaml
+  PY_API_VERSION_FILE: api/python_api/kubeflow_trainer_api/__init__.py
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Parse version and export env vars
+        run: |
+          RAW_VERSION=$(cat VERSION | tr -d ' \n\r')
+          VERSION=${RAW_VERSION#v}
+          if [[ ${RAW_VERSION} =~ ${{ env.SEMVER_PATTERN }} ]]; then
+            echo "Version '${RAW_VERSION}' matches semver pattern."
+          else
+            echo "Version '${RAW_VERSION}' does not match semver pattern."
+            exit 1
+          fi
+          TAG="v${VERSION}"
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
+          echo "TAG=${TAG}" >> $GITHUB_ENV
+
+      - name: Check if tag exists
+        run: |
+          git fetch --tags
+          if git tag -l | grep -q "^${TAG}$"; then
+            echo "Tag '${TAG}' already exists."
+            exit 1
+          else
+            echo "Tag '${TAG}' does not exist."
+          fi
+
+      - name: Check if manifests image tag matches version
+        run: |
+          MANIFEST_TAGS=$(grep -r 'newTag:' manifests | sed 's/.*newTag:[[:space:]]*//' | tr -d '"' | tr -d "'" | sort | uniq)
+          if [ -z "$MANIFEST_TAGS" ]; then
+            echo "No newTag found in manifests."
+            exit 1
+          fi
+          for t in $MANIFEST_TAGS; do
+            if [ "$t" != "$TAG" ]; then
+              echo "Image tag in manifests ($t) does not match version tag ($TAG)."
+              exit 1
+            fi
+          done
+          echo "All image tags in manifests match version tag $TAG."
+
+      - name: Check Helm chart version
+        run: |
+          CHART_VERSION=$(grep -E '^version:' "$CHART_FILE" | head -n1 | awk '{print $2}')
+          if [ -z "$CHART_VERSION" ]; then
+            echo "Chart version not found in $CHART_FILE"
+            exit 1
+          fi
+          if [ "$CHART_VERSION" != "$VERSION" ]; then
+            echo "Chart version ($CHART_VERSION) does not match VERSION ($VERSION)."
+            exit 1
+          fi
+          echo "Chart version matches VERSION ($VERSION)."
+
+      - name: Check Python API version
+        run: |
+          PY_VER=$(python - <<'PY'
+          import os
+          import re
+          import sys
+          from pathlib import Path
+          path = Path(os.environ["PY_API_VERSION_FILE"])
+          text = path.read_text()
+          match = re.search(r"__version__\s*=\s*['\"]([^'\"]+)['\"]", text)
+          if not match:
+              print("__version__ not found", file=sys.stderr)
+              sys.exit(1)
+          print(match.group(1))
+          PY
+          )
+          if [ "$PY_VER" != "$VERSION" ]; then
+            echo "Python API version ($PY_VER) does not match VERSION ($VERSION)."
+            exit 1
+          fi
+          echo "Python API version matches VERSION ($VERSION)."
+
+      - name: Check configmap version in manager overlay
+        run: |
+          MANAGER_KUSTOMIZE="manifests/overlays/manager/kustomization.yaml"
+          if [ ! -f "$MANAGER_KUSTOMIZE" ]; then
+            echo "Manager kustomization not found: $MANAGER_KUSTOMIZE"
+            exit 1
+          fi
+          CM_VERSION=$(grep 'kubeflow_trainer_version=' "$MANAGER_KUSTOMIZE" | sed 's/.*kubeflow_trainer_version=//' | tr -d ' \t')
+          if [ -z "$CM_VERSION" ]; then
+            echo "kubeflow_trainer_version not found in $MANAGER_KUSTOMIZE."
+            exit 1
+          fi
+          if [ "$CM_VERSION" != "$TAG" ]; then
+            echo "Configmap version ($CM_VERSION) does not match version tag ($TAG)."
+            exit 1
+          fi
+          echo "Configmap version matches version tag $TAG."
+
+      - name: Check data-cache image is pinned
+        run: |
+          UNPINNED=$(grep -rn 'ghcr\.io/kubeflow/trainer/[A-Za-z0-9._/-]*:latest' manifests || true)
+          if [ -n "$UNPINNED" ]; then
+            echo "Found unpinned :latest image references in manifests:"
+            echo "$UNPINNED"
+            exit 1
+          fi
+          echo "All inline image references in manifests are pinned (no :latest)."
diff --git a/.github/workflows/publish-helm-charts.yaml b/.github/workflows/publish-helm-charts.yaml
index e785139104..4ec29936bb 100644
--- a/.github/workflows/publish-helm-charts.yaml
+++ b/.github/workflows/publish-helm-charts.yaml
@@ -6,6 +6,7 @@ on:
       - master
     tags:
       - "v*"
+  workflow_dispatch:
 
 env:
   CHART_PATH: charts/kubeflow-trainer
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
new file mode 100644
index 0000000000..bf3d88e48f
--- /dev/null
+++ b/.github/workflows/release.yaml
@@ -0,0 +1,178 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - VERSION
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+env:
+  SEMVER_PATTERN: '^(v)?([0-9]+)\.([0-9]+)\.([0-9]+)(-rc\.([0-9]+))?$'
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.vars.outputs.version }}
+      tag: ${{ steps.vars.outputs.tag }}
+      branch: ${{ steps.vars.outputs.branch }}
+      is-prerelease: ${{ steps.vars.outputs.is-prerelease }}
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Validate version and set outputs
+        id: vars
+        run: |
+          RAW_VERSION=$(cat VERSION | tr -d ' \n\r')
+          VERSION=${RAW_VERSION#v}
+          if [[ ! ${RAW_VERSION} =~ ${{ env.SEMVER_PATTERN }} ]]; then
+            echo "Version '${RAW_VERSION}' does not match semver pattern."
+            exit 1
+          fi
+
+          MAJOR_MINOR=$(echo "$VERSION" | cut -d. -f1,2)
+          BRANCH="release-${MAJOR_MINOR}"
+          TAG="v${VERSION}"
+          IS_PRERELEASE=false
+          if [[ ${VERSION} == *"-rc."* ]]; then
+            IS_PRERELEASE=true
+          fi
+
+          echo "Version '${RAW_VERSION}' matches semver pattern."
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+          echo "branch=${BRANCH}" >> $GITHUB_OUTPUT
+          echo "tag=${TAG}" >> $GITHUB_OUTPUT
+          echo "is-prerelease=${IS_PRERELEASE}" >> $GITHUB_OUTPUT
+
+      - name: Ensure tag does not exist
+        run: |
+          git fetch --tags
+          if git tag -l | grep -q "^${{ steps.vars.outputs.tag }}$"; then
+            echo "Tag '${{ steps.vars.outputs.tag }}' already exists."
+            exit 1
+          fi
+
+      - name: Check manifests image tag matches version
+        run: |
+          TAG="${{ steps.vars.outputs.tag }}"
+          MANIFEST_TAGS=$(grep -r 'newTag:' manifests | sed 's/.*newTag:[[:space:]]*//' | tr -d '"' | tr -d "'" | sort | uniq)
+          if [ -z "$MANIFEST_TAGS" ]; then
+            echo "No newTag found in manifests."
+            exit 1
+          fi
+          for t in $MANIFEST_TAGS; do
+            if [ "$t" != "$TAG" ]; then
+              echo "Image tag in manifests ($t) does not match version tag ($TAG)."
+              exit 1
+            fi
+          done
+          echo "All image tags in manifests match version tag $TAG."
+
+  create_branch_and_tag:
+    needs:
+      - prepare
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "GitHub Actions"
+          git config user.email "actions@github.com"
+
+      - name: Create release branch
+        run: |
+          BRANCH="${{ needs.prepare.outputs.branch }}"
+          git fetch origin
+          if git ls-remote --heads origin "$BRANCH" | grep -q "$BRANCH"; then
+            echo "Release branch $BRANCH already exists."
+          else
+            echo "Creating release branch $BRANCH from $GITHUB_SHA"
+            git checkout -b "$BRANCH" "$GITHUB_SHA"
+            git push origin "$BRANCH"
+          fi
+
+      - name: Create and push tag
+        run: |
+          git tag -a "${{ needs.prepare.outputs.tag }}" "$GITHUB_SHA" -m "Kubeflow Trainer ${{ needs.prepare.outputs.tag }}"
+          git push origin "${{ needs.prepare.outputs.tag }}"
+
+  trigger_builds:
+    needs:
+      - prepare
+      - create_branch_and_tag
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+
+    steps:
+      - name: Trigger image build for release tag
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'build-and-push-images.yaml',
+              ref: '${{ needs.prepare.outputs.tag }}',
+            })
+
+      - name: Trigger Helm chart publish for release tag
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'publish-helm-charts.yaml',
+              ref: '${{ needs.prepare.outputs.tag }}',
+            })
+
+  github_release:
+    needs:
+      - prepare
+      - trigger_builds
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Generate changelog
+        id: changelog
+        uses: orhun/git-cliff-action@v4
+        with:
+          config: cliff.toml
+          args: --latest --tag ${{ needs.prepare.outputs.tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: "Kubeflow Trainer ${{ needs.prepare.outputs.tag }}"
+          tag_name: ${{ needs.prepare.outputs.tag }}
+          target_commitish: ${{ github.sha }}
+          prerelease: ${{ needs.prepare.outputs.is-prerelease == 'true' }}
+          draft: false
+          body: ${{ steps.changelog.outputs.content }}
diff --git a/.github/workflows/template-publish-image/action.yaml b/.github/workflows/template-publish-image/action.yaml
index def7a67d61..1673826396 100644
--- a/.github/workflows/template-publish-image/action.yaml
+++ b/.github/workflows/template-publish-image/action.yaml
@@ -82,6 +82,7 @@ runs:
         images: ${{ inputs.image }}
         tags: |
           type=ref,event=tag
+          type=raw,value=${{ github.ref_name }},enable=${{ github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/tags/') }}
           type=raw,value=latest,enable={{is_default_branch}}
           type=sha
 
diff --git a/Makefile b/Makefile
index 7edccf0f78..4ac653ae5f 100644
--- a/Makefile
+++ b/Makefile
@@ -241,3 +241,39 @@ helm-lint: ## Run Helm chart lint test.
 .PHONY: helm-docs
 helm-docs: helm-docs-plugin ## Generates markdown documentation for helm charts from requirements and values files.
 	$(HELM_DOCS) --sort-values-order=file
+
+##@ Release
+
+# Release version (X.Y.Z or X.Y.Z-rc.N)
+VERSION ?=
+GITHUB_TOKEN ?=
+
+.PHONY: release
+release: ## Create a new release and generate changelog.
+	@if [ -z "$(VERSION)" ] || ! echo "$(VERSION)" | grep -E -q '^[0-9]+\.[0-9]+\.[0-9]+$$'; then \
+		echo "ERROR: VERSION must be set in X.Y.Z format. Usage: make release VERSION=X.Y.Z"; \
+		exit 1; \
+	fi
+
+	@if [ -z "$(GITHUB_TOKEN)" ]; then \
+		echo "ERROR: GITHUB_TOKEN is required. Usage: make release VERSION=X.Y.Z GITHUB_TOKEN=<token>"; \
+		exit 1; \
+	fi
+
+	@echo "Fetching upstream tags..."
+	@git fetch --tags https://github.com/kubeflow/trainer.git
+
+	@echo "Generating changelog for v$(VERSION)..."
+	@if ! git-cliff --unreleased --tag "v$(VERSION)" --prepend CHANGELOG.md; then \
+		echo ""; \
+		echo "ERROR: git-cliff failed."; \
+		echo "Ensure git-cliff is installed and GITHUB_TOKEN is valid."; \
+		exit 1; \
+	fi
+
+	@echo "Running release script..."
+	@export GITHUB_TOKEN=$(GITHUB_TOKEN); \
+	./hack/release.sh $(VERSION)
+
+	@echo ""
+	@echo "✅ Release v$(VERSION) prepared successfully."
diff --git a/cliff.toml b/cliff.toml
new file mode 100644
index 0000000000..2b8f8f8195
--- /dev/null
+++ b/cliff.toml
@@ -0,0 +1,87 @@
+[remote.github]
+owner = "kubeflow"
+repo = "trainer"
+
+[changelog]
+body = """
+{%- if version %}
+## [{{ version }}](https://github.com/kubeflow/trainer/releases/tag/{{ version }}) ({{ timestamp | date(format="%Y-%m-%d") }})
+
+This is Kubeflow Trainer {{ version }} release.
+
+```bash
+kubectl apply --server-side -k "https://github.com/kubeflow/trainer.git/manifests/overlays/manager?ref={{ version }}"
+kubectl apply --server-side -k "https://github.com/kubeflow/trainer.git/manifests/overlays/runtimes?ref={{ version }}"
+```
+
+You can now install controller manager with Helm charts 🚀
+
+```bash
+helm install kubeflow-trainer oci://ghcr.io/kubeflow/charts/kubeflow-trainer --version {{ version | trim_start_matches(pat="v") }}
+```
+
+For more information, please see [the Kubeflow Trainer docs](https://www.kubeflow.org/docs/components/trainer/overview/)
+
+{%- else %}
+## [Unreleased]
+{%- endif %}
+
+{%- set group_order = ["🚀 Features", "🐛 Bug Fixes", "⚙️ Miscellaneous Tasks", "⏪ Reverts"] -%}
+
+{%- for group_name in group_order %}
+{%- set group_commits = commits | filter(attribute="group", value=group_name) -%}
+{%- if group_commits | length > 0 %}
+### {{ group_name }}
+
+{% for commit in group_commits | reverse -%}
+{%- set message = commit.message | split(pat="\n") | first | trim -%}
+{%- set parts = message | split(pat=" (#") -%}
+{%- set author = commit.remote.username | default(value=commit.author.name) -%}
+{% if parts | length > 1 and parts | last | trim | split(pat=")") | length > 1 -%}
+{%- set pr_part = parts | last | trim -%}
+{%- set pr_number = pr_part | replace(from=")", to="") -%}
+- {{ parts | slice(end=-1) | join(sep=" (#") }} ([#{{ pr_number }}](https://github.com/kubeflow/trainer/pull/{{ pr_number }}) by @{{ author }})
+{% else -%}
+- {{ message }} (@{{ author }})
+{% endif -%}
+{% endfor %}
+
+{%- endif %}
+{%- endfor %}
+
+{%- if github -%}
+{%- set new_contributors = github.contributors | filter(attribute="is_first_time", value=true) -%}
+{%- if new_contributors | length != 0 %}
+
+### New Contributors
+{%- for contributor in new_contributors %}
+* @{{ contributor.username }} made their first contribution in \
+[#{{ contributor.pr_number }}](https://github.com/kubeflow/trainer/pull/{{ contributor.pr_number }})
+{%- endfor %}
+{%- endif %}
+{%- endif -%}
+
+{% raw %}\n{% endraw %}
+"""
+
+trim = true
+
+footer = ""
+
+[git]
+conventional_commits = false
+filter_unconventional = false
+split_commits = false
+
+# Match stable and RC release tags
+tag_pattern = "^v?[0-9]+\\.[0-9]+\\.[0-9]+(-rc\\.[0-9]+)?$"
+ignore_tags = ".*-(alpha|beta).*"
+
+# Manually define groups based on conventional patterns
+commit_parsers = [
+  { message = "^feat(\\(.*\\))?:", group = "🚀 Features" },
+  { message = "^fix(\\(.*\\))?:", group = "🐛 Bug Fixes" },
+  { message = "^chore(\\(.*\\))?:", group = "⚙️ Miscellaneous Tasks" },
+  { message = "^revert(\\(.*\\))?:", group = "⏪ Reverts" },
+  { message = ".*", skip = true },
+]
diff --git a/docs/release/RELEASE_TESTING.md b/docs/release/RELEASE_TESTING.md
new file mode 100644
index 0000000000..a31ad425fc
--- /dev/null
+++ b/docs/release/RELEASE_TESTING.md
@@ -0,0 +1,246 @@
+# Testing the Release Process on a Fork
+
+This guide walks through testing the full Kubeflow Trainer release pipeline
+using a personal GitHub fork **without** publishing anything to the upstream
+registries or PyPI.
+
+---
+
+## Prerequisites
+
+| Tool | Purpose |
+|------|---------|
+| Git | Branch and tag management |
+| Docker | Changelog generation via git-cliff (local `hack/release.sh`) |
+| GNU Make | Running `make release` |
+| Python 3.11+ | Chart.yaml patching inside `release.sh` |
+| GitHub CLI (`gh`) | Optional — convenient for creating PRs |
+
+You also need a **GitHub personal access token** (classic) with `repo` and
+`workflow` scopes, referred to as `$GITHUB_TOKEN` below.
+
+---
+
+## 1. Fork and Clone
+
+```bash
+# Fork kubeflow/trainer on GitHub, then:
+git clone https://github.com/<you>/trainer.git
+cd trainer
+git remote add upstream https://github.com/kubeflow/trainer.git
+git fetch upstream
+git checkout -b test-release upstream/master
+git push origin test-release
+```
+
+---
+
+## 2. Remove the Repository Guard
+
+The `build-and-push-images.yaml` workflow skips on forks because of:
+
+```yaml
+if: github.repository == 'kubeflow/trainer'
+```
+
+For fork testing, create an override commit **on your fork only**:
+
+```bash
+# .github/workflows/build-and-push-images.yaml
+# Change line 16:
+#   if: github.repository == 'kubeflow/trainer'
+# To:
+#   if: true
+```
+
+> **Do not include this change in any PR back to upstream.**
+
+---
+
+## 3. Set Up the `release` Environment
+
+The `publish_pypi` and `github_release` jobs require a GitHub Environment
+named **`release`**.
+
+1. Go to your fork → **Settings → Environments → New environment**
+2. Name it `release`
+3. No protection rules are needed for testing
+
+---
+
+## 4. Configure PyPI (Optional — Test PyPI)
+
+The `publish_pypi` job uses **OIDC trusted publishing** (no API token).
+To test actual publishing without touching the real PyPI:
+
+1. Register on [Test PyPI](https://test.pypi.org)
+2. Create a trusted publisher for your fork:
+   - Owner: `<your-github-username>`
+   - Repository: `trainer`
+   - Workflow: `release.yaml`
+   - Environment: `release`
+3. Temporarily edit `release.yaml` to point at Test PyPI:
+   ```yaml
+   # In the publish_pypi job, add:
+   - name: Publish to PyPI
+     uses: pypa/gh-action-pypi-publish@release/v1
+     with:
+       print-hash: true
+       packages-dir: dist/
+       repository-url: https://test.pypi.org/legacy/   # ← add this line
+   ```
+
+If you just want to test everything **except** the actual PyPI upload,
+skip this step — the job will fail at the publish step, but all prior
+jobs will still run and validate correctly.
+
+---
+
+## 5. Prepare the Release Commit Locally
+
+Pick a test version that does not collide with existing tags:
+
+```bash
+export GITHUB_TOKEN=ghp_your_token_here
+
+# Standard release:
+make release VERSION=99.0.0
+
+# Or release candidate:
+make release VERSION=99.0.0-rc.1
+```
+
+This runs `hack/release.sh` which:
+- Writes `v99.0.0` to `VERSION`
+- Updates all `newTag:` values in `manifests/` to `v99.0.0`
+- Pins `ghcr.io/kubeflow/trainer/*:latest` images to `v99.0.0`
+- Updates the configmap version in `manifests/overlays/manager/`
+- Sets `Chart.yaml` version to `99.0.0`
+- Sets `api/python_api/kubeflow_trainer_api/__init__.py` to `99.0.0`
+- Generates CHANGELOG.md via git-cliff (requires Docker)
+- Runs `make generate`
+- Creates a signed commit: `Release v99.0.0`
+
+---
+
+## 6. Push and Open a PR
+
+```bash
+git push origin test-release
+
+# Open PR against your fork's master branch
+gh pr create --base master --title "chore(release): Release v99.0.0" \
+  --body "Test release" --repo <you>/trainer
+```
+
+### What to verify on the PR
+
+The **Check Release** workflow (`check-release.yaml`) runs and validates:
+- VERSION matches semver pattern
+- Tag `v99.0.0` does not already exist
+- All `newTag:` values in manifests match `v99.0.0`
+- `Chart.yaml` version matches `99.0.0`
+- Python API `__version__` matches `99.0.0`
+
+All checks must pass before merging.
+
+---
+
+## 7. Merge and Watch the Release Workflow
+
+Merge the PR into your fork's `master`. This triggers `release.yaml`
+(on push to `master` when `VERSION` changes).
+
+### Job execution order
+
+```
+prepare
+  ├─→ build_python_api
+  │     ├─→ create_branch_and_tag
+  │     │     ├─→ trigger_builds  (dispatches image + helm workflows)
+  │     │     └─→ publish_pypi    (OIDC → PyPI)
+  │     │           └─→ github_release (changelog + GitHub Release)
+```
+
+### What each job does
+
+| Job | What to verify |
+|-----|---------------|
+| `prepare` | Version parsed, tag/branch outputs set correctly |
+| `build_python_api` | Package builds, twine check passes, artifact uploaded |
+| `create_branch_and_tag` | Branch `release-99.0` created, tag `v99.0.0` pushed |
+| `trigger_builds` | `build-and-push-images` and `publish-helm-charts` workflows dispatched |
+| `publish_pypi` | OIDC auth works, package published (or fails gracefully on fork) |
+| `github_release` | GitHub Release created with git-cliff changelog |
+
+---
+
+## 8. Verify Dispatched Workflows
+
+After `trigger_builds` runs, check the **Actions** tab for two additional
+workflow runs:
+
+### build-and-push-images
+- Triggered via `workflow_dispatch` with `ref: v99.0.0`
+- Builds all 7 container images
+- On forks (with the guard removed): pushes to `ghcr.io/<you>/trainer/*`
+- Verify the `template-publish-image` action tags images with `v99.0.0`
+
+### publish-helm-charts
+- Triggered via `workflow_dispatch` with `ref: v99.0.0`
+- Reads `Chart.yaml` version (should be `99.0.0` since ref is the tag)
+- Packages `kubeflow-trainer-99.0.0.tgz`
+- Pushes to `oci://ghcr.io/<you>/charts`
+
+---
+
+## 9. Validate the GitHub Release
+
+Go to your fork's **Releases** page and confirm:
+- Release named `Kubeflow Trainer v99.0.0`
+- Tag: `v99.0.0`
+- Body contains the git-cliff changelog for **only** the latest release
+- `prerelease` is `false` for stable, `true` for `-rc.N`
+
+---
+
+## 10. Cleanup
+
+```bash
+# Delete the test tag and release branch from your fork
+git push origin --delete v99.0.0
+git push origin --delete release-99.0
+
+# Delete the GitHub Release via the UI or:
+gh release delete v99.0.0 --repo <you>/trainer --yes
+
+# Reset your master branch
+git checkout master
+git reset --hard upstream/master
+git push origin master --force
+```
+
+---
+
+## Troubleshooting
+
+| Symptom | Cause | Fix |
+|---------|-------|-----|
+| `build-and-push-images` skipped | Repository guard `github.repository == 'kubeflow/trainer'` | See Step 2 |
+| `publish_pypi` fails with 403 | OIDC trusted publisher not configured for your fork | See Step 4, or ignore — prior jobs still validate |
+| `trigger_builds` fails with 403 | `actions: write` permission missing | Ensure `GITHUB_TOKEN` has `workflow` scope in fork settings |
+| `github_release` body is empty | git-cliff found no conventional commits | Ensure commits use `feat:`, `fix:`, `chore:` prefixes |
+| `release.sh` crashes on `GITHUB_TOKEN` | Unset token with `set -o nounset` | Export: `export GITHUB_TOKEN=ghp_...` |
+| `check-release` fails on Chart version | `release.sh` didn't run or was run with wrong version | Re-run `make release VERSION=X.Y.Z` |
+
+---
+
+## Notes
+
+- The `release.yaml` workflow uses **OIDC trusted publishing** for PyPI —
+  no `PYPI_API_TOKEN` secret is needed. The GitHub environment `release`
+  must match what is configured as a trusted publisher on PyPI.
+- The `--latest` flag on git-cliff ensures only the current release's
+  changelog appears in the GitHub Release body, not the full history.
+- On forks, image pushes go to `ghcr.io/<you>/trainer/*` (GHCR
+  auto-scopes to the repository owner).
diff --git a/hack/release.sh b/hack/release.sh
new file mode 100644
index 0000000000..661d41ef1e
--- /dev/null
+++ b/hack/release.sh
@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This shell is used to prepare a release commit for X.Y.Z version.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [ "$#" -eq 0 ]; then
+  echo "Usage: $0 <version>"
+  echo "You must follow this format: X.Y.Z or X.Y.Z-rc.N"
+  exit 1
+fi
+
+NEW_VERSION=$(echo "$1" | tr -d '\n' | tr -d ' ')
+
+if [[ ! "$NEW_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then
+  echo "Version format is invalid. Use: X.Y.Z or X.Y.Z-rc.N"
+  exit 1
+fi
+
+TAG="v$NEW_VERSION"
+
+REPO_ROOT="$(dirname "$0")/.."
+VERSION_FILE="$REPO_ROOT/VERSION"
+MANIFESTS_DIR="$REPO_ROOT/manifests"
+CHART_DIR="$REPO_ROOT/charts/kubeflow-trainer"
+CHART_FILE="$CHART_DIR/Chart.yaml"
+PYTHON_API_VERSION_FILE="$REPO_ROOT/api/python_api/kubeflow_trainer_api/__init__.py"
+
+# Verify tag doesn't already exist
+git fetch --tags
+if git tag --list | grep -q "^${TAG}$"; then
+  echo "Tag: ${TAG} already exists. Release can't be published."
+  exit 1
+fi
+
+echo -e "\nPreparing release commit for ${TAG}\n"
+
+echo -n "v$NEW_VERSION" > "$VERSION_FILE"
+echo "Updated VERSION file to $NEW_VERSION"
+
+# Update image tags in manifests
+find "$MANIFESTS_DIR" -type f -name '*.yaml' -exec sed -i "s/newTag: .*/newTag: $TAG/" {} +
+echo "Updated image tags in manifests to $TAG"
+
+echo "Pinning ghcr.io image references in manifests to $TAG"
+CHANGED_FILES=$(grep -REl "ghcr\.io/kubeflow/trainer/[A-Za-z0-9._/-]+:latest" "$MANIFESTS_DIR" || true)
+if [ -n "$CHANGED_FILES" ]; then
+  while IFS= read -r f; do
+    sed -i -E "s|(ghcr\.io/kubeflow/trainer/[A-Za-z0-9._/-]+):latest|\\1:${TAG}|g" "$f"
+    echo "  Updated ${f#$MANIFESTS_DIR/}"
+  done <<< "$CHANGED_FILES"
+else
+  echo "  No ghcr.io references pinned to :latest found."
+fi
+
+# Update configmap version in manager overlay
+MANAGER_KUSTOMIZE="$MANIFESTS_DIR/overlays/manager/kustomization.yaml"
+if [ -f "$MANAGER_KUSTOMIZE" ]; then
+  sed -i "s/kubeflow_trainer_version=.*/kubeflow_trainer_version=$TAG/" "$MANAGER_KUSTOMIZE"
+  echo "Updated configmap version in manager overlay to $TAG"
+fi
+
+if [ ! -f "$CHART_FILE" ]; then
+  echo "Helm chart file not found: $CHART_FILE"
+  exit 1
+fi
+
+python3 - "$CHART_FILE" "$NEW_VERSION" <<'PYTHON'
+import pathlib
+import re
+import sys
+
+chart_path = pathlib.Path(sys.argv[1])
+new_version = sys.argv[2]
+data = chart_path.read_text()
+pattern = re.compile(r"^version:\s*.+$", re.MULTILINE)
+
+if not pattern.search(data):
+    print("Unable to locate version field in chart file.")
+    sys.exit(1)
+
+chart_path.write_text(pattern.sub(f"version: {new_version}", data, count=1))
+PYTHON
+echo "Updated Helm chart version to $NEW_VERSION"
+
+# Update Python API version
+sed -i "s/__version__ = \".*\"/__version__ = \"$NEW_VERSION\"/" "$PYTHON_API_VERSION_FILE"
+echo "Updated Python API version to $NEW_VERSION"
+
+CHANGELOG_PATH="$REPO_ROOT/CHANGELOG.md"
+echo "Generating changelog for $TAG"
+ABSOLUTE_REPO_ROOT="$(cd "$REPO_ROOT" && pwd)"
+if [ -z "${GITHUB_TOKEN:-}" ]; then
+  echo "WARNING: GITHUB_TOKEN not set. Set it to avoid GitHub API rate limits."
+  echo "Export GITHUB_TOKEN before running this script: export GITHUB_TOKEN=your_token"
+fi
+
+# Generate and prepend new changelog section
+TEMP_FILE=$(mktemp)
+docker run --rm -u "$(id -u):$(id -g)" -v "$ABSOLUTE_REPO_ROOT:/app" \
+  -e "GITHUB_TOKEN=${GITHUB_TOKEN:-}" -w /app \
+  "ghcr.io/orhun/git-cliff/git-cliff:latest" --unreleased --tag "$TAG" -o - > "$TEMP_FILE"
+
+if [ -f "$CHANGELOG_PATH" ]; then
+  sed -i "1 r $TEMP_FILE" "$CHANGELOG_PATH"
+else
+  { echo "# Changelog"; cat "$TEMP_FILE"; } > "$CHANGELOG_PATH"
+fi
+rm "$TEMP_FILE"
+echo "Changelog generated at $CHANGELOG_PATH"
+
+echo "Running make generate"
+make -C "$REPO_ROOT" generate
+echo "Completed make generate"
+
+git add "$VERSION_FILE" "$MANIFESTS_DIR" "$CHART_DIR" "$REPO_ROOT/api" "$REPO_ROOT/pkg/apis" "$REPO_ROOT/pkg/client" "$CHANGELOG_PATH"
+git commit -s -m "chore(release): Release $TAG"
+
+echo -e "\nRelease commit for $TAG created successfully."
+echo "Next steps:"
+echo "  1. Push your branch to your fork"
+echo "  2. Open a PR against master (add the 'area/release' label to skip PR title check)"
+echo "  3. Once merged, GitHub Actions will create the release branch and tag"