diff --git a/docs/sdk-regeneration-pipeline.md b/docs/sdk-regeneration-pipeline.md index b921e8c615..d9e139ee35 100644 --- a/docs/sdk-regeneration-pipeline.md +++ b/docs/sdk-regeneration-pipeline.md @@ -37,6 +37,7 @@ The SDK regeneration pipeline automates the process of updating Azure SDK packag #### 4. Results & PR Creation - Generates results report (`regenerate-sdk-result.json`) - Creates draft PR in azure-sdk-for-go with all changes, PR title: `[Automation] Regenerate SDK based on typespec-go branch {branch-name}` +- If `CreateSpecPR` is enabled and a module version bump was detected, creates a draft PR in azure-rest-api-specs to bump the go module suffix in `tspconfig.yaml` ### Pipeline Parameters @@ -47,6 +48,7 @@ The SDK regeneration pipeline automates the process of updating Azure SDK packag | `UseLatestSpec` | boolean | `false` | Whether to use the latest API specifications from [azure-rest-api-specs](https://github.com/Azure/azure-rest-api-specs) or the original commit of `tsp-location.yml` | | `ServiceFilter` | string | `.*` | Regex pattern to filter which services to regenerate. Matches against the service package name (e.g., `armcompute`, `armstorage`) | | `UseDevPackage` | boolean | `false` | Whether to use dev package (.tgz) from current branch or the recent released package from npm registry | +| `CreateSpecPR` | boolean | `false` | Whether to create a PR in azure-rest-api-specs to bump go module suffixes in `tspconfig.yaml` when module versions are detected | #### Usage Examples @@ -126,6 +128,9 @@ You can find the generated SDK pull request link from pipeline logs "succeed_to_regenerate": ["package1", "package2"], "fail_to_regenerate": ["package3"], "not_found_api_version": ["package4"], + "module_version_changed": { + "specification/advisor/Advisor.Management": "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/advisor/armadvisor/v2" + }, "time_to_regenerate": "2025-09-25 10:30:45.123456", "typespec_go_commit_hash": "abc123def456789..." } @@ -161,7 +166,7 @@ After each TypeSpec or Go emitter version release: 3. **Merge Regeneration PR**: Merge the refresh PR to keep SDK up-to-date ### Quality Gates -- All pipelines must be pass +- All pipelines must pass - All SDK code changes are made by either TypeSpec changes or Go emitter changes - Module versions must not be changed - API versions must not be changed diff --git a/eng/pipelines/sdk-regenerate.yml b/eng/pipelines/sdk-regenerate.yml index cb09c3df76..75b9d7b3c9 100644 --- a/eng/pipelines/sdk-regenerate.yml +++ b/eng/pipelines/sdk-regenerate.yml @@ -18,6 +18,9 @@ parameters: - name: UseDevPackage default: false type: boolean +- name: CreateSpecPR + default: false + type: boolean variables: - template: /eng/pipelines/templates/variables/globals.yml @@ -35,6 +38,11 @@ resources: name: Azure/azure-sdk-for-go endpoint: azure ref: main + - repository: azure-rest-api-specs + type: github + name: Azure/azure-rest-api-specs + endpoint: azure + ref: main jobs: - job: Generate_SDK @@ -45,6 +53,8 @@ jobs: - checkout: self fetchDepth: 1 - checkout: azure-sdk-for-go + - ${{ if eq(parameters.CreateSpecPR, true) }}: + - checkout: azure-rest-api-specs - task: NodeTool@0 displayName: 'Install Node.js $(NodeVersion)' @@ -73,10 +83,14 @@ jobs: displayName: 'Install tsp-client' - script: | - python3 $(Build.SourcesDirectory)/autorest.go/eng/scripts/sdk_regenerate.py --sdk-root=$(Build.SourcesDirectory)/azure-sdk-for-go --typespec-go-root=$(Build.SourcesDirectory)/autorest.go/packages/typespec-go --typespec-go-branch=$(Build.SourceBranchName) --use-latest-spec=${{ parameters.UseLatestSpec }} --service-filter="${{ parameters.ServiceFilter }}" --use-dev-package=${{ parameters.UseDevPackage }} + python3 $(Build.SourcesDirectory)/autorest.go/eng/scripts/sdk_regenerate.py --sdk-root=$(Build.SourcesDirectory)/azure-sdk-for-go --typespec-go-root=$(Build.SourcesDirectory)/autorest.go/packages/typespec-go --typespec-go-branch=$(Build.SourceBranchName) --use-latest-spec=${{ parameters.UseLatestSpec }} --service-filter="${{ parameters.ServiceFilter }}" --use-dev-package=${{ parameters.UseDevPackage }} --create-spec-pr=${{ parameters.CreateSpecPR }} --spec-root=$(Build.SourcesDirectory)/azure-rest-api-specs displayName: 'Generate SDK' workingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-go + - publish: $(Build.ArtifactStagingDirectory)/regenerate-sdk-result.json + artifact: regenerate-sdk-result + displayName: 'Publish regenerate SDK result' + - template: /eng/common/pipelines/templates/steps/login-to-github.yml@azure-sdk-for-go parameters: ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-for-go/eng/common/scripts @@ -94,3 +108,18 @@ jobs: OpenAsDraft: 'true' PushArgs: '--force' AuthToken: $(GH_TOKEN) + + - ${{ if parameters.CreateSpecPR }}: + - template: /eng/common/pipelines/templates/steps//create-pull-request.yml@azure-sdk-for-go + parameters: + WorkingDirectory: $(Build.SourcesDirectory)/azure-rest-api-specs + ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-for-go/eng/common/scripts + RepoName: azure-rest-api-specs + PROwner: Azure + BaseBranchName: 'refs/heads/main' + PRBranchName: typespec-go-module-suffix-$(Build.SourceBranchName) + CommitMsg: 'Bump go module suffix in tspconfig.yaml' + PRTitle: '[Automation] Bump go module suffix in tspconfig.yaml' + OpenAsDraft: 'true' + PushArgs: '--force' + AuthToken: $(GH_TOKEN) diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index ea301907de..a6c4c3727e 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -4,7 +4,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from pathlib import Path import subprocess from datetime import datetime @@ -12,6 +12,7 @@ import argparse import logging import json +import os import re import glob import urllib.request @@ -165,22 +166,52 @@ def update_commit_id(file: Path, commit_id: str): def get_api_version_from_metadata(package_folder: Path) -> Optional[str]: - """Extract API version from metadata.json file if it exists.""" + """Return the api-version emitter option value from metadata.json, if any. + + Supports the legacy ``{"apiVersion": "..."}`` string and the current + ``{"apiVersions": {namespace: version}}`` map. Returns a plain version when + all namespaces share one version, otherwise a compact JSON namespace->version + map that the emitter uses to regenerate each namespace at its own version. + """ # Construct the metadata.json path based on the package folder structure # {package_folder}/testdata/_metadata.json metadata_path = package_folder / "testdata" / "_metadata.json" - - if metadata_path.exists(): - try: - with open(metadata_path, "r") as f: - metadata = json.load(f) - api_version = metadata.get("apiVersion") - if api_version: - logging.info(f"Found API version {api_version} in metadata.json for {package_folder.name}") - return api_version - except (json.JSONDecodeError, FileNotFoundError) as e: - logging.warning(f"Failed to read metadata.json for {package_folder.name}: {e}") - + + if not metadata_path.exists(): + return None + + try: + with open(metadata_path, "r") as f: + metadata = json.load(f) + except (json.JSONDecodeError, FileNotFoundError) as e: + logging.warning(f"Failed to read metadata.json for {package_folder.name}: {e}") + return None + + # current format: a map of namespace -> API version + api_versions = metadata.get("apiVersions") + if isinstance(api_versions, dict): + # drop namespaces without a version, keeping the namespace->version map + versions = {ns: ver for ns, ver in api_versions.items() if ver} + if versions: + distinct = set(versions.values()) + if len(distinct) == 1: + api_version = next(iter(distinct)) + logging.info(f"Found API version {api_version} in metadata.json for {package_folder.name}") + return api_version + # multiple namespaces at different versions: pass the whole map so the + # emitter regenerates each service namespace at its recorded version + api_version = json.dumps(versions, separators=(",", ":")) + logging.info( + f"Found multiple API versions {versions} in metadata.json for " + f"{package_folder.name}, passing per-namespace versions" + ) + return api_version + + # legacy format: a single API version string + api_version = metadata.get("apiVersion") + if api_version: + logging.info(f"Found API version {api_version} in metadata.json for {package_folder.name}") + return api_version return None @@ -223,13 +254,65 @@ def get_api_version(package_folder: Path) -> Optional[str]: return api_version -def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, typespec_go_root: str) -> Dict[str, List[str]]: + +def get_module_name(package_folder: Path) -> Optional[str]: + """Read the module name from go.mod in the package folder.""" + go_mod_path = package_folder / "go.mod" + if not go_mod_path.exists(): + return None + try: + with open(go_mod_path, "r", encoding="utf-8") as f: + for line in f: + stripped = line.strip() + if stripped.startswith("module "): + return stripped[len("module "):].strip() + except FileNotFoundError as e: + logging.warning(f"Failed to read go.mod for {package_folder.name}: {e}") + return None + + +def detect_module_version_change(package_folder: Path, original_module: str) -> Optional[str]: + """Detect if regen dropped the module version suffix. If so, revert all changes in the + package and return the original suffixed module so the spec PR can bump tspconfig.""" + if not original_module: + return None + suffix_match = re.search(r"/v\d+$", original_module) + if not suffix_match: + return None + current_module = get_module_name(package_folder) + if current_module == original_module: + return None + # Module suffix was dropped during regen; revert the whole package and bump the spec instead. + check_call(["git", "checkout", "--", str(package_folder)]) + logging.info(f"Reverted {package_folder.name}; module version bump will be handled in spec repo") + return original_module + + +def get_spec_directory(package_folder: Path) -> Optional[str]: + """Read the spec repo directory (containing tspconfig.yaml) from tsp-location.yaml.""" + tsp_location = package_folder / "tsp-location.yaml" + if not tsp_location.exists(): + return None + try: + with open(tsp_location, "r", encoding="utf-8") as f: + for line in f: + stripped = line.strip() + if stripped.startswith("directory:"): + return stripped[len("directory:"):].strip().strip('"') + except FileNotFoundError as e: + logging.warning(f"Failed to read tsp-location.yaml for {package_folder.name}: {e}") + return None + + + +def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, typespec_go_root: str) -> Dict[str, Any]: result = { - "succeed_to_regenerate": [], - "fail_to_regenerate": [], - "not_found_api_version": [], + "succeed_to_regenerate": [], + "fail_to_regenerate": [], + "not_found_api_version": [], + "module_version_changed": {}, "time_to_regenerate": str(datetime.now()), - "typespec_go_commit_hash": get_typespec_go_commit_hash(typespec_go_root) + "typespec_go_commit_hash": get_typespec_go_commit_hash(typespec_go_root), } # get all tsp-location.yaml commit_id = get_latest_commit_id() @@ -242,6 +325,8 @@ def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, ty if use_latest_spec: logging.info("Using latest spec") update_commit_id(item, commit_id) + # Record the original module name so it is not changed by regeneration + original_module = get_module_name(package_folder) try: # Get API version for this package api_version = get_api_version(package_folder) @@ -299,7 +384,14 @@ def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, ty else: logging.info(f"Successfully regenerated {package_folder.name}") result["succeed_to_regenerate"].append(package_folder.name) - + finally: + # If regen dropped the module version suffix, revert the package and bump the spec instead + if original_module: + bumped_module = detect_module_version_change(package_folder, original_module) + if bumped_module: + spec_directory = get_spec_directory(package_folder) + if spec_directory: + result["module_version_changed"][spec_directory] = bumped_module result["succeed_to_regenerate"].sort() result["fail_to_regenerate"].sort() result["not_found_api_version"].sort() @@ -329,7 +421,56 @@ def git_add(): check_call("git add .", shell=True) -def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_latest_spec: bool, service_filter: str, use_dev_package: bool): +def bump_tspconfig_module(spec_root: str, spec_directory: str, bumped_module: str) -> bool: + """Update only the go module version suffix in tspconfig.yaml. + + Returns True if the file was changed. + """ + tspconfig_path = Path(spec_root) / spec_directory / "tspconfig.yaml" + if not tspconfig_path.exists(): + logging.warning(f"tspconfig.yaml not found at {tspconfig_path}") + return False + suffix_match = re.search(r"/(v\d+)$", bumped_module) + new_suffix = f"/{suffix_match.group(1)}" if suffix_match else "" + with open(tspconfig_path, "r", encoding="utf-8") as f: + content = f.readlines() + changed = False + for idx in range(len(content)): + match = re.match(r"^(\s*module:\s*\"?)(\S+?)(/v\d+)?(\"?\s*)$", content[idx]) + if match: + updated = f"{match.group(1)}{match.group(2)}{new_suffix}{match.group(4)}" + if updated != content[idx]: + content[idx] = updated + changed = True + logging.info(f"Updated module suffix in {tspconfig_path} to '{new_suffix}'") + break + if changed: + with open(tspconfig_path, "w", encoding="utf-8") as f: + f.writelines(content) + return changed + + +def apply_tspconfig_module_bumps(spec_root: str, module_version_changed: dict) -> bool: + """Bump go module suffixes in tspconfig.yaml for changed packages. + + Returns True if any tspconfig.yaml was modified. The pipeline is responsible for + committing and opening the PR against the spec repo. + """ + if not module_version_changed: + logging.info("No module version changes; nothing to update in spec repo") + return False + if not spec_root or not Path(spec_root).exists(): + logging.warning("spec-root not provided or does not exist; skipping tspconfig update") + return False + + changed_any = False + for spec_directory, bumped_module in module_version_changed.items(): + if bump_tspconfig_module(spec_root, spec_directory, bumped_module): + changed_any = True + return changed_any + + +def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_latest_spec: bool, service_filter: str, use_dev_package: bool, create_spec_pr_flag: bool, spec_root: str): # Configure logging for better pipeline visibility logging.basicConfig( level=logging.INFO, @@ -342,10 +483,24 @@ def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_late prepare_branch(typespec_go_branch) update_emitter_package(sdk_root, typespec_go_root, use_dev_package) result = regenerate_sdk(use_latest_spec, service_filter, sdk_root, typespec_go_root) - with open("regenerate-sdk-result.json", "w") as f: - json.dump(result, f, indent=2) + + # Print the result instead of committing it to the repo + result_json = json.dumps(result, indent=2) + logging.info("Regenerate SDK result:\n%s", result_json) + + # Write the result to the artifact staging directory so it can be published as a pipeline artifact + staging_dir = os.environ.get("BUILD_ARTIFACTSTAGINGDIRECTORY") + if staging_dir: + result_path = Path(staging_dir) / "regenerate-sdk-result.json" + with open(result_path, "w") as f: + f.write(result_json) + logging.info(f"Wrote regenerate-sdk-result.json to {result_path}") + git_add() + if create_spec_pr_flag: + apply_tspconfig_module_bumps(spec_root, result.get("module_version_changed", {})) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="SDK regeneration") @@ -388,6 +543,20 @@ def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_late default=False, ) + parser.add_argument( + "--create-spec-pr", + help="Whether to create a PR in the spec repo to bump go module suffixes in tspconfig.yaml", + type=lambda x: x.lower() == 'true', + default=False, + ) + + parser.add_argument( + "--spec-root", + help="azure-rest-api-specs repo root folder (required when --create-spec-pr is true)", + type=str, + default="", + ) + args = parser.parse_args() - main(args.sdk_root, args.typespec_go_root, args.typespec_go_branch, args.use_latest_spec, args.service_filter, args.use_dev_package) \ No newline at end of file + main(args.sdk_root, args.typespec_go_root, args.typespec_go_branch, args.use_latest_spec, args.service_filter, args.use_dev_package, args.create_spec_pr, args.spec_root) \ No newline at end of file