From 1722c8dbb1ebd51844735d4b93cb572029c7a971 Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Mon, 29 Jun 2026 13:03:11 +0800 Subject: [PATCH 01/13] Refine regen pipeline: publish result artifact, preserve module version, optional spec PR --- eng/pipelines/sdk-regenerate.yml | 15 ++- eng/scripts/sdk_regenerate.py | 182 +++++++++++++++++++++++++++++-- 2 files changed, 187 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/sdk-regenerate.yml b/eng/pipelines/sdk-regenerate.yml index cb09c3df76..046ac9912d 100644 --- a/eng/pipelines/sdk-regenerate.yml +++ b/eng/pipelines/sdk-regenerate.yml @@ -18,6 +18,9 @@ parameters: - name: UseDevPackage default: false type: boolean +- name: CreateSpecPR + default: false + type: boolean variables: - template: /eng/pipelines/templates/variables/globals.yml @@ -35,6 +38,11 @@ resources: name: Azure/azure-sdk-for-go endpoint: azure ref: main + - repository: azure-rest-api-specs + type: github + name: Azure/azure-rest-api-specs + endpoint: azure + ref: main jobs: - job: Generate_SDK @@ -45,6 +53,7 @@ jobs: - checkout: self fetchDepth: 1 - checkout: azure-sdk-for-go + - checkout: azure-rest-api-specs - task: NodeTool@0 displayName: 'Install Node.js $(NodeVersion)' @@ -73,10 +82,14 @@ jobs: displayName: 'Install tsp-client' - script: | - python3 $(Build.SourcesDirectory)/autorest.go/eng/scripts/sdk_regenerate.py --sdk-root=$(Build.SourcesDirectory)/azure-sdk-for-go --typespec-go-root=$(Build.SourcesDirectory)/autorest.go/packages/typespec-go --typespec-go-branch=$(Build.SourceBranchName) --use-latest-spec=${{ parameters.UseLatestSpec }} --service-filter="${{ parameters.ServiceFilter }}" --use-dev-package=${{ parameters.UseDevPackage }} + python3 $(Build.SourcesDirectory)/autorest.go/eng/scripts/sdk_regenerate.py --sdk-root=$(Build.SourcesDirectory)/azure-sdk-for-go --typespec-go-root=$(Build.SourcesDirectory)/autorest.go/packages/typespec-go --typespec-go-branch=$(Build.SourceBranchName) --use-latest-spec=${{ parameters.UseLatestSpec }} --service-filter="${{ parameters.ServiceFilter }}" --use-dev-package=${{ parameters.UseDevPackage }} --create-spec-pr=${{ parameters.CreateSpecPR }} --spec-root=$(Build.SourcesDirectory)/azure-rest-api-specs displayName: 'Generate SDK' workingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-go + - publish: $(Build.ArtifactStagingDirectory)/regenerate-sdk-result.json + artifact: regenerate-sdk-result + displayName: 'Publish regenerate SDK result' + - template: /eng/common/pipelines/templates/steps/login-to-github.yml@azure-sdk-for-go parameters: ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-for-go/eng/common/scripts diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index ea301907de..04af83f951 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -12,6 +12,7 @@ import argparse import logging import json +import os import re import glob import urllib.request @@ -223,12 +224,76 @@ def get_api_version(package_folder: Path) -> Optional[str]: return api_version + +def get_module_name(package_folder: Path) -> Optional[str]: + """Read the module name from go.mod in the package folder.""" + go_mod_path = package_folder / "go.mod" + if not go_mod_path.exists(): + return None + try: + with open(go_mod_path, "r", encoding="utf-8") as f: + for line in f: + stripped = line.strip() + if stripped.startswith("module "): + return stripped[len("module "):].strip() + except FileNotFoundError as e: + logging.warning(f"Failed to read go.mod for {package_folder.name}: {e}") + return None + + +def restore_module_name(package_folder: Path, original_module: str) -> Optional[str]: + """Restore the original module path everywhere in the package after regeneration. + + Regeneration may bump the module version (e.g. .../armadvisor -> .../armadvisor/v2), + which changes both go.mod and import paths across all package files. Replace any new + module path with the original to keep the module version unchanged. + + Returns the bumped module path if a change was detected, otherwise None. + """ + go_mod_path = package_folder / "go.mod" + if not go_mod_path.exists(): + return None + current_module = get_module_name(package_folder) + if not current_module or current_module == original_module: + return None + logging.info( + f"Restoring module path from {current_module} to {original_module} for {package_folder.name}" + ) + for file_path in package_folder.rglob("*"): + if not file_path.is_file(): + continue + try: + content = file_path.read_text(encoding="utf-8") + except (UnicodeDecodeError, FileNotFoundError): + continue + if current_module not in content: + continue + file_path.write_text(content.replace(current_module, original_module), encoding="utf-8") + return current_module + + +def get_spec_directory(package_folder: Path) -> Optional[str]: + """Read the spec repo directory (containing tspconfig.yaml) from tsp-location.yaml.""" + tsp_location = package_folder / "tsp-location.yaml" + if not tsp_location.exists(): + return None + try: + with open(tsp_location, "r", encoding="utf-8") as f: + for line in f: + stripped = line.strip() + if stripped.startswith("directory:"): + return stripped[len("directory:"):].strip().strip('"') + except FileNotFoundError as e: + logging.warning(f"Failed to read tsp-location.yaml for {package_folder.name}: {e}") + return None + + + def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, typespec_go_root: str) -> Dict[str, List[str]]: result = { "succeed_to_regenerate": [], "fail_to_regenerate": [], - "not_found_api_version": [], - "time_to_regenerate": str(datetime.now()), + "not_found_api_version": [], "module_version_changed": {}, "time_to_regenerate": str(datetime.now()), "typespec_go_commit_hash": get_typespec_go_commit_hash(typespec_go_root) } # get all tsp-location.yaml @@ -242,6 +307,8 @@ def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, ty if use_latest_spec: logging.info("Using latest spec") update_commit_id(item, commit_id) + # Record the original module name so it is not changed by regeneration + original_module = get_module_name(package_folder) try: # Get API version for this package api_version = get_api_version(package_folder) @@ -299,9 +366,14 @@ def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, ty else: logging.info(f"Successfully regenerated {package_folder.name}") result["succeed_to_regenerate"].append(package_folder.name) - - result["succeed_to_regenerate"].sort() - result["fail_to_regenerate"].sort() + finally: + # Keep the original module name; do not bump the module version + if original_module: + bumped_module = restore_module_name(package_folder, original_module) + if bumped_module: + spec_directory = get_spec_directory(package_folder) + if spec_directory: + result["module_version_changed"][spec_directory] = bumped_module result["not_found_api_version"].sort() return result @@ -329,7 +401,71 @@ def git_add(): check_call("git add .", shell=True) -def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_latest_spec: bool, service_filter: str, use_dev_package: bool): +def bump_tspconfig_module(spec_root: str, spec_directory: str, bumped_module: str) -> bool: + """Update the go module suffix in tspconfig.yaml to the bumped module path. + + Returns True if the file was changed. + """ + tspconfig_path = Path(spec_root) / spec_directory / "tspconfig.yaml" + if not tspconfig_path.exists(): + logging.warning(f"tspconfig.yaml not found at {tspconfig_path}") + return False + with open(tspconfig_path, "r", encoding="utf-8") as f: + content = f.readlines() + changed = False + for idx in range(len(content)): + match = re.match(r"^(\s*module:\s*\"?)(github.com/[^\"\s]+)(\"?\s*)$", content[idx]) + if match: + if match.group(2) != bumped_module: + content[idx] = f"{match.group(1)}{bumped_module}{match.group(3)}" + changed = True + logging.info(f"Updated module in {tspconfig_path} to {bumped_module}") + break + if changed: + with open(tspconfig_path, "w", encoding="utf-8") as f: + f.writelines(content) + return changed + + +def create_spec_pr(spec_root: str, module_version_changed: dict, typespec_go_branch: str): + """Create a PR in the spec repo to bump go module suffixes in tspconfig.yaml.""" + if not module_version_changed: + logging.info("No module version changes; skipping spec PR") + return + if not spec_root or not Path(spec_root).exists(): + logging.warning("spec-root not provided or does not exist; skipping spec PR") + return + + changed_any = False + for spec_directory, bumped_module in module_version_changed.items(): + if bump_tspconfig_module(spec_root, spec_directory, bumped_module): + changed_any = True + if not changed_any: + logging.info("No tspconfig.yaml changes; skipping spec PR") + return + + branch = f"typespec-go-module-suffix-{typespec_go_branch}" + check_call("git add .", shell=True, cwd=spec_root) + check_call( + ['git', 'commit', '-m', 'Bump go module suffix in tspconfig.yaml'], + cwd=spec_root, + ) + check_call(f"git checkout -b {branch}", shell=True, cwd=spec_root) + check_call(f"git push --force origin {branch}", shell=True, cwd=spec_root) + check_call( + [ + 'gh', 'pr', 'create', + '--repo', 'Azure/azure-rest-api-specs', + '--base', 'main', + '--head', branch, + '--title', '[Automation] Bump go module suffix in tspconfig.yaml', + '--body', 'Automatically bump go module suffix for packages whose module version changed during regeneration.', + ], + cwd=spec_root, + ) + + +def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_latest_spec: bool, service_filter: str, use_dev_package: bool, create_spec_pr_flag: bool, spec_root: str): # Configure logging for better pipeline visibility logging.basicConfig( level=logging.INFO, @@ -342,10 +478,24 @@ def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_late prepare_branch(typespec_go_branch) update_emitter_package(sdk_root, typespec_go_root, use_dev_package) result = regenerate_sdk(use_latest_spec, service_filter, sdk_root, typespec_go_root) - with open("regenerate-sdk-result.json", "w") as f: - json.dump(result, f, indent=2) + + # Print the result instead of committing it to the repo + result_json = json.dumps(result, indent=2) + logging.info("Regenerate SDK result:\n%s", result_json) + + # Write the result to the artifact staging directory so it can be published as a pipeline artifact + staging_dir = os.environ.get("BUILD_ARTIFACTSTAGINGDIRECTORY") + if staging_dir: + result_path = Path(staging_dir) / "regenerate-sdk-result.json" + with open(result_path, "w") as f: + f.write(result_json) + logging.info(f"Wrote regenerate-sdk-result.json to {result_path}") + git_add() + if create_spec_pr_flag: + create_spec_pr(spec_root, result.get("module_version_changed", {}), typespec_go_branch) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="SDK regeneration") @@ -388,6 +538,20 @@ def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_late default=False, ) + parser.add_argument( + "--create-spec-pr", + help="Whether to create a PR in the spec repo to bump go module suffixes in tspconfig.yaml", + type=lambda x: x.lower() == 'true', + default=False, + ) + + parser.add_argument( + "--spec-root", + help="azure-rest-api-specs repo root folder (required when --create-spec-pr is true)", + type=str, + default="", + ) + args = parser.parse_args() - main(args.sdk_root, args.typespec_go_root, args.typespec_go_branch, args.use_latest_spec, args.service_filter, args.use_dev_package) \ No newline at end of file + main(args.sdk_root, args.typespec_go_root, args.typespec_go_branch, args.use_latest_spec, args.service_filter, args.use_dev_package, args.create_spec_pr, args.spec_root) \ No newline at end of file From 9621ea8d0952d0a63c4c5e7419baeacf28e93a23 Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Mon, 29 Jun 2026 15:13:42 +0800 Subject: [PATCH 02/13] Make spec PR a pipeline step using standard templates --- eng/pipelines/sdk-regenerate.yml | 15 ++++++++++++ eng/scripts/sdk_regenerate.py | 42 +++++++++----------------------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/eng/pipelines/sdk-regenerate.yml b/eng/pipelines/sdk-regenerate.yml index 046ac9912d..40a2f13bf2 100644 --- a/eng/pipelines/sdk-regenerate.yml +++ b/eng/pipelines/sdk-regenerate.yml @@ -107,3 +107,18 @@ jobs: OpenAsDraft: 'true' PushArgs: '--force' AuthToken: $(GH_TOKEN) + + - ${{ if parameters.CreateSpecPR }}: + - template: /eng/common/pipelines/templates/steps//create-pull-request.yml@azure-sdk-for-go + parameters: + WorkingDirectory: $(Build.SourcesDirectory)/azure-rest-api-specs + ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-for-go/eng/common/scripts + RepoName: azure-rest-api-specs + PROwner: Azure + BaseBranchName: 'refs/heads/main' + PRBranchName: typespec-go-module-suffix-$(Build.SourceBranchName) + CommitMsg: 'Bump go module suffix in tspconfig.yaml' + PRTitle: '[Automation] Bump go module suffix in tspconfig.yaml' + OpenAsDraft: 'true' + PushArgs: '--force' + AuthToken: $(GH_TOKEN) diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index 04af83f951..6bbcf61986 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -427,42 +427,24 @@ def bump_tspconfig_module(spec_root: str, spec_directory: str, bumped_module: st return changed -def create_spec_pr(spec_root: str, module_version_changed: dict, typespec_go_branch: str): - """Create a PR in the spec repo to bump go module suffixes in tspconfig.yaml.""" +def apply_tspconfig_module_bumps(spec_root: str, module_version_changed: dict) -> bool: + """Bump go module suffixes in tspconfig.yaml for changed packages. + + Returns True if any tspconfig.yaml was modified. The pipeline is responsible for + committing and opening the PR against the spec repo. + """ if not module_version_changed: - logging.info("No module version changes; skipping spec PR") - return + logging.info("No module version changes; nothing to update in spec repo") + return False if not spec_root or not Path(spec_root).exists(): - logging.warning("spec-root not provided or does not exist; skipping spec PR") - return + logging.warning("spec-root not provided or does not exist; skipping tspconfig update") + return False changed_any = False for spec_directory, bumped_module in module_version_changed.items(): if bump_tspconfig_module(spec_root, spec_directory, bumped_module): changed_any = True - if not changed_any: - logging.info("No tspconfig.yaml changes; skipping spec PR") - return - - branch = f"typespec-go-module-suffix-{typespec_go_branch}" - check_call("git add .", shell=True, cwd=spec_root) - check_call( - ['git', 'commit', '-m', 'Bump go module suffix in tspconfig.yaml'], - cwd=spec_root, - ) - check_call(f"git checkout -b {branch}", shell=True, cwd=spec_root) - check_call(f"git push --force origin {branch}", shell=True, cwd=spec_root) - check_call( - [ - 'gh', 'pr', 'create', - '--repo', 'Azure/azure-rest-api-specs', - '--base', 'main', - '--head', branch, - '--title', '[Automation] Bump go module suffix in tspconfig.yaml', - '--body', 'Automatically bump go module suffix for packages whose module version changed during regeneration.', - ], - cwd=spec_root, - ) + return changed_any def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_latest_spec: bool, service_filter: str, use_dev_package: bool, create_spec_pr_flag: bool, spec_root: str): @@ -494,7 +476,7 @@ def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_late git_add() if create_spec_pr_flag: - create_spec_pr(spec_root, result.get("module_version_changed", {}), typespec_go_branch) + apply_tspconfig_module_bumps(spec_root, result.get("module_version_changed", {})) if __name__ == "__main__": From 77a735a13cc3c12e4b47e843cd566e54c1e44394 Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Mon, 29 Jun 2026 15:20:33 +0800 Subject: [PATCH 03/13] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- eng/scripts/sdk_regenerate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index 6bbcf61986..0c6a2c8797 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -374,6 +374,8 @@ def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, ty spec_directory = get_spec_directory(package_folder) if spec_directory: result["module_version_changed"][spec_directory] = bumped_module + result["succeed_to_regenerate"].sort() + result["fail_to_regenerate"].sort() result["not_found_api_version"].sort() return result From d6b898400da6061a7eca82d4721173720db515c0 Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Mon, 29 Jun 2026 15:20:54 +0800 Subject: [PATCH 04/13] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-regenerate.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/sdk-regenerate.yml b/eng/pipelines/sdk-regenerate.yml index 40a2f13bf2..75b9d7b3c9 100644 --- a/eng/pipelines/sdk-regenerate.yml +++ b/eng/pipelines/sdk-regenerate.yml @@ -53,7 +53,8 @@ jobs: - checkout: self fetchDepth: 1 - checkout: azure-sdk-for-go - - checkout: azure-rest-api-specs + - ${{ if eq(parameters.CreateSpecPR, true) }}: + - checkout: azure-rest-api-specs - task: NodeTool@0 displayName: 'Install Node.js $(NodeVersion)' From 7853bd06499f9b1b48c105b8a261f0eacb2d439c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Jun 2026 07:24:42 +0000 Subject: [PATCH 05/13] Fix regenerate_sdk result typing and formatting --- eng/scripts/sdk_regenerate.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index 0c6a2c8797..11208a06a5 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -4,7 +4,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from pathlib import Path import subprocess from datetime import datetime @@ -289,12 +289,14 @@ def get_spec_directory(package_folder: Path) -> Optional[str]: -def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, typespec_go_root: str) -> Dict[str, List[str]]: +def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, typespec_go_root: str) -> Dict[str, Any]: result = { - "succeed_to_regenerate": [], - "fail_to_regenerate": [], - "not_found_api_version": [], "module_version_changed": {}, "time_to_regenerate": str(datetime.now()), - "typespec_go_commit_hash": get_typespec_go_commit_hash(typespec_go_root) + "succeed_to_regenerate": [], + "fail_to_regenerate": [], + "not_found_api_version": [], + "module_version_changed": {}, + "time_to_regenerate": str(datetime.now()), + "typespec_go_commit_hash": get_typespec_go_commit_hash(typespec_go_root), } # get all tsp-location.yaml commit_id = get_latest_commit_id() From 4bc0d4a7ef105b620888beafae6dacf10377473d Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Mon, 29 Jun 2026 15:36:37 +0800 Subject: [PATCH 06/13] fix module suffix replacement logic --- eng/scripts/sdk_regenerate.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index 11208a06a5..3fa7b47e86 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -242,14 +242,7 @@ def get_module_name(package_folder: Path) -> Optional[str]: def restore_module_name(package_folder: Path, original_module: str) -> Optional[str]: - """Restore the original module path everywhere in the package after regeneration. - - Regeneration may bump the module version (e.g. .../armadvisor -> .../armadvisor/v2), - which changes both go.mod and import paths across all package files. Replace any new - module path with the original to keep the module version unchanged. - - Returns the bumped module path if a change was detected, otherwise None. - """ + """Restore the original module path across the package; return the bumped path if changed.""" go_mod_path = package_folder / "go.mod" if not go_mod_path.exists(): return None @@ -259,6 +252,8 @@ def restore_module_name(package_folder: Path, original_module: str) -> Optional[ logging.info( f"Restoring module path from {current_module} to {original_module} for {package_folder.name}" ) + base_module = re.sub(r"/v\d+$", "", current_module) + pattern = re.compile(re.escape(base_module) + r"(?:/v\d+)?") for file_path in package_folder.rglob("*"): if not file_path.is_file(): continue @@ -266,9 +261,9 @@ def restore_module_name(package_folder: Path, original_module: str) -> Optional[ content = file_path.read_text(encoding="utf-8") except (UnicodeDecodeError, FileNotFoundError): continue - if current_module not in content: + if base_module not in content: continue - file_path.write_text(content.replace(current_module, original_module), encoding="utf-8") + file_path.write_text(pattern.sub(original_module, content), encoding="utf-8") return current_module @@ -406,7 +401,7 @@ def git_add(): def bump_tspconfig_module(spec_root: str, spec_directory: str, bumped_module: str) -> bool: - """Update the go module suffix in tspconfig.yaml to the bumped module path. + """Update only the go module version suffix in tspconfig.yaml. Returns True if the file was changed. """ @@ -414,16 +409,19 @@ def bump_tspconfig_module(spec_root: str, spec_directory: str, bumped_module: st if not tspconfig_path.exists(): logging.warning(f"tspconfig.yaml not found at {tspconfig_path}") return False + suffix_match = re.search(r"/(v\d+)$", bumped_module) + new_suffix = f"/{suffix_match.group(1)}" if suffix_match else "" with open(tspconfig_path, "r", encoding="utf-8") as f: content = f.readlines() changed = False for idx in range(len(content)): - match = re.match(r"^(\s*module:\s*\"?)(github.com/[^\"\s]+)(\"?\s*)$", content[idx]) + match = re.match(r"^(\s*module:\s*\"?)(\S+?)(/v\d+)?(\"?\s*)$", content[idx]) if match: - if match.group(2) != bumped_module: - content[idx] = f"{match.group(1)}{bumped_module}{match.group(3)}" + updated = f"{match.group(1)}{match.group(2)}{new_suffix}{match.group(4)}" + if updated != content[idx]: + content[idx] = updated changed = True - logging.info(f"Updated module in {tspconfig_path} to {bumped_module}") + logging.info(f"Updated module suffix in {tspconfig_path} to '{new_suffix}'") break if changed: with open(tspconfig_path, "w", encoding="utf-8") as f: From c43da6e888e47559dbafe587455799fced51b47f Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Mon, 29 Jun 2026 15:53:24 +0800 Subject: [PATCH 07/13] fix --- eng/scripts/sdk_regenerate.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index 3fa7b47e86..02455644be 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -254,7 +254,8 @@ def restore_module_name(package_folder: Path, original_module: str) -> Optional[ ) base_module = re.sub(r"/v\d+$", "", current_module) pattern = re.compile(re.escape(base_module) + r"(?:/v\d+)?") - for file_path in package_folder.rglob("*"): + import_pattern = re.compile(r"\"" + re.escape(base_module) + r"(?:/v\d+)?") + for file_path in [package_folder / "go.mod", package_folder / "README.md"]: if not file_path.is_file(): continue try: @@ -264,6 +265,14 @@ def restore_module_name(package_folder: Path, original_module: str) -> Optional[ if base_module not in content: continue file_path.write_text(pattern.sub(original_module, content), encoding="utf-8") + for file_path in package_folder.rglob("*.go"): + try: + content = file_path.read_text(encoding="utf-8") + except (UnicodeDecodeError, FileNotFoundError): + continue + if base_module not in content: + continue + file_path.write_text(import_pattern.sub('"' + original_module, content), encoding="utf-8") return current_module From 0dd26b003eeb3d1f2f8f0bca320cd42c30e6bfec Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Mon, 29 Jun 2026 15:55:30 +0800 Subject: [PATCH 08/13] fix bug --- eng/scripts/sdk_regenerate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index 02455644be..ffefb8dc24 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -242,7 +242,7 @@ def get_module_name(package_folder: Path) -> Optional[str]: def restore_module_name(package_folder: Path, original_module: str) -> Optional[str]: - """Restore the original module path across the package; return the bumped path if changed.""" + """Restore the original module path across the package; return the preserved path if changed.""" go_mod_path = package_folder / "go.mod" if not go_mod_path.exists(): return None @@ -273,7 +273,7 @@ def restore_module_name(package_folder: Path, original_module: str) -> Optional[ if base_module not in content: continue file_path.write_text(import_pattern.sub('"' + original_module, content), encoding="utf-8") - return current_module + return original_module def get_spec_directory(package_folder: Path) -> Optional[str]: From 7d83227553d3fe4d551805e1f07677548f53dd37 Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Mon, 29 Jun 2026 16:27:05 +0800 Subject: [PATCH 09/13] fix --- eng/scripts/sdk_regenerate.py | 52 +++++++++++++++++------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index ffefb8dc24..aba569eabd 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -242,38 +242,38 @@ def get_module_name(package_folder: Path) -> Optional[str]: def restore_module_name(package_folder: Path, original_module: str) -> Optional[str]: - """Restore the original module path across the package; return the preserved path if changed.""" - go_mod_path = package_folder / "go.mod" - if not go_mod_path.exists(): + """Regen drops the module suffix; restore the original suffixed path. Return it if restored.""" + if not original_module: return None - current_module = get_module_name(package_folder) - if not current_module or current_module == original_module: - return None - logging.info( - f"Restoring module path from {current_module} to {original_module} for {package_folder.name}" - ) - base_module = re.sub(r"/v\d+$", "", current_module) - pattern = re.compile(re.escape(base_module) + r"(?:/v\d+)?") - import_pattern = re.compile(r"\"" + re.escape(base_module) + r"(?:/v\d+)?") - for file_path in [package_folder / "go.mod", package_folder / "README.md"]: + base_module = re.sub(r"/v\d+$", "", original_module) + suffix = re.compile(r"(" + re.escape(base_module) + r")/v\d+") + bumped_module = None + changed_files = check_output( + ["git", "diff", "--name-only", "--", str(package_folder)], text=True + ).splitlines() + for rel in changed_files: + file_path = Path(rel) if not file_path.is_file(): continue try: - content = file_path.read_text(encoding="utf-8") - except (UnicodeDecodeError, FileNotFoundError): + old = check_output(["git", "show", f"HEAD:{rel}"], text=True) + new = file_path.read_text(encoding="utf-8") + except Exception: continue - if base_module not in content: + old_lines = old.splitlines(keepends=True) + new_lines = new.splitlines(keepends=True) + if len(old_lines) != len(new_lines): continue - file_path.write_text(pattern.sub(original_module, content), encoding="utf-8") - for file_path in package_folder.rglob("*.go"): - try: - content = file_path.read_text(encoding="utf-8") - except (UnicodeDecodeError, FileNotFoundError): - continue - if base_module not in content: - continue - file_path.write_text(import_pattern.sub('"' + original_module, content), encoding="utf-8") - return original_module + for idx, new_line in enumerate(new_lines): + if new_line == old_lines[idx]: + continue + if suffix.sub(r"\1", old_lines[idx]) == new_line: + match = suffix.search(old_lines[idx]) + if match: + bumped_module = match.group(0) + new_lines[idx] = old_lines[idx] + file_path.write_text("".join(new_lines), encoding="utf-8") + return bumped_module def get_spec_directory(package_folder: Path) -> Optional[str]: From 7557538fb69380dab64ef1251041b2f451e84d7a Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Mon, 29 Jun 2026 16:59:56 +0800 Subject: [PATCH 10/13] fix --- eng/scripts/sdk_regenerate.py | 37 ++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index aba569eabd..d9a13c1957 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -245,9 +245,11 @@ def restore_module_name(package_folder: Path, original_module: str) -> Optional[ """Regen drops the module suffix; restore the original suffixed path. Return it if restored.""" if not original_module: return None - base_module = re.sub(r"/v\d+$", "", original_module) - suffix = re.compile(r"(" + re.escape(base_module) + r")/v\d+") - bumped_module = None + suffix_match = re.search(r"/v\d+$", original_module) + if not suffix_match: + return None + base_module = original_module[: suffix_match.start()] + pattern = re.compile(re.escape(base_module) + r"(?!/v\d+)") changed_files = check_output( ["git", "diff", "--name-only", "--", str(package_folder)], text=True ).splitlines() @@ -256,24 +258,23 @@ def restore_module_name(package_folder: Path, original_module: str) -> Optional[ if not file_path.is_file(): continue try: - old = check_output(["git", "show", f"HEAD:{rel}"], text=True) - new = file_path.read_text(encoding="utf-8") + old_content = check_output(["git", "show", f"HEAD:{rel}"], text=True) + new_content = file_path.read_text(encoding="utf-8") except Exception: continue - old_lines = old.splitlines(keepends=True) - new_lines = new.splitlines(keepends=True) - if len(old_lines) != len(new_lines): - continue - for idx, new_line in enumerate(new_lines): - if new_line == old_lines[idx]: + old_lines_set = set(old_content.splitlines()) + new_lines = new_content.splitlines(keepends=True) + changed = False + for idx, line in enumerate(new_lines): + if not pattern.search(line): continue - if suffix.sub(r"\1", old_lines[idx]) == new_line: - match = suffix.search(old_lines[idx]) - if match: - bumped_module = match.group(0) - new_lines[idx] = old_lines[idx] - file_path.write_text("".join(new_lines), encoding="utf-8") - return bumped_module + restored = pattern.sub(original_module, line) + if restored.rstrip("\n") in old_lines_set: + new_lines[idx] = restored + changed = True + if changed: + file_path.write_text("".join(new_lines), encoding="utf-8") + return original_module def get_spec_directory(package_folder: Path) -> Optional[str]: From 4449effc09166895bd7e006e812ccc6e7509962d Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Mon, 29 Jun 2026 17:13:26 +0800 Subject: [PATCH 11/13] update doc --- docs/sdk-regeneration-pipeline.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/sdk-regeneration-pipeline.md b/docs/sdk-regeneration-pipeline.md index b921e8c615..83dfb08c89 100644 --- a/docs/sdk-regeneration-pipeline.md +++ b/docs/sdk-regeneration-pipeline.md @@ -37,6 +37,7 @@ The SDK regeneration pipeline automates the process of updating Azure SDK packag #### 4. Results & PR Creation - Generates results report (`regenerate-sdk-result.json`) - Creates draft PR in azure-sdk-for-go with all changes, PR title: `[Automation] Regenerate SDK based on typespec-go branch {branch-name}` +- If `CreateSpecPR` is enabled and module suffixes were restored, creates a draft PR in azure-rest-api-specs to bump the go module suffix in `tspconfig.yaml` ### Pipeline Parameters @@ -47,6 +48,7 @@ The SDK regeneration pipeline automates the process of updating Azure SDK packag | `UseLatestSpec` | boolean | `false` | Whether to use the latest API specifications from [azure-rest-api-specs](https://github.com/Azure/azure-rest-api-specs) or the original commit of `tsp-location.yml` | | `ServiceFilter` | string | `.*` | Regex pattern to filter which services to regenerate. Matches against the service package name (e.g., `armcompute`, `armstorage`) | | `UseDevPackage` | boolean | `false` | Whether to use dev package (.tgz) from current branch or the recent released package from npm registry | +| `CreateSpecPR` | boolean | `false` | Whether to create a PR in azure-rest-api-specs to bump go module suffixes in `tspconfig.yaml` when module versions are detected | #### Usage Examples @@ -126,6 +128,9 @@ You can find the generated SDK pull request link from pipeline logs "succeed_to_regenerate": ["package1", "package2"], "fail_to_regenerate": ["package3"], "not_found_api_version": ["package4"], + "module_version_changed": { + "specification/advisor/Advisor.Management": "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/advisor/armadvisor/v2" + }, "time_to_regenerate": "2025-09-25 10:30:45.123456", "typespec_go_commit_hash": "abc123def456789..." } @@ -161,7 +166,7 @@ After each TypeSpec or Go emitter version release: 3. **Merge Regeneration PR**: Merge the refresh PR to keep SDK up-to-date ### Quality Gates -- All pipelines must be pass +- All pipelines must pass - All SDK code changes are made by either TypeSpec changes or Go emitter changes - Module versions must not be changed - API versions must not be changed From be2f5844eea33def6e35cac0dae898134947b706 Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Tue, 30 Jun 2026 09:55:18 +0800 Subject: [PATCH 12/13] fix --- docs/sdk-regeneration-pipeline.md | 2 +- eng/scripts/sdk_regenerate.py | 41 +++++++++---------------------- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/docs/sdk-regeneration-pipeline.md b/docs/sdk-regeneration-pipeline.md index 83dfb08c89..d9e139ee35 100644 --- a/docs/sdk-regeneration-pipeline.md +++ b/docs/sdk-regeneration-pipeline.md @@ -37,7 +37,7 @@ The SDK regeneration pipeline automates the process of updating Azure SDK packag #### 4. Results & PR Creation - Generates results report (`regenerate-sdk-result.json`) - Creates draft PR in azure-sdk-for-go with all changes, PR title: `[Automation] Regenerate SDK based on typespec-go branch {branch-name}` -- If `CreateSpecPR` is enabled and module suffixes were restored, creates a draft PR in azure-rest-api-specs to bump the go module suffix in `tspconfig.yaml` +- If `CreateSpecPR` is enabled and a module version bump was detected, creates a draft PR in azure-rest-api-specs to bump the go module suffix in `tspconfig.yaml` ### Pipeline Parameters diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index d9a13c1957..c95e44ea0b 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -241,39 +241,20 @@ def get_module_name(package_folder: Path) -> Optional[str]: return None -def restore_module_name(package_folder: Path, original_module: str) -> Optional[str]: - """Regen drops the module suffix; restore the original suffixed path. Return it if restored.""" +def detect_module_version_change(package_folder: Path, original_module: str) -> Optional[str]: + """Detect if regen dropped the module version suffix. If so, revert all changes in the + package and return the original suffixed module so the spec PR can bump tspconfig.""" if not original_module: return None suffix_match = re.search(r"/v\d+$", original_module) if not suffix_match: return None - base_module = original_module[: suffix_match.start()] - pattern = re.compile(re.escape(base_module) + r"(?!/v\d+)") - changed_files = check_output( - ["git", "diff", "--name-only", "--", str(package_folder)], text=True - ).splitlines() - for rel in changed_files: - file_path = Path(rel) - if not file_path.is_file(): - continue - try: - old_content = check_output(["git", "show", f"HEAD:{rel}"], text=True) - new_content = file_path.read_text(encoding="utf-8") - except Exception: - continue - old_lines_set = set(old_content.splitlines()) - new_lines = new_content.splitlines(keepends=True) - changed = False - for idx, line in enumerate(new_lines): - if not pattern.search(line): - continue - restored = pattern.sub(original_module, line) - if restored.rstrip("\n") in old_lines_set: - new_lines[idx] = restored - changed = True - if changed: - file_path.write_text("".join(new_lines), encoding="utf-8") + current_module = get_module_name(package_folder) + if current_module == original_module: + return None + # Module suffix was dropped during regen; revert the whole package and bump the spec instead. + check_call(["git", "checkout", "--", str(package_folder)]) + logging.info(f"Reverted {package_folder.name}; module version bump will be handled in spec repo") return original_module @@ -374,9 +355,9 @@ def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, ty logging.info(f"Successfully regenerated {package_folder.name}") result["succeed_to_regenerate"].append(package_folder.name) finally: - # Keep the original module name; do not bump the module version + # If regen dropped the module version suffix, revert the package and bump the spec instead if original_module: - bumped_module = restore_module_name(package_folder, original_module) + bumped_module = detect_module_version_change(package_folder, original_module) if bumped_module: spec_directory = get_spec_directory(package_folder) if spec_directory: From cf97fb8238defd580350339b6a586d9b45801b32 Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Tue, 30 Jun 2026 11:28:05 +0800 Subject: [PATCH 13/13] support new metadata --- eng/scripts/sdk_regenerate.py | 56 +++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py index c95e44ea0b..a6c4c3727e 100644 --- a/eng/scripts/sdk_regenerate.py +++ b/eng/scripts/sdk_regenerate.py @@ -166,22 +166,52 @@ def update_commit_id(file: Path, commit_id: str): def get_api_version_from_metadata(package_folder: Path) -> Optional[str]: - """Extract API version from metadata.json file if it exists.""" + """Return the api-version emitter option value from metadata.json, if any. + + Supports the legacy ``{"apiVersion": "..."}`` string and the current + ``{"apiVersions": {namespace: version}}`` map. Returns a plain version when + all namespaces share one version, otherwise a compact JSON namespace->version + map that the emitter uses to regenerate each namespace at its own version. + """ # Construct the metadata.json path based on the package folder structure # {package_folder}/testdata/_metadata.json metadata_path = package_folder / "testdata" / "_metadata.json" - - if metadata_path.exists(): - try: - with open(metadata_path, "r") as f: - metadata = json.load(f) - api_version = metadata.get("apiVersion") - if api_version: - logging.info(f"Found API version {api_version} in metadata.json for {package_folder.name}") - return api_version - except (json.JSONDecodeError, FileNotFoundError) as e: - logging.warning(f"Failed to read metadata.json for {package_folder.name}: {e}") - + + if not metadata_path.exists(): + return None + + try: + with open(metadata_path, "r") as f: + metadata = json.load(f) + except (json.JSONDecodeError, FileNotFoundError) as e: + logging.warning(f"Failed to read metadata.json for {package_folder.name}: {e}") + return None + + # current format: a map of namespace -> API version + api_versions = metadata.get("apiVersions") + if isinstance(api_versions, dict): + # drop namespaces without a version, keeping the namespace->version map + versions = {ns: ver for ns, ver in api_versions.items() if ver} + if versions: + distinct = set(versions.values()) + if len(distinct) == 1: + api_version = next(iter(distinct)) + logging.info(f"Found API version {api_version} in metadata.json for {package_folder.name}") + return api_version + # multiple namespaces at different versions: pass the whole map so the + # emitter regenerates each service namespace at its recorded version + api_version = json.dumps(versions, separators=(",", ":")) + logging.info( + f"Found multiple API versions {versions} in metadata.json for " + f"{package_folder.name}, passing per-namespace versions" + ) + return api_version + + # legacy format: a single API version string + api_version = metadata.get("apiVersion") + if api_version: + logging.info(f"Found API version {api_version} in metadata.json for {package_folder.name}") + return api_version return None