Skip to content

cherry pick #4930 to release 4.2.0 #4931

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
PetrusZ wants to merge 2 commits into
base: release-4.2.0
Choose a base branch
from
Open

cherry pick #4930 to release 4.2.0 #4931

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aa3bef6
fix: rollback multi workload service error
PetrusZ Aug 24, 2026
1ea634d
fix: error in updateServiceModuleImages
PetrusZ Aug 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 34 additions & 6 deletions pkg/microservice/aslan/core/common/service/version.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,20 +345,34 @@ func RollbackEnvServiceVersion(ctx *internalhandler.Context, projectName, envNam
return nil, e.ErrRollbackEnvServiceVersion.AddErr(fmt.Errorf("failed to generate service yaml, error: %v", err))
}

rollbackImages := make(map[string]string, len(envSvcVersion.Service.Containers))
for _, rollbackContainer := range envSvcVersion.Service.Containers {
serviceModule := &commonmodels.DeployServiceModule{
ServiceModule: envSvcVersion.Service.ServiceName,
ServiceModule: rollbackContainer.Name,
Image: rollbackContainer.Image,
ImageName: rollbackContainer.ImageName,
}

replaceResources, relatedPodLabels, err := jobcontroller.UpdateExternalServiceModule(ctx, kubeClient, clientSet, resources, env, serviceName, serviceModule, detail, ctx.UserName, nil, log)
replaceResources, relatedPodLabels, err := jobcontroller.UpdateExternalServiceModule(ctx, kubeClient, clientSet, resources, env, serviceName, serviceModule, nil, log)
if err != nil {
return nil, e.ErrRollbackEnvServiceVersion.AddErr(err)
rollbackErr := fmt.Errorf("failed to rollback service %s/%s/%s to revision %d, verify cluster state: %w", projectName, envName, serviceName, revision, err)
if len(rollbackImages) > 0 {
rollbackErr = fmt.Errorf("service %s/%s/%s was partially rolled back to revision %d (%d/%d modules); environment and environment version were not updated: %w", projectName, envName, serviceName, revision, len(rollbackImages), len(envSvcVersion.Service.Containers), err)
}
log.Error(rollbackErr)
return nil, e.ErrRollbackEnvServiceVersion.AddErr(rollbackErr)
}
rollbackImages[serviceModule.ServiceModule] = serviceModule.Image
rollbackStatus.ReplaceResources = append(rollbackStatus.ReplaceResources, replaceResources...)
rollbackStatus.RelatedPodLabels = append(rollbackStatus.RelatedPodLabels, relatedPodLabels...)
}
if len(rollbackImages) > 0 {
if err := commonutil.UpdateProductImage(env.EnvName, env.ProductName, serviceName, rollbackImages, detail, ctx.UserName, log); err != nil {
syncErr := fmt.Errorf("service %s/%s/%s was rolled back to revision %d in the cluster, but failed to sync environment and environment version: %w", projectName, envName, serviceName, revision, err)
log.Error(syncErr)
return nil, e.ErrRollbackEnvServiceVersion.AddErr(syncErr)
}
}
} else {
if envSvcVersion.Service.Type == setting.K8SDeployType {
kubeClient, err := clientmanager.NewKubeClientManager().GetControllerRuntimeClient(env.ClusterID)
Expand Down Expand Up @@ -402,20 +416,34 @@ func RollbackEnvServiceVersion(ctx *internalhandler.Context, projectName, envNam
return nil, e.ErrRollbackEnvServiceVersion.AddErr(fmt.Errorf("failed to generate service yaml, error: %v", err))
}

rollbackImages := make(map[string]string, len(envSvcVersion.Service.Containers))
for _, rollbackContainer := range envSvcVersion.Service.Containers {
serviceModule := &commonmodels.DeployServiceModule{
ServiceModule: envSvcVersion.Service.ServiceName,
ServiceModule: rollbackContainer.Name,
Image: rollbackContainer.Image,
ImageName: rollbackContainer.ImageName,
}

replaceResources, relatedPodLabels, err := jobcontroller.UpdateExternalServiceModule(ctx, kubeClient, clientSet, resources, env, serviceName, serviceModule, detail, ctx.UserName, nil, log)
replaceResources, relatedPodLabels, err := jobcontroller.UpdateExternalServiceModule(ctx, kubeClient, clientSet, resources, env, serviceName, serviceModule, nil, log)
if err != nil {
return nil, e.ErrRollbackEnvServiceVersion.AddErr(err)
rollbackErr := fmt.Errorf("failed to rollback service %s/%s/%s to revision %d, verify cluster state: %w", projectName, envName, serviceName, revision, err)
if len(rollbackImages) > 0 {
rollbackErr = fmt.Errorf("service %s/%s/%s was partially rolled back to revision %d (%d/%d modules); environment and environment version were not updated: %w", projectName, envName, serviceName, revision, len(rollbackImages), len(envSvcVersion.Service.Containers), err)
}
log.Error(rollbackErr)
return nil, e.ErrRollbackEnvServiceVersion.AddErr(rollbackErr)
}
rollbackImages[serviceModule.ServiceModule] = serviceModule.Image
rollbackStatus.ReplaceResources = append(rollbackStatus.ReplaceResources, replaceResources...)
rollbackStatus.RelatedPodLabels = append(rollbackStatus.RelatedPodLabels, relatedPodLabels...)
}
if len(rollbackImages) > 0 {
if err := commonutil.UpdateProductImage(env.EnvName, env.ProductName, serviceName, rollbackImages, detail, ctx.UserName, log); err != nil {
syncErr := fmt.Errorf("service %s/%s/%s was rolled back to revision %d in the cluster, but failed to sync environment and environment version: %w", projectName, envName, serviceName, revision, err)
log.Error(syncErr)
return nil, e.ErrRollbackEnvServiceVersion.AddErr(syncErr)
}
}
} else {
currentDeployStrategy := env.ServiceDeployStrategy[envSvcVersion.Service.ServiceName]
if currentDeployStrategy != "" && currentDeployStrategy != setting.ServiceDeployStrategyDeploy {
Expand Down
22 changes: 16 additions & 6 deletions pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_deploy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,7 @@ func (c *DeployJobCtl) updateSystemService(env *commonmodels.Product, currentYam
return nil
}

func UpdateExternalServiceModule(ctx context.Context, kubeClient client.Client, clientSet *kubernetes.Clientset, resources []*kube.WorkloadResource, env *commonmodels.Product, serviceName string, serviceModule *commonmodels.DeployServiceModule, detail, userName string, jobLogctx *joblog.JobLogContext, logger *zap.SugaredLogger) (replaceResources []commonmodels.Resource, relatedPodLabels []map[string]string, err error) {
func UpdateExternalServiceModule(ctx context.Context, kubeClient client.Client, clientSet *kubernetes.Clientset, resources []*kube.WorkloadResource, env *commonmodels.Product, serviceName string, serviceModule *commonmodels.DeployServiceModule, jobLogctx *joblog.JobLogContext, logger *zap.SugaredLogger) (replaceResources []commonmodels.Resource, relatedPodLabels []map[string]string, err error) {
var replaced bool

deployments, statefulSets, cronJobs, betaCronJobs, jobs, err := kube.FetchSelectedWorkloads(env.Namespace, resources, kubeClient, clientSet)
Expand Down Expand Up @@ -594,9 +594,6 @@ Job:
if !replaced {
return nil, nil, fmt.Errorf("service %s container name %s is not found in env %s", serviceName, serviceModule.ServiceModule, env.EnvName)
}
if err := commonutil.UpdateProductImage(env.EnvName, env.ProductName, serviceName, map[string]string{serviceModule.ServiceModule: serviceModule.Image}, detail, userName, logger); err != nil {
return nil, nil, err
}
return replaceResources, relatedPodLabels, nil
}

Expand All @@ -607,23 +604,36 @@ func (c *DeployJobCtl) updateServiceModuleImages(ctx context.Context, resources
}

errList := new(multierror.Error)
updatedImages := make(map[string]string, len(c.jobTaskSpec.ServiceAndImages))
mutex := sync.Mutex{}
wg := sync.WaitGroup{}
for _, serviceModule := range c.jobTaskSpec.ServiceAndImages {
wg.Add(1)
go func(serviceModule *commonmodels.DeployServiceModule) {
defer wg.Done()
replaceResources, relatedPodLabels, err := UpdateExternalServiceModule(ctx, c.kubeClient, c.clientSet, resources, env, c.jobTaskSpec.ServiceName, serviceModule, "", c.workflowCtx.WorkflowTaskCreatorUsername, jobTaskctx, c.logger)
replaceResources, relatedPodLabels, err := UpdateExternalServiceModule(ctx, c.kubeClient, c.clientSet, resources, env, c.jobTaskSpec.ServiceName, serviceModule, jobTaskctx, c.logger)
mutex.Lock()
defer mutex.Unlock()
if err != nil {
errList = multierror.Append(errList, err)
} else {
updatedImages[serviceModule.ServiceModule] = serviceModule.Image
c.jobTaskSpec.ReplaceResources = append(c.jobTaskSpec.ReplaceResources, replaceResources...)
c.jobTaskSpec.RelatedPodLabels = append(c.jobTaskSpec.RelatedPodLabels, relatedPodLabels...)
}
}(serviceModule)
}
wg.Wait()
if err := errList.ErrorOrNil(); err != nil {
return err
if len(updatedImages) > 0 {
return fmt.Errorf("service %s/%s/%s images were partially updated (%d/%d modules); environment and environment version were not updated: %w", env.ProductName, env.EnvName, c.jobTaskSpec.ServiceName, len(updatedImages), len(c.jobTaskSpec.ServiceAndImages), err)
}
return fmt.Errorf("failed to update images for service %s/%s/%s, verify cluster state: %w", env.ProductName, env.EnvName, c.jobTaskSpec.ServiceName, err)
}
if len(updatedImages) > 0 {
if err := commonutil.UpdateProductImage(env.EnvName, env.ProductName, c.jobTaskSpec.ServiceName, updatedImages, "", c.workflowCtx.WorkflowTaskCreatorUsername, c.logger); err != nil {
return fmt.Errorf("service %s/%s/%s images were updated in the cluster, but failed to sync environment and environment version: %w", env.ProductName, env.EnvName, c.jobTaskSpec.ServiceName, err)
}
}
return nil
}
Expand Down
Loading