From efe5d8af30285fd10c1f6a03b061abe8786208c9 Mon Sep 17 00:00:00 2001 From: Qing Hao Date: Wed, 5 Nov 2025 02:34:44 -0500 Subject: [PATCH] set RecheckTime when minSuccessTime is defined Signed-off-by: Qing Hao --- pkg/apis/cluster/v1alpha1/rollout.go | 73 ++--- pkg/apis/cluster/v1alpha1/rollout_test.go | 309 ++++++++++++++++++---- 2 files changed, 301 insertions(+), 81 deletions(-) diff --git a/pkg/apis/cluster/v1alpha1/rollout.go b/pkg/apis/cluster/v1alpha1/rollout.go index 23e04c47..5c1e267c 100644 --- a/pkg/apis/cluster/v1alpha1/rollout.go +++ b/pkg/apis/cluster/v1alpha1/rollout.go @@ -50,8 +50,10 @@ type ClusterRolloutStatus struct { // Used to calculate timeout for progressing and failed status and minimum success time (i.e. soak // time) for succeeded status. LastTransitionTime *metav1.Time - // TimeOutTime is the timeout time when the status is progressing or failed (optional field). - TimeOutTime *metav1.Time + // RecheckTime is the time when the cluster should be rechecked (optional field). + // For succeeded status, tracks when the minSuccessTime (soak time) period ends. + // For progressing/failed status, tracks when the timeout occurs. + RecheckTime *metav1.Time } // RolloutResult contains list of clusters that are timeOut, removed and required to rollOut. A @@ -306,24 +308,24 @@ func progressivePerCluster( failureBreach = failureCount > maxFailures } - // Return if the list of exsiting rollout clusters has reached the target rollout size + // Return if the list of existing rollout clusters has reached the target rollout size if len(rolloutClusters) >= rolloutSize { return RolloutResult{ ClustersToRollout: rolloutClusters, ClustersTimeOut: timeoutClusters, MaxFailureBreach: failureBreach, - RecheckAfter: minRecheckAfter(rolloutClusters, minSuccessTime), + RecheckAfter: minRecheckAfter(rolloutClusters), } } } - // Return if the exsiting rollout clusters maxFailures is breached. + // Return if the existing rollout clusters maxFailures is breached. if failureBreach { return RolloutResult{ ClustersToRollout: rolloutClusters, ClustersTimeOut: timeoutClusters, MaxFailureBreach: failureBreach, - RecheckAfter: minRecheckAfter(rolloutClusters, minSuccessTime), + RecheckAfter: minRecheckAfter(rolloutClusters), } } @@ -352,7 +354,7 @@ func progressivePerCluster( return RolloutResult{ ClustersToRollout: rolloutClusters, ClustersTimeOut: timeoutClusters, - RecheckAfter: minRecheckAfter(rolloutClusters, minSuccessTime), + RecheckAfter: minRecheckAfter(rolloutClusters), } } } @@ -360,7 +362,7 @@ func progressivePerCluster( return RolloutResult{ ClustersToRollout: rolloutClusters, ClustersTimeOut: timeoutClusters, - RecheckAfter: minRecheckAfter(rolloutClusters, minSuccessTime), + RecheckAfter: minRecheckAfter(rolloutClusters), } } @@ -430,7 +432,7 @@ func progressivePerGroup( ClustersToRollout: rolloutClusters, ClustersTimeOut: timeoutClusters, MaxFailureBreach: failureBreach, - RecheckAfter: minRecheckAfter(rolloutClusters, minSuccessTime), + RecheckAfter: minRecheckAfter(rolloutClusters), } } } @@ -440,7 +442,7 @@ func progressivePerGroup( ClustersToRollout: rolloutClusters, ClustersTimeOut: timeoutClusters, MaxFailureBreach: failureBreach, - RecheckAfter: minRecheckAfter(rolloutClusters, minSuccessTime), + RecheckAfter: minRecheckAfter(rolloutClusters), } } @@ -470,8 +472,10 @@ func determineRolloutStatus( case Succeeded: // If the cluster succeeded but is still within the MinSuccessTime (i.e. "soak" time), // still add it to the list of rolloutClusters - minSuccessTimeTime := getTimeOutTime(status.LastTransitionTime, minSuccessTime) - if RolloutClock.Now().Before(minSuccessTimeTime.Time) { + minSuccessTimeTime := calculateRecheckTime(status.LastTransitionTime, minSuccessTime) + if minSuccessTimeTime != nil && RolloutClock.Now().Before(minSuccessTimeTime.Time) { + // Set RecheckTime to track when the soak period ends + status.RecheckTime = minSuccessTimeTime rolloutClusters = append(rolloutClusters, *status) } @@ -479,8 +483,8 @@ func determineRolloutStatus( case TimeOut, Skip: return rolloutClusters, timeoutClusters default: // For progressing, failed, or unknown status. - timeOutTime := getTimeOutTime(status.LastTransitionTime, timeout) - status.TimeOutTime = timeOutTime + timeOutTime := calculateRecheckTime(status.LastTransitionTime, timeout) + status.RecheckTime = timeOutTime // check if current time is before the timeout time if timeOutTime == nil || RolloutClock.Now().Before(timeOutTime.Time) { rolloutClusters = append(rolloutClusters, *status) @@ -493,20 +497,21 @@ func determineRolloutStatus( return rolloutClusters, timeoutClusters } -// getTimeOutTime calculates the timeout time given a start time and duration, instantiating the -// RolloutClock if a start time isn't provided. -func getTimeOutTime(startTime *metav1.Time, timeout time.Duration) *metav1.Time { - var timeoutTime time.Time - // if timeout is not set (default to maxTimeDuration), the timeout time should not be set - if timeout == maxTimeDuration { +// calculateRecheckTime calculates the recheck time by adding a duration to a start time. +// If startTime is nil, it uses the current time from RolloutClock. +// If duration is maxTimeDuration (indicating no timeout/soak period), it returns nil. +func calculateRecheckTime(startTime *metav1.Time, duration time.Duration) *metav1.Time { + var recheckTime time.Time + // if duration is not set (default to maxTimeDuration), the recheck time should not be set + if duration == maxTimeDuration { return nil } if startTime == nil { - timeoutTime = RolloutClock.Now().Add(timeout) + recheckTime = RolloutClock.Now().Add(duration) } else { - timeoutTime = startTime.Add(timeout) + recheckTime = startTime.Add(duration) } - return &metav1.Time{Time: timeoutTime} + return &metav1.Time{Time: recheckTime} } // calculateRolloutSize calculates the maximum portion from a total number of clusters by parsing a @@ -598,19 +603,21 @@ func decisionGroupsToGroupKeys(decisionsGroup []clusterv1alpha1.MandatoryDecisio return result } -func minRecheckAfter(rolloutClusters []ClusterRolloutStatus, minSuccessTime time.Duration) *time.Duration { - var minRecheckAfter *time.Duration +func minRecheckAfter(rolloutClusters []ClusterRolloutStatus) *time.Duration { + var minDuration *time.Duration + for _, r := range rolloutClusters { - if r.TimeOutTime != nil { - timeOut := r.TimeOutTime.Sub(RolloutClock.Now()) - if minRecheckAfter == nil || *minRecheckAfter > timeOut { - minRecheckAfter = &timeOut + // Check RecheckTime for both Progressing/Failed clusters (timeout) and Succeeded clusters (soak period) + if r.RecheckTime != nil { + recheckDuration := r.RecheckTime.Sub(RolloutClock.Now()) + // Only consider positive durations (future recheck times) + if recheckDuration > 0 { + if minDuration == nil || *minDuration > recheckDuration { + minDuration = &recheckDuration + } } } } - if minSuccessTime != 0 && (minRecheckAfter == nil || minSuccessTime < *minRecheckAfter) { - minRecheckAfter = &minSuccessTime - } - return minRecheckAfter + return minDuration } diff --git a/pkg/apis/cluster/v1alpha1/rollout_test.go b/pkg/apis/cluster/v1alpha1/rollout_test.go index 5af9e9df..9e75f4f5 100644 --- a/pkg/apis/cluster/v1alpha1/rollout_test.go +++ b/pkg/apis/cluster/v1alpha1/rollout_test.go @@ -142,13 +142,13 @@ func TestGetRolloutCluster_All(t *testing.T) { expectRolloutStrategy: &clusterv1alpha1.RolloutStrategy{Type: clusterv1alpha1.All, All: &clusterv1alpha1.RolloutAll{RolloutConfig: clusterv1alpha1.RolloutConfig{ProgressDeadline: "90s"}}}, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, {ClusterName: "cluster6", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, RecheckAfter: &recheckTime, }, @@ -208,7 +208,6 @@ func TestGetRolloutCluster_All(t *testing.T) { func TestGetRolloutCluster_Progressive(t *testing.T) { recheck30Duration := 30 * time.Second - recheck60Duration := 60 * time.Second tests := []testCase{ { name: "test progressive rollout deprecated timeout", @@ -347,7 +346,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupIndex: 1}, Status: ToApply}, @@ -433,7 +432,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupIndex: 1}, Status: ToApply}, @@ -492,10 +491,10 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, MaxFailureBreach: true, RecheckAfter: &recheck30Duration, @@ -552,14 +551,82 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Succeeded, LastTransitionTime: &fakeTime_30s}, - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_30s, TimeOutTime: &fakeTime60s}, + {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Succeeded, LastTransitionTime: &fakeTime_30s, RecheckTime: &fakeTime30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_30s, RecheckTime: &fakeTime60s}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, MaxFailureBreach: false, - RecheckAfter: &recheck60Duration, + RecheckAfter: &recheck30Duration, + }, + }, + { + name: "test progressive rollout with mixed timeout and minSuccessTime scenarios", + rolloutStrategy: clusterv1alpha1.RolloutStrategy{ + Type: clusterv1alpha1.Progressive, + Progressive: &clusterv1alpha1.RolloutProgressive{ + RolloutConfig: clusterv1alpha1.RolloutConfig{ + ProgressDeadline: "90s", + MaxFailures: intstr.FromString("100%"), + MinSuccessTime: metav1.Duration{Duration: time.Minute}, + }, + MaxConcurrency: intstr.FromInt32(4), + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1sdk.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3", "cluster4"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &clusterv1alpha1.RolloutStrategy{ + Type: clusterv1alpha1.Progressive, + Progressive: &clusterv1alpha1.RolloutProgressive{ + RolloutConfig: clusterv1alpha1.RolloutConfig{ + ProgressDeadline: "90s", + MaxFailures: intstr.FromString("100%"), + MinSuccessTime: metav1.Duration{Duration: time.Minute}, + }, + MaxConcurrency: intstr.FromInt32(4), + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_30s, // Succeeded 30s ago, still in 60s soak period + }, + { + ClusterGroup: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: applying, + LastTransitionTime: &fakeTime_60s, // Progressing 60s ago, will timeout in 30s + }, + { + ClusterGroup: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_30s, // Progressing 30s ago, will timeout in 60s + }, + { + ClusterGroup: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster4", + State: done, + LastTransitionTime: &fakeTime_120s, // Succeeded 120s ago, outside 60s soak period + }, + }, + expectRolloutResult: RolloutResult{ + ClustersToRollout: []ClusterRolloutStatus{ + // cluster1: in soak period (30s into 60s soak), RecheckTime in 30s + {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Succeeded, LastTransitionTime: &fakeTime_30s, RecheckTime: &fakeTime30s}, + // cluster2: progressing, will timeout in 30s + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, + // cluster3: progressing, will timeout in 60s + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_30s, RecheckTime: &fakeTime60s}, + // cluster4: succeeded and outside soak period, not included + }, + MaxFailureBreach: false, + RecheckAfter: &recheck30Duration, // Min of cluster1 soak (30s) and cluster2 timeout (30s) }, }, { @@ -605,11 +672,11 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, RecheckAfter: &recheck30Duration, }, @@ -661,8 +728,8 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_30s, TimeOutTime: &fakeTime_30s}, - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_30s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_30s, RecheckTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_30s, RecheckTime: &fakeTime_30s}, }, }, }, @@ -719,10 +786,10 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, MaxFailureBreach: true, RecheckAfter: &recheck30Duration, @@ -874,10 +941,10 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, MaxFailureBreach: true, RecheckAfter: &recheck30Duration, @@ -1076,7 +1143,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, }, @@ -1124,10 +1191,10 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, MaxFailureBreach: true, RecheckAfter: &recheck30Duration, @@ -1180,7 +1247,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { {ClusterName: "cluster6", GroupKey: clusterv1beta1sdk.GroupKey{GroupIndex: 1}, Status: ToApply}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, }, }, @@ -1249,7 +1316,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { {ClusterName: "cluster9", GroupKey: clusterv1beta1sdk.GroupKey{GroupIndex: 2}, Status: ToApply}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, }, }, @@ -1448,8 +1515,8 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, MaxFailureBreach: true, }, @@ -1506,7 +1573,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, MaxFailureBreach: true, }, @@ -1569,6 +1636,65 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { }, }, }, + { + name: "test progressivePerGroup rollout with minSuccessTime and mixed cluster states", + rolloutStrategy: clusterv1alpha1.RolloutStrategy{ + Type: clusterv1alpha1.ProgressivePerGroup, + ProgressivePerGroup: &clusterv1alpha1.RolloutProgressivePerGroup{ + RolloutConfig: clusterv1alpha1.RolloutConfig{ + ProgressDeadline: "90s", + MaxFailures: intstr.FromString("50%"), + MinSuccessTime: metav1.Duration{Duration: time.Minute}, + }, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1sdk.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "group2", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &clusterv1alpha1.RolloutStrategy{ + Type: clusterv1alpha1.ProgressivePerGroup, + ProgressivePerGroup: &clusterv1alpha1.RolloutProgressivePerGroup{ + RolloutConfig: clusterv1alpha1.RolloutConfig{ + ProgressDeadline: "90s", + MaxFailures: intstr.FromString("50%"), + MinSuccessTime: metav1.Duration{Duration: time.Minute}, + }, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_30s, // Succeeded 30s ago, in soak period + }, + { + ClusterGroup: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: done, + LastTransitionTime: &fakeTime_120s, // Succeeded 120s ago, outside soak period + }, + { + ClusterGroup: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_30s, // Progressing 30s ago + }, + }, + expectRolloutResult: RolloutResult{ + ClustersToRollout: []ClusterRolloutStatus{ + // cluster1: in soak period (30s into 60s soak) + {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Succeeded, LastTransitionTime: &fakeTime_30s, RecheckTime: &fakeTime30s}, + // cluster2: succeeded and outside soak period, not included + // cluster3: progressing, will timeout in 60s + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_30s, RecheckTime: &fakeTime60s}, + }, + MaxFailureBreach: false, + RecheckAfter: func() *time.Duration { d := 30 * time.Second; return &d }(), + }, + }, } // Set the fake time for testing @@ -1670,14 +1796,14 @@ func TestGetRolloutCluster_ClusterAdded(t *testing.T) { expectRolloutStrategy: &clusterv1alpha1.RolloutStrategy{Type: clusterv1alpha1.All, All: &clusterv1alpha1.RolloutAll{RolloutConfig: clusterv1alpha1.RolloutConfig{ProgressDeadline: "90s"}}}, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, {ClusterName: "cluster6", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, {ClusterName: "cluster7", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, RecheckAfter: &recheck30Time, }, @@ -1748,8 +1874,8 @@ func TestGetRolloutCluster_ClusterAdded(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 2}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 2}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, {ClusterName: "cluster7", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, }, RecheckAfter: &recheck30Time, @@ -1802,8 +1928,8 @@ func TestGetRolloutCluster_ClusterAdded(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, }, RecheckAfter: &recheck30Time, @@ -1910,10 +2036,10 @@ func TestGetRolloutCluster_ClusterRemoved(t *testing.T) { expectRolloutStrategy: &clusterv1alpha1.RolloutStrategy{Type: clusterv1alpha1.All, All: &clusterv1alpha1.RolloutAll{RolloutConfig: clusterv1alpha1.RolloutConfig{ProgressDeadline: "90s"}}}, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, ClustersRemoved: []ClusterRolloutStatus{ {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_120s}, @@ -1965,11 +2091,11 @@ func TestGetRolloutCluster_ClusterRemoved(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, {ClusterName: "cluster4", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, ClustersRemoved: []ClusterRolloutStatus{ {ClusterName: "cluster1", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Succeeded, LastTransitionTime: &fakeTime_60s}, @@ -2079,7 +2205,7 @@ func TestGetRolloutCluster_ClusterRemoved(t *testing.T) { }, expectRolloutResult: RolloutResult{ ClustersToRollout: []ClusterRolloutStatus{ - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime30s}, }, ClustersRemoved: []ClusterRolloutStatus{ {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s}, @@ -2133,7 +2259,7 @@ func TestGetRolloutCluster_ClusterRemoved(t *testing.T) { {ClusterName: "cluster6", GroupKey: clusterv1beta1sdk.GroupKey{GroupIndex: 1}, Status: ToApply}, }, ClustersTimeOut: []ClusterRolloutStatus{ - {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, RecheckTime: &fakeTime_30s}, }, ClustersRemoved: []ClusterRolloutStatus{ {ClusterName: "cluster2", GroupKey: clusterv1beta1sdk.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s}, @@ -2286,7 +2412,12 @@ func TestDetermineRolloutStatus(t *testing.T) { name: "Succeeded status within the minSuccessTime", clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: Succeeded}, minSuccessTime: time.Minute, - expectRolloutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: Succeeded}}, + expectRolloutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: Succeeded, RecheckTime: &fakeTime60s}}, + }, + { + name: "Succeeded status outside the minSuccessTime", + clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: Succeeded, LastTransitionTime: &fakeTime_120s}, + minSuccessTime: time.Minute, }, { name: "TimeOut status", @@ -2297,25 +2428,31 @@ func TestDetermineRolloutStatus(t *testing.T) { name: "Progressing status within the timeout duration", clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: Progressing, LastTransitionTime: &fakeTime_30s}, timeout: time.Minute, - expectRolloutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: Progressing, LastTransitionTime: &fakeTime_30s, TimeOutTime: &fakeTime30s}}, + expectRolloutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: Progressing, LastTransitionTime: &fakeTime_30s, RecheckTime: &fakeTime30s}}, }, { name: "Failed status outside of the timeout duration", clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: Failed, LastTransitionTime: &fakeTime_60s}, timeout: time.Minute, - expectTimeOutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: TimeOut, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime}}, + expectTimeOutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: TimeOut, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime}}, }, { name: "unknown status outside of the timeout duration", clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: 8, LastTransitionTime: &fakeTime_60s}, timeout: time.Minute, - expectTimeOutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: TimeOut, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime}}, + expectTimeOutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: TimeOut, LastTransitionTime: &fakeTime_60s, RecheckTime: &fakeTime}}, }, { name: "unknown status within the timeout duration", clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: 9, LastTransitionTime: &fakeTime_30s}, timeout: time.Minute, - expectRolloutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: 9, LastTransitionTime: &fakeTime_30s, TimeOutTime: &fakeTime30s}}, + expectRolloutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: 9, LastTransitionTime: &fakeTime_30s, RecheckTime: &fakeTime30s}}, + }, + { + name: "Succeeded status with nil LastTransitionTime and minSuccessTime", + clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: Succeeded}, + minSuccessTime: 2 * time.Minute, + expectRolloutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: Succeeded, RecheckTime: func() *metav1.Time { t := metav1.NewTime(fakeTime.Add(2 * time.Minute)); return &t }()}}, }, } @@ -2334,6 +2471,82 @@ func TestDetermineRolloutStatus(t *testing.T) { } } +func TestMinRecheckAfter(t *testing.T) { + RolloutClock = testingclock.NewFakeClock(fakeTime.Time) + + testCases := []struct { + name string + rolloutClusters []ClusterRolloutStatus + expectedRecheckAfter *time.Duration + }{ + { + name: "Single cluster with RecheckTime in 30s", + rolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", Status: Progressing, RecheckTime: &fakeTime30s}, + }, + expectedRecheckAfter: func() *time.Duration { d := 30 * time.Second; return &d }(), + }, + { + name: "Multiple clusters, minimum RecheckTime should be selected", + rolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", Status: Succeeded, RecheckTime: &fakeTime60s}, // 60s from now + {ClusterName: "cluster2", Status: Progressing, RecheckTime: &fakeTime30s}, // 30s from now (min) + {ClusterName: "cluster3", Status: Progressing, RecheckTime: &fakeTime60s}, // 60s from now + }, + expectedRecheckAfter: func() *time.Duration { d := 30 * time.Second; return &d }(), + }, + { + name: "Mixed timeout and soak period scenarios", + rolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", Status: Succeeded, RecheckTime: &fakeTime30s}, // Soak period ends in 30s + {ClusterName: "cluster2", Status: Progressing, RecheckTime: &fakeTime30s}, // Timeout in 30s + {ClusterName: "cluster3", Status: Progressing, RecheckTime: &fakeTime60s}, // Timeout in 60s + }, + expectedRecheckAfter: func() *time.Duration { d := 30 * time.Second; return &d }(), + }, + { + name: "No RecheckTime set, should return nil", + rolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", Status: ToApply}, + {ClusterName: "cluster2", Status: ToApply}, + }, + expectedRecheckAfter: nil, + }, + { + name: "No clusters, should return nil", + rolloutClusters: []ClusterRolloutStatus{}, + expectedRecheckAfter: nil, + }, + { + name: "Only clusters with RecheckTime are considered", + rolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", Status: ToApply}, // No RecheckTime + {ClusterName: "cluster2", Status: Progressing, RecheckTime: &fakeTime60s}, + {ClusterName: "cluster3", Status: ToApply}, // No RecheckTime + }, + expectedRecheckAfter: func() *time.Duration { d := 60 * time.Second; return &d }(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := minRecheckAfter(tc.rolloutClusters) + + if tc.expectedRecheckAfter == nil { + if result != nil { + t.Errorf("Expected nil RecheckAfter, got: %v", *result) + } + } else { + if result == nil { + t.Errorf("Expected RecheckAfter: %v, got nil", *tc.expectedRecheckAfter) + } else if *result != *tc.expectedRecheckAfter { + t.Errorf("Expected RecheckAfter: %v, got: %v", *tc.expectedRecheckAfter, *result) + } + } + }) + } +} + func TestCalculateRolloutSize(t *testing.T) { total := 100