diff --git a/scripts/test-runner.sh b/scripts/test-runner.sh index 5ce83cc292..495d85254a 100755 --- a/scripts/test-runner.sh +++ b/scripts/test-runner.sh @@ -44,7 +44,7 @@ fi # Build ginkgo command -cmd="${GINKGO} -timeout=24h --keep-going --require-suite -r" +cmd="${GINKGO} -timeout=24h --keep-going --require-suite --randomize-all -r" if [[ "${ECO_TEST_VERBOSE}" == "true" ]]; then cmd+=" -vv" diff --git a/tests/far-operator/README.md b/tests/far-operator/README.md index 435a86bb11..5463c0cb6b 100644 --- a/tests/far-operator/README.md +++ b/tests/far-operator/README.md @@ -114,3 +114,73 @@ Validates that the FAR controller container image ships the minimum expected set - **Environment**: Connected or disconnected - **Standalone**: `ginkgo --label-filter="far" --focus="fence agents" ./tests/far-operator/...` - **Pass criteria**: All expected fence agent binaries are present in the container + +## Destructive Tests + +Tests that trigger node fencing via `fence_aws` and cause node reboots. Require AWS IPI cluster with 3+ worker nodes and AWS fencing credentials. + +### 10. Verify Standalone FAR Remediation ([OCP-61229](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-61229)) + +Creates a FenceAgentsRemediation CR targeting a worker node. Validates that the fence agent reboots the node and the node object is preserved (not re-created). + +- **Operators**: FAR v0.8.0+ +- **Cluster**: AWS IPI, 3+ worker nodes +- **Storage**: None +- **Environment**: Connected +- **Standalone**: `ginkgo --label-filter="far && disruption:destructive" --focus="standalone FAR CR" ./tests/far-operator/...` +- **Pass criteria**: Node boot ID changes, node creation timestamp unchanged, node returns to Ready + +### 11. Verify Remediation on Active Controller Node ([OCP-70638](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-70638)) + +Creates a FAR CR targeting the node hosting the active FAR controller pod. Validates that controller failover occurs and remediation completes despite the leader being fenced. + +- **Operators**: FAR v0.8.0+ +- **Cluster**: AWS IPI, 3+ worker nodes +- **Storage**: None +- **Environment**: Connected +- **Standalone**: `ginkgo --label-filter="far && disruption:destructive" --focus="active FAR controller" ./tests/far-operator/...` +- **Pass criteria**: Node reboots, node returns to Ready, FAR controller replicas recover + +### 12. Verify FAR NoSchedule Taint During Remediation ([OCP-65960](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-65960)) + +Creates a FAR CR and verifies that the FAR NoSchedule taint is applied to the target node during the remediation process. + +- **Operators**: FAR v0.8.0+ +- **Cluster**: AWS IPI, 3+ worker nodes +- **Storage**: None +- **Environment**: Connected +- **Standalone**: `ginkgo --label-filter="far && disruption:destructive" --focus="NoSchedule taint" ./tests/far-operator/...` +- **Pass criteria**: FAR taint `remediation.medik8s.io/fence-agents-remediation:NoSchedule` applied during remediation + +### 13. Verify FAR CR Status Conditions After Remediation ([OCP-67015](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-67015)) + +Creates a FAR CR and after remediation completes, verifies the CR status conditions match the expected terminal state: Processing=False, FenceAgentActionSucceeded=True, Succeeded=True. + +- **Operators**: FAR v0.8.0+ +- **Cluster**: AWS IPI, 3+ worker nodes +- **Storage**: None +- **Environment**: Connected +- **Standalone**: `ginkgo --label-filter="far && disruption:destructive" --focus="status conditions" ./tests/far-operator/...` +- **Pass criteria**: All three FAR CR conditions present with expected values + +### 14. Verify FAR Default Reboot Action ([OCP-66203](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-66203)) + +Creates a FAR CR without the `--action` parameter in shared parameters. Validates that FAR defaults to the reboot action and the node is successfully rebooted. + +- **Operators**: FAR v0.8.0+ +- **Cluster**: AWS IPI, 3+ worker nodes +- **Storage**: None +- **Environment**: Connected +- **Standalone**: `ginkgo --label-filter="far && disruption:destructive" --focus="action is omitted" ./tests/far-operator/...` +- **Pass criteria**: Node reboots despite no explicit action parameter + +### 15. Verify Controller Leadership Handover ([OCP-70636](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-70636)) + +Deletes the active FAR controller pod and validates that a new pod acquires the controller lease. This test does not fence any nodes; it verifies leader election recovery only. + +- **Operators**: FAR v0.8.0+ +- **Cluster**: Multi-node, 2+ controller replicas +- **Storage**: None +- **Environment**: Connected or disconnected +- **Standalone**: `ginkgo --label-filter="far" --focus="controller leadership" ./tests/far-operator/...` +- **Pass criteria**: FAR deployment becomes ready, controller lease is held by a different pod diff --git a/tests/far-operator/far_suite_test.go b/tests/far-operator/far_suite_test.go index a4df9eb85f..3c95cbadec 100644 --- a/tests/far-operator/far_suite_test.go +++ b/tests/far-operator/far_suite_test.go @@ -1,12 +1,17 @@ package far import ( + "context" "runtime" "testing" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/medik8s/system-tests/tests/far-operator/internal/farparams" _ "github.com/medik8s/system-tests/tests/far-operator/tests" . "github.com/medik8s/system-tests/tests/internal/medik8sinittools" + "github.com/medik8s/system-tests/tests/internal/medik8sparams" "github.com/medik8s/system-tests/tests/internal/reporter" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -28,6 +33,17 @@ var _ = JustAfterEach(func() { CurrentSpecReport(), currentFile, farparams.ReporterNamespacesToDump, farparams.ReporterCRDsToDump) }) +var _ = AfterSuite(func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: farparams.SharedCredentialsSecretName, + Namespace: medik8sparams.OperatorNs, + }, + } + + _ = APIClient.Delete(context.Background(), secret) +}) + var _ = ReportAfterSuite("", func(report Report) { reportxml.Create( report, Medik8sConfig.GetReportPath(), Medik8sConfig.TCPrefix) diff --git a/tests/far-operator/internal/farparams/const.go b/tests/far-operator/internal/farparams/const.go index 1fefa0df75..1adf6dbc56 100644 --- a/tests/far-operator/internal/farparams/const.go +++ b/tests/far-operator/internal/farparams/const.go @@ -77,4 +77,40 @@ const ( // ControllerLeaseName is the FAR leader election lease name (LeaderElectionID in cmd/main.go). ControllerLeaseName = "cb305759.medik8s.io" + + // FARConditionProcessing is the condition type for remediation progress. + FARConditionProcessing = "Processing" + // FARConditionFenceAgentSucceeded is the condition type for fence agent action result. + FARConditionFenceAgentSucceeded = "FenceAgentActionSucceeded" + // FARConditionSucceeded is the condition type for overall remediation outcome. + FARConditionSucceeded = "Succeeded" + + // FARNoScheduleTaintKey is the taint key applied by FAR during remediation. + FARNoScheduleTaintKey = "remediation.medik8s.io/fence-agents-remediation" + + // ControllerHandoverTimeout is how long to wait for controller leadership transfer. + ControllerHandoverTimeout = 3 * time.Minute + // WorkloadEvictionTimeout is how long to wait for workload pods to be evicted. + WorkloadEvictionTimeout = 5 * time.Minute + // WorkloadPodReadyTimeout is how long to wait for a test workload pod to reach Running. + WorkloadPodReadyTimeout = 2 * time.Minute + + // FARCRRetryCount is the retry count for FAR/FART CR spec (matches upstream default). + FARCRRetryCount = 10 + // FARCRRetryInterval is the retry interval for FAR/FART CR spec. + FARCRRetryInterval = "20s" + // FARCRTimeout is the fence agent command timeout for FAR/FART CR spec. + FARCRTimeout = "60s" + // FARCRRemediationStrategy is the default remediation strategy for FAR CRs. + FARCRRemediationStrategy = "OutOfServiceTaint" + + // CrioCleanupTimeout is the timeout for the post-remediation CRI-O overlay cleanup. + CrioCleanupTimeout = 2 * time.Minute + + // SharedCredentialsSecretName is the Secret created by the test suite to hold + // fence agent credentials in the format expected by SharedSecretName. + SharedCredentialsSecretName = "far-test-shared-credentials" + + // WorkloadTestImage is the container image used for test workload pods. + WorkloadTestImage = "registry.access.redhat.com/ubi9/ubi-minimal" ) diff --git a/tests/far-operator/tests/far.go b/tests/far-operator/tests/far.go index 3e49ff160c..ccbab3d592 100644 --- a/tests/far-operator/tests/far.go +++ b/tests/far-operator/tests/far.go @@ -65,23 +65,17 @@ var _ = Describe( LabelSelector: farparams.OperatorControllerPodLabelSelector, } - _, err := pod.WaitForAllPodsInNamespaceRunning( - APIClient, - medik8sparams.OperatorNs, - medik8sparams.DefaultTimeout, - listOptions, - ) - Expect(err).ToNot(HaveOccurred(), "Pod is not ready") + By("Waiting for expected number of Running FAR pods") - By("Verifying pod count matches expected replicas") + Eventually(func(assertion Gomega) { + farPods, err := pod.List(APIClient, medik8sparams.OperatorNs, listOptions) + assertion.Expect(err).ToNot(HaveOccurred(), "Failed to list FAR pods") - farPods, err := pod.List(APIClient, medik8sparams.OperatorNs, listOptions) - Expect(err).ToNot(HaveOccurred(), "Failed to list FAR pods") - - runningPods := helpers.FilterRunningPods(farPods) + runningPods := helpers.FilterRunningPods(farPods) - Expect(int32(len(runningPods))).To(Equal(expectedCount), - "Expected %d running FAR pod(s), found %d", expectedCount, len(runningPods)) + assertion.Expect(int32(len(runningPods))).To(Equal(expectedCount), + "Expected %d running FAR pod(s), found %d", expectedCount, len(runningPods)) + }, medik8sparams.DefaultTimeout, farparams.DefaultPollInterval).Should(Succeed()) }) It("Verify FAR CSV has required annotations", diff --git a/tests/far-operator/tests/far_controller_lifecycle.go b/tests/far-operator/tests/far_controller_lifecycle.go new file mode 100644 index 0000000000..713759490c --- /dev/null +++ b/tests/far-operator/tests/far_controller_lifecycle.go @@ -0,0 +1,118 @@ +package tests + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/rh-ecosystem-edge/eco-goinfra/pkg/deployment" + "github.com/rh-ecosystem-edge/eco-goinfra/pkg/reportxml" + + "github.com/medik8s/system-tests/tests/far-operator/internal/farparams" + "github.com/medik8s/system-tests/tests/far-operator/internal/farutils" + "github.com/medik8s/system-tests/tests/internal/labels" + . "github.com/medik8s/system-tests/tests/internal/medik8sinittools" + "github.com/medik8s/system-tests/tests/internal/medik8sparams" +) + +var _ = Describe("FAR Controller Lifecycle Tests", + Serial, + Label(labels.OperatorFAR, farparams.Label, + labels.DisruptionNonDestructive), + func() { + var ctx context.Context + + BeforeEach(func() { + ctx = context.Background() + + By("Verifying FAR controller deployment is Ready") + + farDeployment, err := deployment.Pull( + APIClient, farparams.OperatorDeploymentName, medik8sparams.OperatorNs) + Expect(err).ToNot(HaveOccurred(), "Failed to get FAR deployment") + Expect(farDeployment.IsReady(medik8sparams.DefaultTimeout)).To(BeTrue(), + "FAR deployment is not Ready") + }) + + It("should transfer controller leadership when the active pod is deleted", + Label(labels.TierAcceptance, labels.ComponentController), + reportxml.ID("OCP-70636"), + func() { + By("Getting the current active FAR controller pod") + + pods, err := farutils.GetFARControllerPods(ctx, APIClient) + Expect(err).ToNot(HaveOccurred()) + Expect(pods).ToNot(BeEmpty(), "No running FAR controller pods found") + + oldLeaderNode, err := farutils.GetActiveFARControllerNode(ctx, APIClient) + Expect(err).ToNot(HaveOccurred()) + + var oldLeaderPod *corev1.Pod + + for i := range pods { + if pods[i].Spec.NodeName == oldLeaderNode { + oldLeaderPod = &pods[i] + + break + } + } + + Expect(oldLeaderPod).ToNot(BeNil(), + "Could not find controller pod on leader node %s", oldLeaderNode) + + oldPodName := oldLeaderPod.Name + GinkgoWriter.Printf("Active controller pod: %s on node %s\n", + oldPodName, oldLeaderNode) + + By("Deleting the active controller pod " + oldPodName) + + Expect(APIClient.Delete(ctx, oldLeaderPod)).To(Succeed()) + + By("Waiting for FAR controller deployment to become ready") + + farDeployment, err := deployment.Pull( + APIClient, farparams.OperatorDeploymentName, medik8sparams.OperatorNs) + Expect(err).ToNot(HaveOccurred(), "Failed to pull FAR controller deployment") + Expect(farDeployment.IsReady(farparams.ControllerHandoverTimeout)).To(BeTrue(), + "FAR deployment did not become ready after pod deletion") + + By("Verifying controller lease transferred to a different pod") + + Eventually(func(assertion Gomega) { + lease := &coordinationv1.Lease{} + assertion.Expect(APIClient.Get(ctx, client.ObjectKey{ + Name: farparams.ControllerLeaseName, + Namespace: medik8sparams.OperatorNs, + }, lease)).To(Succeed()) + assertion.Expect(lease.Spec.HolderIdentity).ToNot(BeNil(), + "Lease has no holder after pod deletion") + + if lease.Spec.HolderIdentity != nil { + assertion.Expect(*lease.Spec.HolderIdentity).ToNot(Equal(oldPodName), + "Lease is still held by deleted pod %s", oldPodName) + } + + newPods, err := farutils.GetFARControllerPods(ctx, APIClient) + assertion.Expect(err).ToNot(HaveOccurred()) + + hasNewRunningPod := false + + for _, p := range newPods { + if p.Name != oldPodName && p.Status.Phase == corev1.PodRunning { + hasNewRunningPod = true + + break + } + } + + assertion.Expect(hasNewRunningPod).To(BeTrue(), + "No new Running controller pod found after deleting %s", oldPodName) + }, farparams.ControllerHandoverTimeout, farparams.DefaultPollInterval).Should(Succeed(), + "Controller leadership did not transfer after pod deletion") + }) + }) diff --git a/tests/far-operator/tests/far_destructive.go b/tests/far-operator/tests/far_destructive.go index 59e467e553..9f445358ba 100644 --- a/tests/far-operator/tests/far_destructive.go +++ b/tests/far-operator/tests/far_destructive.go @@ -8,14 +8,17 @@ import ( . "github.com/onsi/gomega" configv1 "github.com/openshift/api/config/v1" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/rh-ecosystem-edge/eco-goinfra/pkg/deployment" + "github.com/rh-ecosystem-edge/eco-goinfra/pkg/reportxml" "github.com/medik8s/system-tests/tests/far-operator/internal/farparams" "github.com/medik8s/system-tests/tests/far-operator/internal/farutils" @@ -38,7 +41,7 @@ var fartGVK = schema.GroupVersionKind{ } var _ = Describe("FAR Destructive Tests", - Serial, Ordered, ContinueOnFailure, + Serial, Label(labels.OperatorFAR, farparams.Label, labels.DisruptionDestructive, labels.PlatformAWS, labels.FrequencyWeekly), @@ -48,18 +51,26 @@ var _ = Describe("FAR Destructive Tests", platform configv1.PlatformType region string fenceAgent string - nodeIDParam string - awsAccessKey string - awsSecretKey string leaderNode string targetNode *corev1.Node sharedParams map[string]interface{} nodeParams map[string]interface{} currentFARTName string currentFARName string + + destructiveSetupDone bool + destructiveSetupSkipped bool ) - BeforeAll(func() { + BeforeEach(func() { + if destructiveSetupSkipped { + Skip("FAR destructive tests require AWS") + } + + if destructiveSetupDone { + return + } + ctx = context.Background() By("Detecting cluster platform") @@ -70,13 +81,15 @@ var _ = Describe("FAR Destructive Tests", Expect(err).ToNot(HaveOccurred()) if platform != configv1.AWSPlatformType { + destructiveSetupSkipped = true + Skip(fmt.Sprintf( "FAR destructive tests require AWS, got %s", platform)) } By("Resolving fence agent for platform") - fenceAgent, nodeIDParam, err = farutils.FenceAgentForPlatform(platform) + fenceAgent, _, err = farutils.FenceAgentForPlatform(platform) Expect(err).ToNot(HaveOccurred()) GinkgoWriter.Printf( "Platform: %s, Agent: %s, Region: %s\n", @@ -102,20 +115,37 @@ var _ = Describe("FAR Destructive Tests", By("Reading AWS credentials from CCO Secret") - awsAccessKey, awsSecretKey, err = farutils.GetAWSCredentials( + awsAccessKey, awsSecretKey, err := farutils.GetAWSCredentials( ctx, APIClient, medik8sparams.OperatorNs) Expect(err).ToNot(HaveOccurred(), "AWS credentials must be provisioned by the "+ "medik8s-aws-credentials CI step") + By("Creating shared credentials Secret for FAR SharedSecretName") + + credentialsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: farparams.SharedCredentialsSecretName, + Namespace: medik8sparams.OperatorNs, + }, + StringData: map[string]string{ + "--access-key": awsAccessKey, + "--secret-key": awsSecretKey, + }, + } + + err = APIClient.Create(ctx, credentialsSecret) + if err != nil && !k8serrors.IsAlreadyExists(err) { + Expect(err).ToNot(HaveOccurred(), + "Failed to create shared credentials Secret") + } + By("Building fence_aws shared parameters") sharedParams = map[string]interface{}{ "--region": region, "--action": "reboot", "--skip-race-check": "", - "--access-key": awsAccessKey, - "--secret-key": awsSecretKey, } By("Building node parameters (--plug = EC2 instance ID)") @@ -137,28 +167,109 @@ var _ = Describe("FAR Destructive Tests", By("Identifying active FAR controller node") - leaderNode, err = farutils.GetActiveFARControllerNode( - ctx, APIClient) - Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { + var leaderErr error + + leaderNode, leaderErr = farutils.GetActiveFARControllerNode(ctx, APIClient) + + return leaderErr + }, farparams.ControllerHandoverTimeout, farparams.DefaultPollInterval).Should(Succeed(), + "FAR leader election did not settle") GinkgoWriter.Printf("FAR leader is on node: %s\n", leaderNode) - // TODO(RHWA-963): remove when destructive test specs consume these variables. - _ = nodeIDParam - _ = sharedParams - _ = nodeParams + destructiveSetupDone = true }) JustAfterEach(func() { spec := CurrentSpecReport() if spec.Failed() { GinkgoWriter.Println( - "Test failed - running safety net cleanup") + "Test failed - collecting diagnostics") + logFARControllerState(ctx, APIClient) } if currentFARName != "" { - By("Safety net: deleting FAR CR " + currentFARName) + By("Waiting for FAR CR to reach Succeeded before cleanup") + + pollCtx, pollCancel := context.WithTimeout(ctx, farparams.FARConditionTimeout) + defer pollCancel() + + if waitErr := wait.PollUntilContextCancel(pollCtx, farparams.DefaultPollInterval, true, + func(ctx context.Context) (bool, error) { + farObj := &unstructured.Unstructured{} + farObj.SetGroupVersionKind(farGVK) + + if err := APIClient.Get(ctx, client.ObjectKey{ + Name: currentFARName, + Namespace: medik8sparams.OperatorNs, + }, farObj); err != nil { + return false, nil + } + + conditions, found, condErr := unstructured.NestedSlice( + farObj.Object, "status", "conditions") + if condErr != nil { + GinkgoWriter.Printf( + "WARNING: failed to read FAR CR conditions: %v\n", condErr) + + return false, nil + } + + if !found { + return false, nil + } + + for _, c := range conditions { + condMap, ok := c.(map[string]interface{}) + if !ok { + continue + } + + if condMap["type"] == farparams.FARConditionSucceeded && + condMap["status"] == string(metav1.ConditionTrue) { + return true, nil + } + } + + return false, nil + }, + ); waitErr != nil { + GinkgoWriter.Printf( + "WARNING: FAR CR %s did not reach Succeeded within %s: %v\n", + currentFARName, farparams.FARConditionTimeout, waitErr) + } + + By("Deleting FAR CR " + currentFARName) + farNodeName := currentFARName deleteRemediationCR(ctx, APIClient, farGVK, currentFARName) currentFARName = "" + + By("Verifying FAR NoSchedule taint removed after CR deletion") + + taintCtx, taintCancel := context.WithTimeout(ctx, farparams.FARConditionTimeout) + defer taintCancel() + + if taintErr := wait.PollUntilContextCancel(taintCtx, farparams.DefaultPollInterval, true, + func(ctx context.Context) (bool, error) { + node := &corev1.Node{} + if err := APIClient.Get(ctx, client.ObjectKey{Name: farNodeName}, node); err != nil { + return false, nil + } + + for _, taint := range node.Spec.Taints { + if taint.Key == farparams.FARNoScheduleTaintKey { + return false, nil + } + } + + return true, nil + }, + ); taintErr != nil { + GinkgoWriter.Printf( + "WARNING: FAR taint %s still present on node %s after %s: %v\n", + farparams.FARNoScheduleTaintKey, farNodeName, + farparams.FARConditionTimeout, taintErr) + } } if currentFARTName != "" { @@ -171,29 +282,386 @@ var _ = Describe("FAR Destructive Tests", nodeName := targetNode.Name targetNode = nil - By("Safety net: ensuring kubelet is running on " + nodeName) - Expect(farutils.StartKubelet(ctx, nodeName)).To(Succeed(), - "safety net: failed to restart kubelet on %s", nodeName) + By("Safety net: waiting for node " + nodeName + " to become Ready") - By("Safety net: waiting for node to become Ready") - Expect(farutils.WaitForNodeReady( + if err := farutils.WaitForNodeReady( ctx, APIClient, nodeName, - farparams.NodeReadyTimeout)).To(Succeed(), - "safety net: node %s did not become Ready", nodeName) + farparams.NodeReadyTimeout); err != nil { + GinkgoWriter.Printf( + "WARNING: safety net: node %s did not become Ready within %s: %v\n", + nodeName, farparams.NodeReadyTimeout, err) + AddReportEntry("safety-net-recovery-failed", + fmt.Sprintf("node %s did not recover: %v", nodeName, err)) + } } }) Context("Standalone FAR remediation", func() { - // RHWA-963: 7 standalone destructive tests will be added here. - // Each test follows this flow: - // 1. Select target worker (exclude leader node) - // 2. Record boot ID - // 3. Create FART + deploy workload pod - // 4. Stop kubelet (simulate unhealthy node) - // 5. WaitForNodeNotReady (verify kubelet actually stopped) - // 6. Create FAR CR (trigger remediation) - // 7. Verify: taint applied, node rebooted, pod evicted - // 8. Cleanup via DeferCleanup + JustAfterEach safety net + BeforeEach(func() { + By("Verifying FAR controller is Ready before test") + + farDeployment, err := deployment.Pull( + APIClient, farparams.OperatorDeploymentName, medik8sparams.OperatorNs) + Expect(err).ToNot(HaveOccurred()) + Expect(farDeployment.IsReady(medik8sparams.DefaultTimeout)).To(BeTrue(), + "FAR controller is not Ready - webhook will be unreachable") + + By("Finding active leader node") + + Eventually(func() error { + var leaderErr error + + leaderNode, leaderErr = farutils.GetActiveFARControllerNode(ctx, APIClient) + + return leaderErr + }, farparams.ControllerHandoverTimeout, farparams.DefaultPollInterval).Should(Succeed(), + "FAR leader election did not settle - lease may point to a replaced pod") + GinkgoWriter.Printf("FAR controller Ready, leader on node: %s\n", leaderNode) + }) + + Context("non-leader worker target", func() { + var ( + oldBootID string + workloadPod *corev1.Pod + ) + + BeforeEach(func() { + By("Selecting a non-leader worker node") + + var err error + + targetNode, err = helpers.SelectWorkerNode(ctx, APIClient, leaderNode) + Expect(err).ToNot(HaveOccurred()) + + By("Cleaning CRI-O overlay storage on " + targetNode.Name) + removeWorkloadImage(ctx, targetNode.Name) + + By("Recording boot ID before remediation") + + oldBootID, err = farutils.GetNodeBootIDFromAPI(ctx, APIClient, targetNode.Name) + Expect(err).ToNot(HaveOccurred()) + + By("Creating a test workload pod pinned to " + targetNode.Name) + + workloadPod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "far-workload-test-", + Namespace: medik8sparams.OperatorNs, + }, + Spec: corev1.PodSpec{ + NodeName: targetNode.Name, + RestartPolicy: corev1.RestartPolicyAlways, + Containers: []corev1.Container{{ + Name: "workload", + Image: farparams.WorkloadTestImage, + Command: []string{"sleep", "infinity"}, + }}, + }, + } + + Expect(APIClient.Create(ctx, workloadPod)).To(Succeed()) + DeferCleanup(func() { + _ = APIClient.Delete(ctx, workloadPod) + }) + + By("Waiting for workload pod to be Running") + + Eventually(func() corev1.PodPhase { + pod := &corev1.Pod{} + if err := APIClient.Get(ctx, client.ObjectKey{ + Name: workloadPod.Name, Namespace: workloadPod.Namespace, + }, pod); err != nil { + return corev1.PodPending + } + + return pod.Status.Phase + }, farparams.WorkloadPodReadyTimeout, farparams.DefaultPollInterval).Should(Equal(corev1.PodRunning)) + }) + + JustAfterEach(func() { + if CurrentSpecReport().Failed() { + logPodDiagnostics(ctx, APIClient, workloadPod) + + return + } + + By("Verifying workload pod was deleted or evicted") + + Eventually(func() bool { + pod := &corev1.Pod{} + err := APIClient.Get(ctx, client.ObjectKey{ + Name: workloadPod.Name, Namespace: workloadPod.Namespace, + }, pod) + + return k8serrors.IsNotFound(err) || pod.DeletionTimestamp != nil + }, farparams.WorkloadEvictionTimeout, farparams.DefaultPollInterval).Should(BeTrue(), + "Workload pod was not deleted/evicted after remediation") + }) + + It("should remediate a worker node via standalone FAR CR", + Label(labels.TierAcceptance, labels.ComponentRemediation), + reportxml.ID("OCP-61229"), + func() { + creationTimestamp := targetNode.CreationTimestamp + + By("Creating FAR CR targeting " + targetNode.Name) + + farCR := buildFARUnstructured(targetNode.Name, fenceAgent, sharedParams, nodeParams) + createFARCR(ctx, APIClient, farCR) + + currentFARName = targetNode.Name + + waitForRemediation(ctx, APIClient, targetNode.Name, oldBootID) + + By("Verifying node was rebooted, not re-created") + + node := &corev1.Node{} + Expect(APIClient.Get(ctx, client.ObjectKey{Name: targetNode.Name}, node)).To(Succeed()) + Expect(node.CreationTimestamp.Equal(&creationTimestamp)).To(BeTrue(), + "Node creation timestamp changed - node was re-created instead of rebooted") + }) + + It("should apply FAR NoSchedule taint during remediation", + Label(labels.TierAcceptance, labels.ComponentRemediation), + reportxml.ID("OCP-65960"), + func() { + By("Creating FAR CR targeting " + targetNode.Name) + + farCR := buildFARUnstructured(targetNode.Name, fenceAgent, sharedParams, nodeParams) + createFARCR(ctx, APIClient, farCR) + + currentFARName = targetNode.Name + + By("Verifying FAR NoSchedule taint is applied to the node") + + Eventually(func(assertion Gomega) { + node := &corev1.Node{} + assertion.Expect(APIClient.Get(ctx, client.ObjectKey{Name: targetNode.Name}, node)).To(Succeed()) + + found := false + + for _, taint := range node.Spec.Taints { + if taint.Key == farparams.FARNoScheduleTaintKey && + taint.Effect == corev1.TaintEffectNoSchedule { + found = true + + break + } + } + + assertion.Expect(found).To(BeTrue(), + "FAR NoSchedule taint %s not found on node %s", + farparams.FARNoScheduleTaintKey, targetNode.Name) + }, farparams.FARConditionTimeout, farparams.DefaultPollInterval).Should(Succeed()) + + waitForRemediation(ctx, APIClient, targetNode.Name, oldBootID) + }) + + It("should report correct FAR CR status conditions after remediation", + Label(labels.TierAcceptance, labels.ComponentRemediation), + reportxml.ID("OCP-67015"), + func() { + By("Creating FAR CR targeting " + targetNode.Name) + + farCR := buildFARUnstructured(targetNode.Name, fenceAgent, sharedParams, nodeParams) + createFARCR(ctx, APIClient, farCR) + + currentFARName = targetNode.Name + + waitForRemediation(ctx, APIClient, targetNode.Name, oldBootID) + + By("Verifying FAR CR status conditions") + + expectedConditions := map[string]string{ + farparams.FARConditionProcessing: string(metav1.ConditionFalse), + farparams.FARConditionFenceAgentSucceeded: string(metav1.ConditionTrue), + farparams.FARConditionSucceeded: string(metav1.ConditionTrue), + } + + Eventually(func(assertion Gomega) { + farObj := &unstructured.Unstructured{} + farObj.SetGroupVersionKind(farGVK) + assertion.Expect(APIClient.Get(ctx, client.ObjectKey{ + Name: targetNode.Name, Namespace: medik8sparams.OperatorNs, + }, farObj)).To(Succeed()) + + conditions, found, condErr := unstructured.NestedSlice( + farObj.Object, "status", "conditions") + assertion.Expect(condErr).ToNot(HaveOccurred()) + assertion.Expect(found).To(BeTrue(), "FAR CR has no status.conditions") + + for condType, expectedStatus := range expectedConditions { + condFound := false + + for _, c := range conditions { + condMap, ok := c.(map[string]interface{}) + if !ok { + continue + } + + if condMap["type"] == condType { + condFound = true + + assertion.Expect(condMap["status"]).To(Equal(expectedStatus), + "Condition %s has unexpected status", condType) + + break + } + } + + assertion.Expect(condFound).To(BeTrue(), + "Condition %s not found in FAR CR status", condType) + } + }, farparams.FARConditionTimeout, farparams.DefaultPollInterval).Should(Succeed()) + }) + + Context("without --action parameter", func() { + It("should default to reboot action when --action is omitted", + Label(labels.TierAcceptance, labels.ComponentRemediation), + reportxml.ID("OCP-66203"), + func() { + By("Building shared parameters WITHOUT --action") + + noActionParams := make(map[string]interface{}, len(sharedParams)) + for k, v := range sharedParams { + if k != "--action" { + noActionParams[k] = v + } + } + + By("Creating FAR CR without explicit action") + + farCR := buildFARUnstructured(targetNode.Name, fenceAgent, noActionParams, nodeParams) + createFARCR(ctx, APIClient, farCR) + + currentFARName = targetNode.Name + + waitForRemediation(ctx, APIClient, targetNode.Name, oldBootID) + }) + }) + }) + + Context("leader node target", func() { + It("should remediate the node hosting the active FAR controller", + Label(labels.TierAcceptance, labels.ComponentRemediation), + reportxml.ID("OCP-70638"), + func() { + By("Targeting the active FAR controller node") + + var err error + + activeLeader, err := farutils.GetActiveFARControllerNode(ctx, APIClient) + Expect(err).ToNot(HaveOccurred()) + Expect(activeLeader).ToNot(BeEmpty()) + + node := &corev1.Node{} + Expect(APIClient.Get(ctx, client.ObjectKey{Name: activeLeader}, node)).To(Succeed()) + targetNode = node + + By("Cleaning CRI-O overlay storage on " + targetNode.Name) + removeWorkloadImage(ctx, targetNode.Name) + + By("Recording boot ID before remediation") + + oldBootID, err := farutils.GetNodeBootIDFromAPI(ctx, APIClient, targetNode.Name) + Expect(err).ToNot(HaveOccurred()) + + By("Creating a test workload pod pinned to " + targetNode.Name) + + workloadPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "far-workload-test-", + Namespace: medik8sparams.OperatorNs, + }, + Spec: corev1.PodSpec{ + NodeName: targetNode.Name, + RestartPolicy: corev1.RestartPolicyAlways, + Containers: []corev1.Container{{ + Name: "workload", + Image: farparams.WorkloadTestImage, + Command: []string{"sleep", "infinity"}, + }}, + }, + } + + Expect(APIClient.Create(ctx, workloadPod)).To(Succeed()) + DeferCleanup(func() { + _ = APIClient.Delete(ctx, workloadPod) + }) + + By("Waiting for workload pod to be Running") + + Eventually(func() corev1.PodPhase { + pod := &corev1.Pod{} + if err := APIClient.Get(ctx, client.ObjectKey{ + Name: workloadPod.Name, Namespace: workloadPod.Namespace, + }, pod); err != nil { + return corev1.PodPending + } + + return pod.Status.Phase + }, farparams.WorkloadPodReadyTimeout, farparams.DefaultPollInterval).Should(Equal(corev1.PodRunning)) + + By("Recording pre-reboot lease holder for failover verification") + + preRebootLease := &coordinationv1.Lease{} + Expect(APIClient.Get(ctx, client.ObjectKey{ + Name: farparams.ControllerLeaseName, + Namespace: medik8sparams.OperatorNs, + }, preRebootLease)).To(Succeed()) + Expect(preRebootLease.Spec.HolderIdentity).ToNot(BeNil(), + "Lease has no holder before reboot") + oldLeaderHolder := *preRebootLease.Spec.HolderIdentity + GinkgoWriter.Printf("Pre-reboot lease holder: %s\n", oldLeaderHolder) + + By("Creating FAR CR targeting the active controller node " + targetNode.Name) + + farCR := buildFARUnstructured(targetNode.Name, fenceAgent, sharedParams, nodeParams) + createFARCR(ctx, APIClient, farCR) + + currentFARName = targetNode.Name + + waitForRemediation(ctx, APIClient, targetNode.Name, oldBootID) + + By("Verifying FAR controller replicas recovered") + + farDeployment, err := deployment.Pull( + APIClient, farparams.OperatorDeploymentName, medik8sparams.OperatorNs) + Expect(err).ToNot(HaveOccurred()) + Expect(farDeployment.IsReady(medik8sparams.DefaultTimeout)).To(BeTrue(), + "FAR controller replicas did not recover after leader node reboot") + + By("Verifying controller lease transferred to a different pod") + + Eventually(func(assertion Gomega) { + lease := &coordinationv1.Lease{} + assertion.Expect(APIClient.Get(ctx, client.ObjectKey{ + Name: farparams.ControllerLeaseName, + Namespace: medik8sparams.OperatorNs, + }, lease)).To(Succeed()) + assertion.Expect(lease.Spec.HolderIdentity).ToNot(BeNil(), + "Lease has no holder after leader node reboot") + + if lease.Spec.HolderIdentity != nil { + assertion.Expect(*lease.Spec.HolderIdentity).ToNot(Equal(oldLeaderHolder), + "Lease is still held by pre-reboot pod %s", oldLeaderHolder) + } + }, farparams.ControllerHandoverTimeout, farparams.DefaultPollInterval).Should(Succeed(), + "Controller lease did not transfer to a different pod after leader node reboot") + + By("Verifying workload pod was evicted from leader node") + + Eventually(func() bool { + pod := &corev1.Pod{} + err := APIClient.Get(ctx, client.ObjectKey{ + Name: workloadPod.Name, Namespace: workloadPod.Namespace, + }, pod) + + return k8serrors.IsNotFound(err) || pod.DeletionTimestamp != nil + }, farparams.WorkloadEvictionTimeout, farparams.DefaultPollInterval).Should(BeTrue(), + "Workload pod was not evicted from leader node after remediation") + }) + }) }) Context("NHC+FAR interop", func() { @@ -205,6 +673,34 @@ var _ = Describe("FAR Destructive Tests", }) }) +func buildFARUnstructured( + nodeName, agent string, + sharedParams, nodeParams map[string]interface{}, +) *unstructured.Unstructured { + spec := map[string]interface{}{ + "agent": agent, + "sharedparameters": sharedParams, + "nodeparameters": nodeParams, + "retrycount": farparams.FARCRRetryCount, + "retryinterval": farparams.FARCRRetryInterval, + "timeout": farparams.FARCRTimeout, + "remediationStrategy": farparams.FARCRRemediationStrategy, + "sharedSecretName": farparams.SharedCredentialsSecretName, + } + + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "fence-agents-remediation.medik8s.io/v1alpha1", + "kind": "FenceAgentsRemediation", + "metadata": map[string]interface{}{ + "name": nodeName, + "namespace": medik8sparams.OperatorNs, + }, + "spec": spec, + }, + } +} + //nolint:unused // scaffold helper for upcoming destructive test specs func buildFARTUnstructured( name, agent string, @@ -224,10 +720,11 @@ func buildFARTUnstructured( "agent": agent, "sharedparameters": sharedParams, "nodeparameters": nodeParams, - "retrycount": 10, - "retryinterval": "20s", - "timeout": "60s", - "remediationStrategy": "OutOfServiceTaint", + "retrycount": farparams.FARCRRetryCount, + "retryinterval": farparams.FARCRRetryInterval, + "timeout": farparams.FARCRTimeout, + "remediationStrategy": farparams.FARCRRemediationStrategy, + "sharedSecretName": farparams.SharedCredentialsSecretName, }, }, }, @@ -235,6 +732,175 @@ func buildFARTUnstructured( } } +func waitForRemediation( + ctx context.Context, k8sClient client.Client, + nodeName, oldBootID string, +) { + By("Waiting for node to reboot") + + Expect(farutils.WaitForNodeReboot( + ctx, k8sClient, nodeName, oldBootID, + farparams.NodeRebootTimeout)).To(Succeed(), + "Node %s did not reboot", nodeName) + + By("Waiting for node to become Ready") + + Expect(farutils.WaitForNodeReady( + ctx, k8sClient, nodeName, + farparams.NodeReadyTimeout)).To(Succeed(), + "Node %s did not become Ready after reboot", nodeName) +} + +func createFARCR( + ctx context.Context, k8sClient client.Client, + farCR *unstructured.Unstructured, +) { + deleteRemediationCR(ctx, k8sClient, farCR.GroupVersionKind(), + farCR.GetName()) + + Eventually(func(assertion Gomega) { + err := k8sClient.Create(ctx, farCR) + if err != nil { + if k8serrors.IsAlreadyExists(err) { + GinkgoWriter.Printf( + "INFO: FAR CR %s already exists (prior delete may not have finalized), treating as success\n", + farCR.GetName()) + + return + } + + assertion.Expect(err).ToNot(HaveOccurred(), + "Failed to create FAR CR") + } + }, farparams.FARConditionTimeout, 10*farparams.DefaultPollInterval).Should(Succeed(), + "FAR CR creation timed out - webhook may be unreachable") +} + +func logFARControllerState(ctx context.Context, k8sClient client.Client) { + pods := &corev1.PodList{} + + if err := k8sClient.List(ctx, pods, + client.InNamespace(medik8sparams.OperatorNs), + client.MatchingLabels(farparams.OperatorControllerPodLabels)); err != nil { + GinkgoWriter.Printf("WARNING: could not list controller pods: %v\n", err) + + return + } + + for i := range pods.Items { + pod := &pods.Items[i] + ready := false + + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + ready = true + + break + } + } + + GinkgoWriter.Printf("FAR controller pod %s: Phase=%s, Node=%s, Ready=%v\n", + pod.Name, pod.Status.Phase, pod.Spec.NodeName, ready) + } +} + +func logPodDiagnostics(ctx context.Context, k8sClient client.Client, pod *corev1.Pod) { + if pod == nil || pod.Name == "" { + return + } + + fresh := &corev1.Pod{} + + if err := k8sClient.Get(ctx, client.ObjectKey{ + Name: pod.Name, Namespace: pod.Namespace, + }, fresh); err != nil { + GinkgoWriter.Printf("WARNING: could not fetch pod %s for diagnostics: %v\n", + pod.Name, err) + + return + } + + GinkgoWriter.Printf("Pod %s diagnostics: Phase=%s, Node=%s\n", + fresh.Name, fresh.Status.Phase, fresh.Spec.NodeName) + + for _, cond := range fresh.Status.Conditions { + if cond.Status != corev1.ConditionTrue { + GinkgoWriter.Printf(" Condition %s=%s: %s (%s)\n", + cond.Type, cond.Status, cond.Reason, cond.Message) + } + } + + for _, ctrStatus := range fresh.Status.ContainerStatuses { + GinkgoWriter.Printf(" Container %s: Ready=%v, RestartCount=%d\n", + ctrStatus.Name, ctrStatus.Ready, ctrStatus.RestartCount) + + if ctrStatus.State.Waiting != nil { + GinkgoWriter.Printf(" Waiting: %s - %s\n", + ctrStatus.State.Waiting.Reason, ctrStatus.State.Waiting.Message) + } + + if ctrStatus.State.Terminated != nil { + GinkgoWriter.Printf(" Terminated: %s (exit %d) - %s\n", + ctrStatus.State.Terminated.Reason, + ctrStatus.State.Terminated.ExitCode, + ctrStatus.State.Terminated.Message) + } + } + + eventList := &corev1.EventList{} + + if err := k8sClient.List(ctx, eventList, + client.InNamespace(pod.Namespace)); err != nil { + GinkgoWriter.Printf("WARNING: could not list events: %v\n", err) + + return + } + + GinkgoWriter.Printf(" Events for pod %s:\n", fresh.Name) + + eventFound := false + + for i := range eventList.Items { + podEvent := &eventList.Items[i] + + if podEvent.InvolvedObject.Name != fresh.Name || + podEvent.InvolvedObject.Kind != "Pod" { + continue + } + + eventFound = true + + ts := podEvent.LastTimestamp.Format("15:04:05") + GinkgoWriter.Printf(" [%s] %s %s: %s (x%d)\n", + ts, podEvent.Type, podEvent.Reason, podEvent.Message, podEvent.Count) + } + + if !eventFound { + GinkgoWriter.Println(" (no events found)") + } +} + +func removeWorkloadImage(ctx context.Context, nodeName string) { + GinkgoWriter.Printf("Removing workload image from node %s to prevent corrupt overlay layers\n", nodeName) + + output, err := helpers.RunOnNode( + ctx, nodeName, farparams.CrioCleanupTimeout, + "bash", "-c", + "crictl rmi "+farparams.WorkloadTestImage+" 2>/dev/null; "+ + "echo done", + ) + if err != nil { + GinkgoWriter.Printf( + "WARNING: image removal on node %s failed: %v (output: %s)\n", + nodeName, err, output) + + return + } + + GinkgoWriter.Printf("Workload image removed from node %s (output: %s)\n", + nodeName, output) +} + func deleteRemediationCR( ctx context.Context, k8sClient client.Client, gvk schema.GroupVersionKind, name string, diff --git a/tests/internal/helpers/node_ops.go b/tests/internal/helpers/node_ops.go index a7fba2bea2..1fa3f71adf 100644 --- a/tests/internal/helpers/node_ops.go +++ b/tests/internal/helpers/node_ops.go @@ -31,7 +31,7 @@ func RunOnNode( } args := append( - []string{"debug", "node/" + nodeName, "--", "chroot", "/host"}, + []string{"debug", "node/" + nodeName, "-n", "default", "--", "chroot", "/host"}, cmd..., ) diff --git a/tests/internal/helpers/nodes.go b/tests/internal/helpers/nodes.go index 9676ab1c80..350e02fe22 100644 --- a/tests/internal/helpers/nodes.go +++ b/tests/internal/helpers/nodes.go @@ -3,6 +3,7 @@ package helpers import ( "context" "fmt" + "math/rand" "sort" corev1 "k8s.io/api/core/v1" @@ -20,8 +21,9 @@ func IsNodeReady(node *corev1.Node) bool { return false } -// SelectWorkerNode returns a Ready, schedulable worker node that is not in the -// excludeNodes list. Returns an error if no eligible node is found. +// SelectWorkerNode returns a random Ready, schedulable worker node that is not +// in the excludeNodes list. Randomization prevents deterministic reuse of the +// same node across sequential destructive tests. func SelectWorkerNode(ctx context.Context, k8sClient client.Client, excludeNodes ...string) (*corev1.Node, error) { nodeList := &corev1.NodeList{} @@ -29,15 +31,13 @@ func SelectWorkerNode(ctx context.Context, k8sClient client.Client, excludeNodes return nil, fmt.Errorf("failed to list worker nodes: %w", err) } - sort.Slice(nodeList.Items, func(i, j int) bool { - return nodeList.Items[i].Name < nodeList.Items[j].Name - }) - excluded := make(map[string]bool, len(excludeNodes)) for _, name := range excludeNodes { excluded[name] = true } + var eligible []corev1.Node + for i := range nodeList.Items { node := &nodeList.Items[i] @@ -46,11 +46,21 @@ func SelectWorkerNode(ctx context.Context, k8sClient client.Client, excludeNodes } if IsNodeReady(node) { - return node, nil + eligible = append(eligible, *node) } } - return nil, fmt.Errorf("no eligible Ready worker node found (excluded: %v)", excludeNodes) + if len(eligible) == 0 { + return nil, fmt.Errorf("no eligible Ready worker node found (excluded: %v)", excludeNodes) + } + + sort.Slice(eligible, func(i, j int) bool { + return eligible[i].Name < eligible[j].Name + }) + + selected := &eligible[rand.Intn(len(eligible))] + + return selected, nil } // CountReadyWorkerNodes returns the number of Ready, schedulable worker nodes.