Add 6 standalone FAR destructive tests and improve test resiliency

scripts/test-runner.sh

@@ -44,7 +44,7 @@ fi
 
 
 # Build ginkgo command
-cmd="${GINKGO} -timeout=24h --keep-going --require-suite -r"
+cmd="${GINKGO} -timeout=24h --keep-going --require-suite --randomize-all -r"
 
 if [[ "${ECO_TEST_VERBOSE}" == "true" ]]; then
     cmd+=" -vv"

tests/far-operator/README.md

@@ -114,3 +114,73 @@ Validates that the FAR controller container image ships the minimum expected set
 - **Environment**: Connected or disconnected
 - **Standalone**: `ginkgo --label-filter="far" --focus="fence agents" ./tests/far-operator/...`
 - **Pass criteria**: All expected fence agent binaries are present in the container
+
+## Destructive Tests
+
+Tests that trigger node fencing via `fence_aws` and cause node reboots. Require AWS IPI cluster with 3+ worker nodes and AWS fencing credentials.
+
+### 10. Verify Standalone FAR Remediation ([OCP-61229](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-61229))
+
+Creates a FenceAgentsRemediation CR targeting a worker node. Validates that the fence agent reboots the node and the node object is preserved (not re-created).
+
+- **Operators**: FAR v0.8.0+
+- **Cluster**: AWS IPI, 3+ worker nodes
+- **Storage**: None
+- **Environment**: Connected
+- **Standalone**: `ginkgo --label-filter="far && disruption:destructive" --focus="standalone FAR CR" ./tests/far-operator/...`
+- **Pass criteria**: Node boot ID changes, node creation timestamp unchanged, node returns to Ready
+
+### 11. Verify Remediation on Active Controller Node ([OCP-70638](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-70638))
+
+Creates a FAR CR targeting the node hosting the active FAR controller pod. Validates that controller failover occurs and remediation completes despite the leader being fenced.
+
+- **Operators**: FAR v0.8.0+
+- **Cluster**: AWS IPI, 3+ worker nodes
+- **Storage**: None
+- **Environment**: Connected
+- **Standalone**: `ginkgo --label-filter="far && disruption:destructive" --focus="active FAR controller" ./tests/far-operator/...`
+- **Pass criteria**: Node reboots, node returns to Ready, FAR controller replicas recover
+
+### 12. Verify FAR NoSchedule Taint During Remediation ([OCP-65960](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-65960))
+
+Creates a FAR CR and verifies that the FAR NoSchedule taint is applied to the target node during the remediation process.
+
+- **Operators**: FAR v0.8.0+
+- **Cluster**: AWS IPI, 3+ worker nodes
+- **Storage**: None
+- **Environment**: Connected
+- **Standalone**: `ginkgo --label-filter="far && disruption:destructive" --focus="NoSchedule taint" ./tests/far-operator/...`
+- **Pass criteria**: FAR taint `remediation.medik8s.io/fence-agents-remediation:NoSchedule` applied during remediation
+
+### 13. Verify FAR CR Status Conditions After Remediation ([OCP-67015](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-67015))
+
+Creates a FAR CR and after remediation completes, verifies the CR status conditions match the expected terminal state: Processing=False, FenceAgentActionSucceeded=True, Succeeded=True.
+
+- **Operators**: FAR v0.8.0+
+- **Cluster**: AWS IPI, 3+ worker nodes
+- **Storage**: None
+- **Environment**: Connected
+- **Standalone**: `ginkgo --label-filter="far && disruption:destructive" --focus="status conditions" ./tests/far-operator/...`
+- **Pass criteria**: All three FAR CR conditions present with expected values
+
+### 14. Verify FAR Default Reboot Action ([OCP-66203](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-66203))
+
+Creates a FAR CR without the `--action` parameter in shared parameters. Validates that FAR defaults to the reboot action and the node is successfully rebooted.
+
+- **Operators**: FAR v0.8.0+
+- **Cluster**: AWS IPI, 3+ worker nodes
+- **Storage**: None
+- **Environment**: Connected
+- **Standalone**: `ginkgo --label-filter="far && disruption:destructive" --focus="action is omitted" ./tests/far-operator/...`
+- **Pass criteria**: Node reboots despite no explicit action parameter
+
+### 15. Verify Controller Leadership Handover ([OCP-70636](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-70636))
+
+Deletes the active FAR controller pod and validates that a new pod acquires the controller lease. This test does not fence any nodes; it verifies leader election recovery only.
+
+- **Operators**: FAR v0.8.0+
+- **Cluster**: Multi-node, 2+ controller replicas
+- **Storage**: None
+- **Environment**: Connected or disconnected
+- **Standalone**: `ginkgo --label-filter="far" --focus="controller leadership" ./tests/far-operator/...`
+- **Pass criteria**: FAR deployment becomes ready, controller lease is held by a different pod

tests/far-operator/far_suite_test.go

@@ -1,12 +1,17 @@
 package far
 
 import (
+	"context"
 	"runtime"
 	"testing"
 
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	"github.com/medik8s/system-tests/tests/far-operator/internal/farparams"
 	_ "github.com/medik8s/system-tests/tests/far-operator/tests"
 	. "github.com/medik8s/system-tests/tests/internal/medik8sinittools"
+	"github.com/medik8s/system-tests/tests/internal/medik8sparams"
 	"github.com/medik8s/system-tests/tests/internal/reporter"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -28,6 +33,17 @@ var _ = JustAfterEach(func() {
 		CurrentSpecReport(), currentFile, farparams.ReporterNamespacesToDump, farparams.ReporterCRDsToDump)
 })
 
+var _ = AfterSuite(func() {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      farparams.SharedCredentialsSecretName,
+			Namespace: medik8sparams.OperatorNs,
+		},
+	}
+
+	_ = APIClient.Delete(context.Background(), secret)
+})
+
 var _ = ReportAfterSuite("", func(report Report) {
 	reportxml.Create(
 		report, Medik8sConfig.GetReportPath(), Medik8sConfig.TCPrefix)

tests/far-operator/internal/farparams/const.go

@@ -77,4 +77,40 @@ const (
 
 	// ControllerLeaseName is the FAR leader election lease name (LeaderElectionID in cmd/main.go).
 	ControllerLeaseName = "cb305759.medik8s.io"
+
+	// FARConditionProcessing is the condition type for remediation progress.
+	FARConditionProcessing = "Processing"
+	// FARConditionFenceAgentSucceeded is the condition type for fence agent action result.
+	FARConditionFenceAgentSucceeded = "FenceAgentActionSucceeded"
+	// FARConditionSucceeded is the condition type for overall remediation outcome.
+	FARConditionSucceeded = "Succeeded"
+
+	// FARNoScheduleTaintKey is the taint key applied by FAR during remediation.
+	FARNoScheduleTaintKey = "remediation.medik8s.io/fence-agents-remediation"
+
+	// ControllerHandoverTimeout is how long to wait for controller leadership transfer.
+	ControllerHandoverTimeout = 3 * time.Minute
+	// WorkloadEvictionTimeout is how long to wait for workload pods to be evicted.
+	WorkloadEvictionTimeout = 5 * time.Minute
+	// WorkloadPodReadyTimeout is how long to wait for a test workload pod to reach Running.
+	WorkloadPodReadyTimeout = 2 * time.Minute
+
+	// FARCRRetryCount is the retry count for FAR/FART CR spec (matches upstream default).
+	FARCRRetryCount = 10
+	// FARCRRetryInterval is the retry interval for FAR/FART CR spec.
+	FARCRRetryInterval = "20s"
+	// FARCRTimeout is the fence agent command timeout for FAR/FART CR spec.
+	FARCRTimeout = "60s"
+	// FARCRRemediationStrategy is the default remediation strategy for FAR CRs.
+	FARCRRemediationStrategy = "OutOfServiceTaint"
+
+	// CrioCleanupTimeout is the timeout for the post-remediation CRI-O overlay cleanup.
+	CrioCleanupTimeout = 2 * time.Minute
+
+	// SharedCredentialsSecretName is the Secret created by the test suite to hold
+	// fence agent credentials in the format expected by SharedSecretName.
+	SharedCredentialsSecretName = "far-test-shared-credentials"
+
+	// WorkloadTestImage is the container image used for test workload pods.
+	WorkloadTestImage = "registry.access.redhat.com/ubi9/ubi-minimal"
 )

tests/far-operator/tests/far.go

@@ -65,23 +65,17 @@ var _ = Describe(
 					LabelSelector: farparams.OperatorControllerPodLabelSelector,
 				}
 
-				_, err := pod.WaitForAllPodsInNamespaceRunning(
-					APIClient,
-					medik8sparams.OperatorNs,
-					medik8sparams.DefaultTimeout,
-					listOptions,
-				)
-				Expect(err).ToNot(HaveOccurred(), "Pod is not ready")
+				By("Waiting for expected number of Running FAR pods")
 
-				By("Verifying pod count matches expected replicas")
+				Eventually(func(assertion Gomega) {
+					farPods, err := pod.List(APIClient, medik8sparams.OperatorNs, listOptions)
+					assertion.Expect(err).ToNot(HaveOccurred(), "Failed to list FAR pods")
 
-				farPods, err := pod.List(APIClient, medik8sparams.OperatorNs, listOptions)
-				Expect(err).ToNot(HaveOccurred(), "Failed to list FAR pods")
-
-				runningPods := helpers.FilterRunningPods(farPods)
+					runningPods := helpers.FilterRunningPods(farPods)
 
-				Expect(int32(len(runningPods))).To(Equal(expectedCount),
-					"Expected %d running FAR pod(s), found %d", expectedCount, len(runningPods))
+					assertion.Expect(int32(len(runningPods))).To(Equal(expectedCount),
+						"Expected %d running FAR pod(s), found %d", expectedCount, len(runningPods))
+				}, medik8sparams.DefaultTimeout, farparams.DefaultPollInterval).Should(Succeed())
 			})
 
 		It("Verify FAR CSV has required annotations",

tests/far-operator/tests/far_controller_lifecycle.go

@@ -0,0 +1,118 @@
+package tests
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	coordinationv1 "k8s.io/api/coordination/v1"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/rh-ecosystem-edge/eco-goinfra/pkg/deployment"
+	"github.com/rh-ecosystem-edge/eco-goinfra/pkg/reportxml"
+
+	"github.com/medik8s/system-tests/tests/far-operator/internal/farparams"
+	"github.com/medik8s/system-tests/tests/far-operator/internal/farutils"
+	"github.com/medik8s/system-tests/tests/internal/labels"
+	. "github.com/medik8s/system-tests/tests/internal/medik8sinittools"
+	"github.com/medik8s/system-tests/tests/internal/medik8sparams"
+)
+
+var _ = Describe("FAR Controller Lifecycle Tests",
+	Serial,
+	Label(labels.OperatorFAR, farparams.Label,
+		labels.DisruptionNonDestructive),
+	func() {
+		var ctx context.Context
+
+		BeforeEach(func() {
+			ctx = context.Background()
+
+			By("Verifying FAR controller deployment is Ready")
+
+			farDeployment, err := deployment.Pull(
+				APIClient, farparams.OperatorDeploymentName, medik8sparams.OperatorNs)
+			Expect(err).ToNot(HaveOccurred(), "Failed to get FAR deployment")
+			Expect(farDeployment.IsReady(medik8sparams.DefaultTimeout)).To(BeTrue(),
+				"FAR deployment is not Ready")
+		})
+
+		It("should transfer controller leadership when the active pod is deleted",
+			Label(labels.TierAcceptance, labels.ComponentController),
+			reportxml.ID("OCP-70636"),
+			func() {
+				By("Getting the current active FAR controller pod")
+
+				pods, err := farutils.GetFARControllerPods(ctx, APIClient)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(pods).ToNot(BeEmpty(), "No running FAR controller pods found")
+
+				oldLeaderNode, err := farutils.GetActiveFARControllerNode(ctx, APIClient)
+				Expect(err).ToNot(HaveOccurred())
+
+				var oldLeaderPod *corev1.Pod
+
+				for i := range pods {
+					if pods[i].Spec.NodeName == oldLeaderNode {
+						oldLeaderPod = &pods[i]
+
+						break
+					}
+				}
+
+				Expect(oldLeaderPod).ToNot(BeNil(),
+					"Could not find controller pod on leader node %s", oldLeaderNode)
+
+				oldPodName := oldLeaderPod.Name
+				GinkgoWriter.Printf("Active controller pod: %s on node %s\n",
+					oldPodName, oldLeaderNode)
+
+				By("Deleting the active controller pod " + oldPodName)
+
+				Expect(APIClient.Delete(ctx, oldLeaderPod)).To(Succeed())
+
+				By("Waiting for FAR controller deployment to become ready")
+
+				farDeployment, err := deployment.Pull(
+					APIClient, farparams.OperatorDeploymentName, medik8sparams.OperatorNs)
+				Expect(err).ToNot(HaveOccurred(), "Failed to pull FAR controller deployment")
+				Expect(farDeployment.IsReady(farparams.ControllerHandoverTimeout)).To(BeTrue(),
+					"FAR deployment did not become ready after pod deletion")
+
+				By("Verifying controller lease transferred to a different pod")
+
+				Eventually(func(assertion Gomega) {
+					lease := &coordinationv1.Lease{}
+					assertion.Expect(APIClient.Get(ctx, client.ObjectKey{
+						Name:      farparams.ControllerLeaseName,
+						Namespace: medik8sparams.OperatorNs,
+					}, lease)).To(Succeed())
+					assertion.Expect(lease.Spec.HolderIdentity).ToNot(BeNil(),
+						"Lease has no holder after pod deletion")
+
+					if lease.Spec.HolderIdentity != nil {
+						assertion.Expect(*lease.Spec.HolderIdentity).ToNot(Equal(oldPodName),
+							"Lease is still held by deleted pod %s", oldPodName)
+					}
+
+					newPods, err := farutils.GetFARControllerPods(ctx, APIClient)
+					assertion.Expect(err).ToNot(HaveOccurred())
+
+					hasNewRunningPod := false
+
+					for _, p := range newPods {
+						if p.Name != oldPodName && p.Status.Phase == corev1.PodRunning {
+							hasNewRunningPod = true
+
+							break
+						}
+					}
+
+					assertion.Expect(hasNewRunningPod).To(BeTrue(),
+						"No new Running controller pod found after deleting %s", oldPodName)
+				}, farparams.ControllerHandoverTimeout, farparams.DefaultPollInterval).Should(Succeed(),
+					"Controller leadership did not transfer after pod deletion")
+			})
+	})