[RayCluster] Status includes head containter status message

Signed-off-by: Spencer Peterson <[email protected]>

Author: spencer-p

ray-operator/controllers/ray/utils/util.go

@@ -91,6 +91,16 @@ func FindHeadPodReadyCondition(headPod *corev1.Pod) metav1.Condition {
 		headPodReadyCondition.Status = metav1.ConditionStatus(cond.Status)
 		headPodReadyCondition.Message = cond.Message
 
+		// Add details from failed or waiting container statuses if available.
+		details := containerStatusDetails(headPod)
+		if details != "" {
+			if headPodReadyCondition.Message == "" {
+				headPodReadyCondition.Message = details
+			} else {
+				headPodReadyCondition.Message += "; " + details
+			}
+		}
+
 		// Determine the reason; default to HeadPodRunningAndReady if the headPod is ready but no specific reason is provided
 		reason := cond.Reason
 		if cond.Status == corev1.ConditionTrue && reason == "" {
@@ -108,6 +118,18 @@ func FindHeadPodReadyCondition(headPod *corev1.Pod) metav1.Condition {
 	return headPodReadyCondition
 }
 
+func containerStatusDetails(pod *corev1.Pod) string {
+	var details []string
+	for _, status := range pod.Status.ContainerStatuses {
+		if status.State.Waiting != nil {
+			details = append(details, fmt.Sprintf("%s: %s: %s", status.Name, status.State.Waiting.Reason, status.State.Waiting.Message))
+		} else if status.State.Terminated != nil {
+			details = append(details, fmt.Sprintf("%s: %s: %s", status.Name, status.State.Terminated.Reason, status.State.Terminated.Message))
+		}
+	}
+	return strings.Join(details, ", ")
+}
+
 // FindRayClusterSuspendStatus returns the current suspend status from two conditions:
 //  1. rayv1.RayClusterSuspending
 //  2. rayv1.RayClusterSuspended

ray-operator/controllers/ray/utils/util_test.go

@@ -977,6 +977,79 @@ func TestFindHeadPodReadyCondition(t *testing.T) {
 	}
 }
 
+func TestFindHeadPodReadyMessage(t *testing.T) {
+	tests := []struct {
+		name            string
+		message         string
+		status          []corev1.ContainerStatus
+		expectedMessage string
+	}{{
+		name: "no message no status want nothing",
+	}, {
+		name:            "only reason",
+		message:         "TooEarlyInTheMorning",
+		expectedMessage: "TooEarlyInTheMorning",
+	}, {
+		name:    "one reason one status",
+		message: "containers not ready",
+		status: []corev1.ContainerStatus{{
+			Name: "ray",
+			State: corev1.ContainerState{
+				Waiting: &corev1.ContainerStateWaiting{
+					Reason:  "ImagePullBackOff",
+					Message: `Back-off pulling image royproject/roy:latest: ErrImagePull: rpc error: code = NotFound`,
+				},
+			},
+		}},
+		expectedMessage: `containers not ready; ray: ImagePullBackOff: Back-off pulling image royproject/roy:latest: ErrImagePull: rpc error: code = NotFound`,
+	}, {
+		name:    "one reason two statuses",
+		message: "aesthetic problems",
+		status: []corev1.ContainerStatus{{
+			Name: "indigo",
+			State: corev1.ContainerState{
+				Waiting: &corev1.ContainerStateWaiting{
+					Reason:  "BadColor",
+					Message: "too blue",
+				},
+			},
+		}, {
+			Name: "circle",
+			State: corev1.ContainerState{
+				Terminated: &corev1.ContainerStateTerminated{
+					Reason:  "BadGeometry",
+					Message: "too round",
+				},
+			},
+		}},
+		expectedMessage: "aesthetic problems; indigo: BadColor: too blue, circle: BadGeometry: too round",
+	}, {
+		name: "no reason one status",
+		status: []corev1.ContainerStatus{{
+			Name: "my-image",
+			State: corev1.ContainerState{
+				Terminated: &corev1.ContainerStateTerminated{
+					Reason:  "Crashed",
+					Message: "bash not found",
+				},
+			},
+		}},
+		expectedMessage: "my-image: Crashed: bash not found",
+	}}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			pod := createRayHeadPodWithPhaseAndCondition(corev1.PodPending, corev1.PodReady, corev1.ConditionFalse)
+			pod.Status.Conditions[0].Message = tc.message
+			pod.Status.ContainerStatuses = tc.status
+			cond := FindHeadPodReadyCondition(pod)
+			if cond.Message != tc.expectedMessage {
+				t.Errorf("FindHeadPodReadyCondition(...) returned condition with message %q, but wanted %q", cond.Message, tc.expectedMessage)
+			}
+		})
+	}
+}
+
 func TestErrRayClusterReplicaFailureReason(t *testing.T) {
 	assert.Equal(t, "FailedDeleteAllPods", RayClusterReplicaFailureReason(ErrFailedDeleteAllPods))
 	assert.Equal(t, "FailedDeleteHeadPod", RayClusterReplicaFailureReason(ErrFailedDeleteHeadPod))