feat: upgrade to Ray 2.52.0 to support token auth mode (#4152)

* update

Signed-off-by: Future-Outlier <[email protected]>

* update

Signed-off-by: Future-Outlier <[email protected]>

* update

Signed-off-by: Future-Outlier <[email protected]>

* update

Signed-off-by: Future-Outlier <[email protected]>

* Trigger CI

Signed-off-by: Future-Outlier <[email protected]>

* andrew's comment

Signed-off-by: Future-Outlier <[email protected]>

* update

Signed-off-by: Future-Outlier <[email protected]>

* Revert ray ml image

Signed-off-by: Future-Outlier <[email protected]>

---------

Signed-off-by: Future-Outlier <[email protected]>

Author: Future-Outlier

.buildkite/setup-env.sh

@@ -35,7 +35,7 @@ apt-get update
 apt-get install -y python3.11 python3-pip python3-venv python3-dev build-essential
 
 # Install requirements
-pip install --break-system-packages ray[default]==2.46.0
+pip install --break-system-packages ray[default]==2.52.0
 
 # Bypass Git's ownership check due to unconventional user IDs in Docker containers
 git config --global --add safe.directory /workdir

apiserver/test/cluster/cluster_external_redis.yaml

@@ -10,7 +10,7 @@ metadata:
     # ray.io/external-storage-namespace: "my-raycluster-storage"
   name: raycluster-external-redis
 spec:
-  rayVersion: '2.46.0'
+  rayVersion: '2.52.0'
   headGroupSpec:
     # The `rayStartParams` are used to configure the `ray start` command.
     # See https://github.com/ray-project/kuberay/blob/master/docs/guidance/rayStartParams.md for the default settings of `rayStartParams` in KubeRay.
@@ -26,7 +26,7 @@ spec:
       spec:
         containers:
           - name: ray-head
-            image: rayproject/ray:2.46.0
+            image: rayproject/ray:2.52.0
             resources:
               limits:
                 cpu: "1"
@@ -80,7 +80,7 @@ spec:
         spec:
           containers:
             - name: ray-worker
-              image: rayproject/ray:2.46.0
+              image: rayproject/ray:2.52.0
               volumeMounts:
                 - mountPath: /tmp/ray
                   name: ray-logs

benchmark/perf-tests/100-raycluster/raycluster.yaml

@@ -5,15 +5,15 @@ metadata:
   labels:
     perf-test: ray-cluster
 spec:
-  rayVersion: '2.46.0'
+  rayVersion: '2.52.0'
   headGroupSpec:
     rayStartParams:
       dashboard-host: '0.0.0.0'
     template:
       spec:
         containers:
         - name: ray-head
-          image: rayproject/ray:2.46.0
+          image: rayproject/ray:2.52.0
           ports:
           - containerPort: 6379
             name: gcs
@@ -42,7 +42,7 @@ spec:
       spec:
         containers:
         - name: ray-worker
-          image: rayproject/ray:2.46.0
+          image: rayproject/ray:2.52.0
           volumeMounts:
             - mountPath: /tmp/ray
               name: ray-logs

benchmark/perf-tests/100-rayjob/config.yaml

@@ -39,7 +39,7 @@ steps:
     - basename: pytorch-mnist
       objectTemplatePath: pytorch-mnist-rayjob.yaml
       templateFillMap:
-        Image: "rayproject/ray:2.46.0" # replace with image built from images/ray-pytorch
+        Image: "rayproject/ray:2.52.0" # replace with image built from images/ray-pytorch
 - name: Creating RayJobs for Ray Data Image Resizing
   phases:
   - namespaceRange:
@@ -51,7 +51,7 @@ steps:
     - basename: ray-data-image-resize
       objectTemplatePath: ray-data-image-resize.yaml
       templateFillMap:
-        Image: "rayproject/ray:2.46.0" # replace with image built from images/ray-pytorch
+        Image: "rayproject/ray:2.52.0" # replace with image built from images/ray-pytorch
 - name: Wait for RayJobs complete
   measurements:
   - Identifier: WaitForRayJob

benchmark/perf-tests/100-rayjob/pytorch-mnist-rayjob.yaml

@@ -13,7 +13,7 @@ spec:
       CPUS_PER_WORKER: "1"
       OMP_NUM_THREADS: "1" # Set OMP_NUM_THREADS to avoid KeyErorr race condition.
   rayClusterSpec:
-    rayVersion: '2.46.0'
+    rayVersion: '2.52.0'
     headGroupSpec:
       template:
         spec:

benchmark/perf-tests/100-rayjob/ray-data-image-resize.yaml

@@ -8,7 +8,7 @@ spec:
   shutdownAfterJobFinishes: true
   entrypoint: python ray_data_image_resize.py
   rayClusterSpec:
-    rayVersion: '2.46.0'
+    rayVersion: '2.52.0'
     headGroupSpec:
       template:
         spec:

benchmark/perf-tests/1000-raycluster/config.yaml

@@ -43,7 +43,7 @@ steps:
       objectTemplatePath: raycluster.yaml
       templateFillMap:
         Replicas: 3
-        Image: "rayproject/ray:2.46.0"
+        Image: "rayproject/ray:2.52.0"
 - name: Wait for RayClusters ready
   measurements:
   - Identifier: WaitForRayCluster

benchmark/perf-tests/1000-raycluster/raycluster.yaml

@@ -5,7 +5,7 @@ metadata:
   labels:
     perf-test: ray-cluster
 spec:
-  rayVersion: '2.46.0'
+  rayVersion: '2.52.0'
   headGroupSpec:
     serviceType: ClusterIP
     rayStartParams:

benchmark/perf-tests/1000-rayjob/config.yaml

@@ -39,7 +39,7 @@ steps:
     - basename: pytorch-mnist
       objectTemplatePath: pytorch-mnist-rayjob.yaml
       templateFillMap:
-        Image: "rayproject/ray:2.46.0"
+        Image: "rayproject/ray:2.52.0"
 - name: Creating RayJobs for Ray Data Image Resizing
   phases:
   - namespaceRange:
@@ -51,7 +51,7 @@ steps:
     - basename: ray-data-image-resize
       objectTemplatePath: ray-data-image-resize.yaml
       templateFillMap:
-        Image: "rayproject/ray:2.46.0"
+        Image: "rayproject/ray:2.52.0"
 - name: Wait for RayJobs complete
   measurements:
   - Identifier: WaitForRayJob

benchmark/perf-tests/1000-rayjob/pytorch-mnist-rayjob.yaml

@@ -26,7 +26,7 @@ spec:
             requests:
               cpu: "10m"
   rayClusterSpec:
-    rayVersion: '2.46.0'
+    rayVersion: '2.52.0'
     headGroupSpec:
       rayStartParams:
         disable-usage-stats: 'true'