[pull] master from ray-project:master

.buildkite/core.rayci.yml

@@ -376,10 +376,8 @@ steps:
       - gpu
     instance_type: gpu-large
     commands:
-      # This machine has 4 GPUs, and we need 2 GPUs, so allow 2 tests to run in
-      # parallel.
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... core
-        --parallelism-per-worker 2 --gpus 2
+        --gpus 4
         --build-name coregpubuild
         --only-tags multi_gpu
     depends_on: coregpubuild

.buildkite/data.rayci.yml

@@ -4,8 +4,8 @@ depends_on:
   - oss-ci-base_ml
 steps:
   # builds
-  - name: data6build
-    wanda: ci/docker/data6.build.wanda.yaml
+  - name: data9build
+    wanda: ci/docker/data9.build.wanda.yaml
 
   - name: datalbuild
     wanda: ci/docker/datal.build.wanda.yaml
@@ -28,38 +28,38 @@ steps:
     wanda: ci/docker/datatfxbsl.build.wanda.yaml
 
   # tests
-  - label: ":database: data: arrow 6 tests"
-    tags: 
+  - label: ":database: data: arrow v9 tests"
+    tags:
       - python
       - data
     instance_type: medium
     parallelism: 2
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data 
-        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" 
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
-        --build-name data6build
+        --build-name data9build
         --except-tags data_integration,doctest
-    depends_on: data6build
+    depends_on: data9build
 
-  - label: ":database: data: arrow 17 tests"
-    tags: 
+  - label: ":database: data: arrow v17 tests"
+    tags:
       - python
       - data
     instance_type: medium
     parallelism: 2
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
-        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" 
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name datalbuild
         --except-tags data_integration,doctest
     depends_on: datalbuild
 
-  - label: ":database: data: arrow 17 {{matrix.python}} tests ({{matrix.worker_id}})"
+  - label: ":database: data: arrow v17 {{matrix.python}} tests ({{matrix.worker_id}})"
     key: datal_python_tests
     if: build.pull_request.labels includes "continuous-build" || pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
-    tags: 
+    tags:
       - python
       - data
     instance_type: medium
@@ -76,21 +76,21 @@ steps:
 
   - label: ":database: data: arrow nightly tests"
     if: pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
-    tags: 
+    tags:
       - python
       - data
     instance_type: medium
     parallelism: 2
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data 
-        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" 
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name datanbuild
         --except-tags data_integration,doctest
     depends_on: datanbuild
-  
+
   - label: ":database: data: TFRecords (tfx-bsl) tests"
-    tags: 
+    tags:
       - python
       - data
     instance_type: medium
@@ -102,39 +102,39 @@ steps:
     depends_on: datatfxbslbuild
 
   - label: ":database: data: doc tests"
-    tags: 
+    tags:
       - data
       - doc
     instance_type: medium
     commands:
       # doc tests
-      - bazel run //ci/ray_ci:test_in_docker -- python/ray/... //doc/... data 
+      - bazel run //ci/ray_ci:test_in_docker -- python/ray/... //doc/... data
         --build-name datalbuild
         --except-tags gpu
         --only-tags doctest
         --parallelism-per-worker 2
       # doc examples
-      - bazel run //ci/ray_ci:test_in_docker -- //doc/... data 
+      - bazel run //ci/ray_ci:test_in_docker -- //doc/... data
         --build-name datalbuild
         --except-tags gpu,post_wheel_build,doctest
         --parallelism-per-worker 2
         --skip-ray-installation
     depends_on: datalbuild
 
   - label: ":database: data: doc gpu tests"
-    tags: 
+    tags:
       - data
       - doc
       - gpu
     instance_type: gpu-large
     commands:
       # doc tests
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //doc/... data 
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //doc/... data
         --build-name docgpubuild
         --only-tags doctest
         --except-tags cpu
       # doc examples
-      - bazel run //ci/ray_ci:test_in_docker -- //doc/... data 
+      - bazel run //ci/ray_ci:test_in_docker -- //doc/... data
         --build-name docgpubuild
         --except-tags doctest
         --only-tags gpu
@@ -147,7 +147,7 @@ steps:
       - data
     instance_type: medium
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data 
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
         --build-name datamongobuild
         --build-type java
         --only-tags data_integration
@@ -168,29 +168,29 @@ steps:
 
   - label: ":database: data: flaky tests"
     key: data_flaky_tests
-    tags: 
+    tags:
       - python
       - data
       - skip-on-premerge
     instance_type: medium
     soft_fail: true
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests 
+      - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests
         --parallelism-per-worker 3
         --build-name datalbuild
         --except-tags gpu_only,gpu
     depends_on: datalbuild
 
   - label: ":database: data: flaky gpu tests"
     key: data_flaky_gpu_tests
-    tags: 
+    tags:
       - python
       - data
       - skip-on-premerge
     instance_type: gpu-large
     soft_fail: true
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests 
+      - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests
         --build-name docgpubuild
         --only-tags gpu,gpu_only
     depends_on: docgpubuild

.buildkite/linux_aarch64.rayci.yml

@@ -99,7 +99,7 @@ steps:
     tags: linux_wheels
     instance_type: medium-arm64
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //doc/... core 
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //doc/... core
         --build-type wheel-aarch64
         --build-name oss-ci-base_build-aarch64
         --parallelism-per-worker 3
@@ -115,7 +115,7 @@ steps:
     tags: linux_wheels
     instance_type: medium-arm64
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //doc/... serve 
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //doc/... serve
         --build-type wheel-aarch64
         --build-name oss-ci-base_build-aarch64
         --parallelism-per-worker 3