Skip to content

Commit

Permalink
Fix third-party cudf.pandas tests (#17900)
Browse files Browse the repository at this point in the history
## Description
This PR fixes cudf ci nightly test failures:
https://github.com/rapidsai/cudf/actions/runs/13097249137/job/36541039646

## Checklist
- [x] I am familiar with the [Contributing
Guidelines](https://github.com/rapidsai/cudf/blob/HEAD/CONTRIBUTING.md).
- [x] New or existing tests cover these changes.
- [x] The documentation is up to date with these changes.
  • Loading branch information
galipremsagar authored Feb 3, 2025
1 parent 03742ac commit f1c2f2a
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ runtest() {
local lib=$1
local mode=$2

echo "Running tests for $lib in $mode mode"
local plugin=""
if [ "$mode" = "cudf" ]; then
plugin="-p cudf.pandas"
Expand Down
4 changes: 2 additions & 2 deletions ci/cudf_pandas_scripts/third-party-integration/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ main() {
lib=$(echo "$lib" | tr -d '""')
echo "Running tests for library $lib"

CUDA_MAJOR=$(if [ "$lib" = "tensorflow" ]; then echo "11"; else echo "12"; fi)
CUDA_VERSION=$(if [ "$lib" = "tensorflow" ]; then echo "11.8"; else echo "${RAPIDS_CUDA_VERSION%.*}"; fi)

. /opt/conda/etc/profile.d/conda.sh

Expand All @@ -36,7 +36,7 @@ main() {
--config "$dependencies_yaml" \
--output conda \
--file-key "test_${lib}" \
--matrix "cuda=${CUDA_MAJOR};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
--matrix "cuda=${CUDA_VERSION};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml

rapids-mamba-retry env create --yes -f env.yaml -n test

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# Copyright (c) 2023-2024, NVIDIA CORPORATION.
# Copyright (c) 2023-2025, NVIDIA CORPORATION.

from __future__ import annotations

import glob
import os
import pickle
from typing import TYPE_CHECKING, BinaryIO
Expand Down Expand Up @@ -75,23 +76,40 @@ def swap_xfail(item: _pytest.nodes.Item, name: str):
swap_xfail(item, "xfail_compare")


def get_full_nodeid(pyfuncitem):
# Get the full path to the test file
filepath = pyfuncitem.path
# Get the test name and any parameters
test_name = "::".join(pyfuncitem.nodeid.split("::")[1:])
# Combine the full file path with the test name
full_nodeid = f"{filepath}::{test_name}"
return full_nodeid


def read_all_results(pattern):
results = {}
for filepath in glob.glob(pattern):
with open(filepath, "rb") as f:
results.update(dict(read_results(f)))
return results


def pytest_configure(config: _pytest.config.Config):
gold_basename = "results-gold"
cudf_basename = "results-cudf-pandas"
test_folder = os.path.join(os.path.dirname(__file__))

if config.getoption("--compare"):
# Everyone reads everything
gold_path = os.path.join(test_folder, f"{gold_basename}.pickle")
cudf_path = os.path.join(test_folder, f"{cudf_basename}.pickle")
gold_path = os.path.join(test_folder, f"{gold_basename}*.pickle")
cudf_path = os.path.join(test_folder, f"{cudf_basename}*.pickle")
with disable_module_accelerator():
with open(gold_path, "rb") as f:
gold_results = dict(read_results(f))
with open(cudf_path, "rb") as f:
cudf_results = dict(read_results(f))
gold_results = read_all_results(gold_path)
cudf_results = read_all_results(cudf_path)
config.stash[results] = (gold_results, cudf_results)
else:
if "cudf.pandas" in config.option.plugins:
if any(
plugin.strip() == "cudf.pandas" for plugin in config.option.plugins
):
basename = cudf_basename
else:
basename = gold_basename
Expand All @@ -112,7 +130,7 @@ def pytest_configure(config: _pytest.config.Config):
def pytest_pyfunc_call(pyfuncitem: _pytest.python.Function):
if pyfuncitem.config.getoption("--compare"):
gold_results, cudf_results = pyfuncitem.config.stash[results]
key = pyfuncitem.nodeid
key = get_full_nodeid(pyfuncitem)
try:
gold = gold_results[key]
except KeyError:
Expand Down Expand Up @@ -140,7 +158,7 @@ def pytest_pyfunc_call(pyfuncitem: _pytest.python.Function):
# Tuple-based key-value pairs, key is the node-id
try:
pickle.dump(
(pyfuncitem.nodeid, result),
(get_full_nodeid(pyfuncitem), result),
pyfuncitem.config.stash[file_handle_key],
)
except pickle.PicklingError:
Expand Down

0 comments on commit f1c2f2a

Please sign in to comment.