From 97095ed5a27a46f7c9b016050b1f29c930e287b7 Mon Sep 17 00:00:00 2001 From: nethum529 Date: Fri, 3 Jul 2026 17:54:01 -0500 Subject: [PATCH 1/6] fix(compose): support object dtype in ColumnTransformer + SimpleImputer on cuDF impute-then-transform is one of the most common sklearn pipeline shapes, but ColumnTransformer + SimpleImputer crashed on a native cuDF DataFrame whenever categorical/string columns were involved (e.g. missing_values=pd.NA, strategy="constant"/"most_frequent" on object dtype data). Root causes, all stemming from cuML forcing data through cupy/CumlArray regardless of dtype, even though cupy has no way to represent Python object dtype (strings) at all: - check_array unconditionally tried to move object-dtype input to device. - CumlArrayDescriptor._to_output and coerce_arrays (the output-coercion paths behind every `@reflect`-decorated method and fitted attribute) had the same device-forcing assumption. - ColumnTransformer._hstack used cupy's hstack unconditionally, which can't combine a numeric cupy block with an object-dtype host block. - _get_mask's sentinel-detection logic crashed with a pd.NA truthiness error, since cuDF's null string values round-trip through `to_numpy(dtype="object")` as pd.NA (not None/nan). - SimpleImputer._get_param_names omitted "missing_values" and "add_indicator", so sklearn.base.clone() (used by ColumnTransformer to clone each transformer before fitting) silently dropped a user-supplied missing_values sentinel back to the default np.nan. Each fix targets only the previously-always-broken object-dtype path; numeric/device flows are untouched. Non-numpy extension dtypes (e.g. pandas/cudf `category`) are conservatively excluded from the new dtype checks rather than passed to `np.dtype()`, which would raise TypeError. Adds a regression test reproducing the exact issue shape (cuDF DataFrame, numeric + categorical constant-fill + categorical most-frequent-fill columns) and asserting cuML's output matches scikit-learn's bit-for-bit. Closes #6183 Signed-off-by: nethum529 --- .../preprocessing/_column_transformer.py | 11 +++ .../sklearn/preprocessing/_imputation.py | 23 ++++-- python/cuml/cuml/internals/outputs.py | 17 ++++ python/cuml/cuml/internals/validation.py | 9 ++ .../cuml/cuml/thirdparty_adapters/adapters.py | 32 ++++++-- python/cuml/tests/test_compose.py | 82 +++++++++++++++++++ python/cuml/tests/test_validation.py | 25 ++---- 7 files changed, 170 insertions(+), 29 deletions(-) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py index 85b2460ebe..e1e5b85ece 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py @@ -25,6 +25,7 @@ import cudf import cupy as np import numba +import numpy as cpu_np import pandas as pd import scipy.sparse as sp_sparse from cupyx.scipy import sparse as cu_sparse @@ -982,6 +983,16 @@ def _hstack(self, Xs): return cu_sparse.hstack(converted_Xs).tocsr() else: Xs = [f.toarray() if issparse(f) else f for f in Xs] + if any( + isinstance(getattr(X, "dtype", None), cpu_np.dtype) + and X.dtype.kind == "O" + for X in Xs + ): + # Object dtype (e.g. string columns from a categorical + # SimpleImputer) has no device representation - cupy has no + # way to store it. Stack on host instead. + Xs = [X.get() if isinstance(X, np.ndarray) else X for X in Xs] + return cpu_np.hstack(Xs) return np.hstack(Xs) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py index 20adbf1a42..865c139a99 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py @@ -260,10 +260,12 @@ def __init__(self, *, missing_values=np.nan, strategy="mean", @classmethod def _get_param_names(cls): return super()._get_param_names() + [ + "missing_values", "strategy", "fill_value", "verbose", - "copy" + "copy", + "add_indicator", ] def _validate_input(self, X, in_fit): @@ -423,7 +425,11 @@ def _dense_fit(self, X, strategy, missing_values, fill_value): # Constant elif strategy == "constant": - return np.full(X.shape[1], fill_value, dtype=X.dtype) + # Object-dtype X (e.g. strings) lives on host, since cupy has no + # way to represent it on device. Dispatch to X's own module so + # this works for both device (numeric) and host (object) input. + xp = np.get_array_module(X) + return xp.full(X.shape[1], fill_value, dtype=X.dtype) @mlfunc def transform(self, X): @@ -449,14 +455,18 @@ def transform(self, X): if self.strategy == "constant": valid_statistics = statistics else: + # Object-dtype statistics (e.g. strings) live on host, since cupy + # has no way to represent them on device. Dispatch to their own + # module so this works for both device and host input. + xp = np.get_array_module(statistics) # same as np.isnan but also works for object dtypes invalid_mask = _get_mask(statistics, np.nan) - valid_mask = np.logical_not(invalid_mask) + valid_mask = xp.logical_not(invalid_mask) valid_statistics = statistics[valid_mask] - valid_statistics_indexes = np.flatnonzero(valid_mask) + valid_statistics_indexes = xp.flatnonzero(valid_mask) if invalid_mask.any(): - missing = np.arange(X.shape[1])[invalid_mask] + missing = xp.arange(X.shape[1])[invalid_mask] if self.verbose: warnings.warn("Deleting features without " "observed values: %s" % missing) @@ -481,8 +491,9 @@ def transform(self, X): if self.strategy == "constant": X[mask] = valid_statistics[0] else: + xp = np.get_array_module(mask) for i, vi in enumerate(valid_statistics_indexes): - feature_idxs = np.flatnonzero(mask[:, vi]) + feature_idxs = xp.flatnonzero(mask[:, vi]) X[feature_idxs, vi] = valid_statistics[i] X = super()._concatenate_indicator(X, X_indicator) diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index b583803ab6..1b90be1bfe 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -297,6 +297,18 @@ def infer_output_type(array, array_like="numpy"): return None +def _is_object_dtype(res): + """Check for NumPy object dtype on an array-like or dataframe-like.""" + dtypes = getattr(res, "dtypes", None) + if dtypes is not None: + return any( + isinstance(dtype, np.dtype) and dtype.kind == "O" + for dtype in dtypes + ) + dtype = getattr(res, "dtype", None) + return isinstance(dtype, np.dtype) and dtype.kind == "O" + + class ArrayIndexPair: """An array paired with an aligned index. @@ -494,6 +506,11 @@ def convert_arrays( if isinstance(obj, ClassLabels): return obj.to_output(output_type, index=index) + # NumPy object arrays have no device representation. + # Keep them on host for method outputs and reflected attributes. + if isinstance(obj, np.ndarray) and _is_object_dtype(obj): + return obj + if isinstance(obj, np.ndarray): if output_type == "numpy": return obj diff --git a/python/cuml/cuml/internals/validation.py b/python/cuml/cuml/internals/validation.py index 7c03621f03..d24c5e8540 100644 --- a/python/cuml/cuml/internals/validation.py +++ b/python/cuml/cuml/internals/validation.py @@ -697,6 +697,15 @@ def check_array( # No original dtype, use first provided dtype dtype = dtype[0] + # Object dtype (e.g. strings) has no device-memory representation - cupy + # has no way to store arbitrary Python objects on the GPU. Force host + # output in that case regardless of the requested `mem_type`, rather than + # crashing when attempting the device conversion below. This is required + # for estimators (e.g. SimpleImputer) that explicitly support categorical + # / string data with an object dtype. + if isinstance(dtype, np.dtype) and dtype.kind == "O": + mem_type = "host" + # Coerce `array` to numpy/cupy/scipy.sparse/cupyx.scipy.sparse values as # requested. For dataframe-like inputs also extract the index for later use. index = None diff --git a/python/cuml/cuml/thirdparty_adapters/adapters.py b/python/cuml/cuml/thirdparty_adapters/adapters.py index c799301495..1980b69c1f 100644 --- a/python/cuml/cuml/thirdparty_adapters/adapters.py +++ b/python/cuml/cuml/thirdparty_adapters/adapters.py @@ -4,12 +4,28 @@ # import cupy as cp import numpy as np +import pandas as pd def _get_mask(X, value_to_mask): """Compute the boolean mask X == missing_values.""" - if value_to_mask == "NaN" or cp.isnan(value_to_mask): - return cp.isnan(X) + if isinstance(value_to_mask, str) and value_to_mask == "NaN": + xp = cp.get_array_module(X) + return xp.isnan(X) + try: + # NaN-like sentinels (np.nan, None, pd.NA, pd.NaT, ...) require an + # NA-aware comparison: a plain `==` against e.g. pd.NA propagates + # instead of returning a boolean mask, and `cp.isnan` doesn't accept + # non-numeric scalars like pd.NA in the first place. + is_na_sentinel = pd.isna(value_to_mask) + except (TypeError, ValueError): + is_na_sentinel = False + if is_na_sentinel: + if isinstance(X, cp.ndarray): + return cp.isnan(X) + # Host (e.g. object dtype) arrays can't use isnan - fall back to an + # NA-aware elementwise check that also recognizes None/pd.NA. + return pd.isna(X) else: return X == value_to_mask @@ -68,18 +84,22 @@ def _masked_column_mean(arr, masked_value): def _masked_column_mode(arr, masked_value): """Determine the most frequently appearing element in each column in the 2D array arr, ignoring any instances of masked_value""" + # Object-dtype arrays (e.g. strings) live on host, since cupy has no way + # to represent them on device. Dispatch to the array's own module so this + # works for both device (numeric) and host (object dtype) input. + xp = cp.get_array_module(arr) mask = _get_mask(arr, masked_value) n_features = arr.shape[1] most_frequent = np.empty(n_features, dtype=arr.dtype) for i in range(n_features): - feature_mask_idxs = cp.where(~mask[:, i])[0] - values, counts = cp.unique( + feature_mask_idxs = xp.where(~mask[:, i])[0] + values, counts = xp.unique( arr[feature_mask_idxs, i], return_counts=True ) count_max = counts.max() if count_max > 0: value = values[counts == count_max].min() else: - value = cp.nan + value = xp.nan most_frequent[i] = value - return cp.array(most_frequent) + return xp.array(most_frequent) diff --git a/python/cuml/tests/test_compose.py b/python/cuml/tests/test_compose.py index d5f65adfb4..e93b4873ec 100644 --- a/python/cuml/tests/test_compose.py +++ b/python/cuml/tests/test_compose.py @@ -13,6 +13,7 @@ from sklearn.compose import ( make_column_transformer as sk_make_column_transformer, ) +from sklearn.impute import SimpleImputer as skSimpleImputer from sklearn.preprocessing import Normalizer as skNormalizer from sklearn.preprocessing import OneHotEncoder as skOneHotEncoder from sklearn.preprocessing import PolynomialFeatures as skPolynomialFeatures @@ -24,6 +25,7 @@ from cuml.preprocessing import Normalizer as cuNormalizer from cuml.preprocessing import OneHotEncoder as cuOneHotEncoder from cuml.preprocessing import PolynomialFeatures as cuPolynomialFeatures +from cuml.preprocessing import SimpleImputer as cuSimpleImputer from cuml.preprocessing import StandardScaler as cuStandardScaler from cuml.testing.test_preproc_utils import ( # noqa: F401 assert_allclose, @@ -352,3 +354,83 @@ def test_column_transform_properly_handles_sub_output_type(): ] ).fit(df) transformer.transform(df) + + +def test_column_transformer_simple_imputer_categorical_cudf(): + """Regression test for https://github.com/rapidsai/cuml/issues/6183 + + ColumnTransformer + SimpleImputer on a native cuDF DataFrame with + categorical/string columns used to raise (KeyError / AttributeError on + ``.dtype``, later ``ValueError: Unsupported dtype object`` after + unrelated refactors). impute-then-transform is one of the most common + sklearn pipeline shapes, so this must work end to end. + """ + df = cudf.DataFrame( + { + "num1": [1.0, np.nan, 3.0], + "num2": [4.0, 5.0, np.nan], + "cat1": ["a", None, "c"], + "cat2": ["x", "y", None], + } + ) + df_np = df.to_pandas() + + num_cols = ["num1", "num2"] + cat_cols = ["cat1"] + mode_cols = ["cat2"] + + cu_transformer = cuColumnTransformer( + transformers=[ + ( + "num", + cuSimpleImputer(strategy="constant", fill_value=0), + num_cols, + ), + ( + "cat", + cuSimpleImputer( + strategy="constant", + fill_value="missing", + missing_values=pd.NA, + ), + cat_cols, + ), + ( + "mod", + cuSimpleImputer( + strategy="most_frequent", missing_values=pd.NA + ), + mode_cols, + ), + ] + ) + cu_result = cu_transformer.fit_transform(df) + + sk_transformer = skColumnTransformer( + transformers=[ + ( + "num", + skSimpleImputer(strategy="constant", fill_value=0), + num_cols, + ), + ( + "cat", + skSimpleImputer( + strategy="constant", + fill_value="missing", + missing_values=pd.NA, + ), + cat_cols, + ), + ( + "mod", + skSimpleImputer( + strategy="most_frequent", missing_values=pd.NA + ), + mode_cols, + ), + ] + ) + sk_result = sk_transformer.fit_transform(df_np) + + np.testing.assert_array_equal(np.asarray(cu_result), sk_result) diff --git a/python/cuml/tests/test_validation.py b/python/cuml/tests/test_validation.py index d466ffd9c7..a7002882a7 100644 --- a/python/cuml/tests/test_validation.py +++ b/python/cuml/tests/test_validation.py @@ -863,16 +863,11 @@ def test_check_array_dataframe_mixed_dtypes(kind, mem_type): "z": ["1", "2", "3.5", "4", "5"], } ) - # Non-numeric columns -> object dtype by default - if is_cuda_output(mem_type, df): - # cupy doesn't support object dtypes. We don't care what the exception - # is here, just that one is raised. - with pytest.raises(Exception, match="object"): - check_array(df, mem_type=mem_type) - else: - # dtype=None does no conversion by default - out = check_array(df, mem_type=mem_type) - assert out.dtype == "object" + # Non-numeric columns use host object dtype because cupy cannot represent + # arbitrary Python objects. + out = check_array(df, mem_type=mem_type) + assert isinstance(out, np.ndarray) + assert out.dtype == "object" # Can coerce all columns to specified dtype out = check_array(df, mem_type=mem_type, dtype=("float32", "float64")) @@ -928,13 +923,9 @@ def test_check_array_object_dtype(kind, mem_type): elif kind == "pandas": array = pd.Series(array) - if is_cuda_output(mem_type, array): - # cupy doesn't support object dtypes - with pytest.raises((ValueError, TypeError), match="object|str"): - check_array(array, mem_type=mem_type, ensure_2d=False) - else: - out = check_array(array, mem_type=mem_type, ensure_2d=False) - assert out.dtype == "object" + out = check_array(array, mem_type=mem_type, ensure_2d=False) + assert isinstance(out, np.ndarray) + assert out.dtype == "object" # Can coerce to numeric if specified out = check_array( From 96c48498601416e12bf5e8b65f106c6e473084fa Mon Sep 17 00:00:00 2001 From: nethum529 Date: Sat, 18 Jul 2026 15:14:06 -0500 Subject: [PATCH 2/6] fix(compose): address review feedback on object-dtype support - outputs._is_object_dtype: check scalar `.dtype` before per-column `.dtypes`, so extension-dtype Series (category/string/tz-datetime) no longer raise on iteration; DataFrame per-column detection is preserved. - ColumnTransformer._hstack and CumlArrayDescriptor: reuse the shared `_is_object_dtype` helper so object-dtype DataFrame outputs are caught, not just array `.dtype`. - SimpleImputer._concatenate_indicator: stack object-dtype (host) imputed data with the indicator mask on host instead of routing the host array through cupy.hstack, so add_indicator=True works for string columns. - _get_mask: drop the try/except that silently defaulted to False and could disable NA masking; pd.isna is total over the scalar sentinels. - De-duplicate: _imputation.py now imports the shared _is_object_dtype instead of a local copy. - Tests: add regression coverage for add_indicator=True on object data and for _is_object_dtype on Series / extension-dtype / DataFrame inputs. Signed-off-by: nethum529 --- .../preprocessing/_column_transformer.py | 7 +--- .../sklearn/preprocessing/_imputation.py | 15 ++++++-- python/cuml/cuml/internals/outputs.py | 22 ++++++++--- .../cuml/cuml/thirdparty_adapters/adapters.py | 13 +++---- python/cuml/tests/test_compose.py | 37 +++++++++++++++++++ 5 files changed, 72 insertions(+), 22 deletions(-) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py index e1e5b85ece..7697a83b7a 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py @@ -36,6 +36,7 @@ import cuml from cuml.internals.global_settings import _global_settings_data +from cuml.internals.outputs import _is_object_dtype from cuml.internals.validation import check_is_fitted, check_features, check_array from ..preprocessing._function_transformer import FunctionTransformer @@ -983,11 +984,7 @@ def _hstack(self, Xs): return cu_sparse.hstack(converted_Xs).tocsr() else: Xs = [f.toarray() if issparse(f) else f for f in Xs] - if any( - isinstance(getattr(X, "dtype", None), cpu_np.dtype) - and X.dtype.kind == "O" - for X in Xs - ): + if any(_is_object_dtype(X) for X in Xs): # Object dtype (e.g. string columns from a categorical # SimpleImputer) has no device representation - cupy has no # way to store it. Stack on host instead. diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py index 865c139a99..78a094f74e 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py @@ -29,7 +29,7 @@ StringInputTagMixin, _ensure_transformer_tags, ) -from cuml.internals.outputs import mlfunc, ReflectedAttr +from cuml.internals.outputs import _is_object_dtype, mlfunc, ReflectedAttr from cuml.internals.validation import check_is_fitted, check_inputs from ....thirdparty_adapters import ( @@ -140,7 +140,6 @@ def _concatenate_indicator(self, X_imputed, X_indicator): if not self.add_indicator: return X_imputed - hstack = sparse.hstack if sparse.issparse(X_imputed) else np.hstack if X_indicator is None: raise ValueError( "Data from the missing indicator are not provided. Call " @@ -148,7 +147,17 @@ def _concatenate_indicator(self, X_imputed, X_indicator): "implementation." ) - return hstack((X_imputed, X_indicator)) + if sparse.issparse(X_imputed): + return sparse.hstack((X_imputed, X_indicator)) + + if _is_object_dtype(X_imputed) or _is_object_dtype(X_indicator): + arrays = [ + array.get() if isinstance(array, np.ndarray) else array + for array in (X_imputed, X_indicator) + ] + return cpu_np.hstack(arrays) + + return np.hstack((X_imputed, X_indicator)) def __sklearn_tags__(self): tags = super().__sklearn_tags__() diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index 1b90be1bfe..0d2b6fa1ad 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -299,14 +299,24 @@ def infer_output_type(array, array_like="numpy"): def _is_object_dtype(res): """Check for NumPy object dtype on an array-like or dataframe-like.""" + dtype = getattr(res, "dtype", None) + if dtype is not None: + # Array-like or Series: a single (possibly extension) dtype. Only a + # plain numpy object dtype has no device representation; extension + # dtypes are a separate, unsupported case handled elsewhere. + return isinstance(dtype, np.dtype) and dtype.kind == "O" + dtypes = getattr(res, "dtypes", None) if dtypes is not None: - return any( - isinstance(dtype, np.dtype) and dtype.kind == "O" - for dtype in dtypes - ) - dtype = getattr(res, "dtype", None) - return isinstance(dtype, np.dtype) and dtype.kind == "O" + # DataFrame-like: dtypes is per-column. + try: + return any( + isinstance(dt, np.dtype) and dt.kind == "O" for dt in dtypes + ) + except TypeError: + return False + + return False class ArrayIndexPair: diff --git a/python/cuml/cuml/thirdparty_adapters/adapters.py b/python/cuml/cuml/thirdparty_adapters/adapters.py index 1980b69c1f..89572fb08f 100644 --- a/python/cuml/cuml/thirdparty_adapters/adapters.py +++ b/python/cuml/cuml/thirdparty_adapters/adapters.py @@ -12,14 +12,11 @@ def _get_mask(X, value_to_mask): if isinstance(value_to_mask, str) and value_to_mask == "NaN": xp = cp.get_array_module(X) return xp.isnan(X) - try: - # NaN-like sentinels (np.nan, None, pd.NA, pd.NaT, ...) require an - # NA-aware comparison: a plain `==` against e.g. pd.NA propagates - # instead of returning a boolean mask, and `cp.isnan` doesn't accept - # non-numeric scalars like pd.NA in the first place. - is_na_sentinel = pd.isna(value_to_mask) - except (TypeError, ValueError): - is_na_sentinel = False + # NaN-like sentinels (np.nan, None, pd.NA, pd.NaT, ...) require an + # NA-aware comparison: a plain `==` against e.g. pd.NA propagates + # instead of returning a boolean mask, and `cp.isnan` doesn't accept + # non-numeric scalars like pd.NA in the first place. + is_na_sentinel = pd.isna(value_to_mask) if is_na_sentinel: if isinstance(X, cp.ndarray): return cp.isnan(X) diff --git a/python/cuml/tests/test_compose.py b/python/cuml/tests/test_compose.py index e93b4873ec..2084913d0d 100644 --- a/python/cuml/tests/test_compose.py +++ b/python/cuml/tests/test_compose.py @@ -434,3 +434,40 @@ def test_column_transformer_simple_imputer_categorical_cudf(): sk_result = sk_transformer.fit_transform(df_np) np.testing.assert_array_equal(np.asarray(cu_result), sk_result) + + +def test_simple_imputer_add_indicator_object_cudf(): + """Regression: SimpleImputer(add_indicator=True) on string/object columns + must stack the imputed (host object) data with the indicator mask on host + instead of routing the host array through cupy.hstack (issue #6183 follow-up). + """ + df = cudf.DataFrame( + { + "cat1": ["a", None, "c", "a"], + "cat2": ["x", "y", None, "x"], + } + ) + df_np = df.to_pandas() + cu_imp = cuSimpleImputer( + strategy="most_frequent", missing_values=pd.NA, add_indicator=True + ) + sk_imp = skSimpleImputer( + strategy="most_frequent", missing_values=pd.NA, add_indicator=True + ) + + cu_result = cu_imp.fit_transform(df) + sk_result = sk_imp.fit_transform(df_np) + + np.testing.assert_array_equal(np.asarray(cu_result), np.asarray(sk_result)) + + +def test_is_object_dtype_handles_series_and_extension_dtypes(): + from cuml.internals.outputs import _is_object_dtype + + assert _is_object_dtype(pd.Series(["a", "b"])) is True + assert _is_object_dtype(pd.Series([1, 2, 3])) is False + assert _is_object_dtype(pd.Series(pd.Categorical(["a", "b"]))) is False + assert _is_object_dtype(pd.Series(["a"], dtype="string")) is False + assert _is_object_dtype(pd.DataFrame({"a": ["x"], "b": [1]})) is True + assert _is_object_dtype(np.array(["a", "b"], dtype=object)) is True + assert _is_object_dtype(np.array([1, 2, 3])) is False From 090007f3b6cb68b979bdcca25dfe4ef43cecc5f3 Mon Sep 17 00:00:00 2001 From: nethum529 Date: Mon, 20 Jul 2026 23:47:11 -0500 Subject: [PATCH 3/6] fix object output review findings Signed-off-by: nethum529 --- python/cuml/cuml/internals/outputs.py | 55 +++++++++++++++++-- .../cuml/cuml/thirdparty_adapters/adapters.py | 19 +++++-- python/cuml/tests/test_adapters.py | 25 +++++++++ python/cuml/tests/test_compose.py | 10 ++++ python/cuml/tests/test_reflection.py | 48 ++++++++++++++++ 5 files changed, 146 insertions(+), 11 deletions(-) diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index 0d2b6fa1ad..fa7fe15a65 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -516,11 +516,6 @@ def convert_arrays( if isinstance(obj, ClassLabels): return obj.to_output(output_type, index=index) - # NumPy object arrays have no device representation. - # Keep them on host for method outputs and reflected attributes. - if isinstance(obj, np.ndarray) and _is_object_dtype(obj): - return obj - if isinstance(obj, np.ndarray): if output_type == "numpy": return obj @@ -532,6 +527,56 @@ def convert_arrays( return pd.Series(obj.flatten(), index=index) return pd.DataFrame(obj, index=index) return pd.Series(obj, index=index) + elif _is_object_dtype(obj): + # NumPy object arrays have no device representation. Preserve + # host arrays unless the requested output type wraps them. + if output_type not in ( + "cudf", + "df_obj", + "dataframe", + "series", + ): + return obj + + if hasattr(index, "to_pandas"): + index = index.to_pandas() + if output_type == "series": + if obj.ndim == 2: + if obj.shape[1] == 1: + obj = obj.flatten() + else: + raise ValueError( + "Only single dimensional arrays can be transformed to" + " Series." + ) + elif obj.ndim == 0: + obj = obj[None] + elif output_type == "dataframe": + if obj.ndim == 1: + obj = obj[:, None] + elif obj.ndim == 0: + obj = obj[None, None] + + if obj.ndim == 2: + if ( + one_col_2d_as_series + and obj.shape[1] == 1 + and output_type != "dataframe" + ): + host_df = pd.Series(obj.flatten(), index=index) + else: + host_df = pd.DataFrame(obj, index=index) + else: + host_df = pd.Series(obj, index=index) + + try: + return cudf.from_pandas(host_df) + except (TypeError, ValueError, NotImplementedError): + if isinstance(host_df, pd.DataFrame): + # cudf cannot represent every mixed object DataFrame. + # Preserve the host DataFrame instead of losing its data. + return host_df + raise else: # Other output types use device memory, coerce to cupy and take # cupy code path. diff --git a/python/cuml/cuml/thirdparty_adapters/adapters.py b/python/cuml/cuml/thirdparty_adapters/adapters.py index 89572fb08f..653f2fd8a6 100644 --- a/python/cuml/cuml/thirdparty_adapters/adapters.py +++ b/python/cuml/cuml/thirdparty_adapters/adapters.py @@ -7,17 +7,24 @@ import pandas as pd +def _is_na_sentinel(value): + """Return whether a scalar missing-value sentinel is NA-like.""" + if isinstance(value, str): + return value == "NaN" + return pd.isna(value) + + def _get_mask(X, value_to_mask): """Compute the boolean mask X == missing_values.""" if isinstance(value_to_mask, str) and value_to_mask == "NaN": - xp = cp.get_array_module(X) - return xp.isnan(X) + if isinstance(X, cp.ndarray): + return cp.isnan(X) + return pd.isna(X) # NaN-like sentinels (np.nan, None, pd.NA, pd.NaT, ...) require an # NA-aware comparison: a plain `==` against e.g. pd.NA propagates # instead of returning a boolean mask, and `cp.isnan` doesn't accept # non-numeric scalars like pd.NA in the first place. - is_na_sentinel = pd.isna(value_to_mask) - if is_na_sentinel: + if _is_na_sentinel(value_to_mask): if isinstance(X, cp.ndarray): return cp.isnan(X) # Host (e.g. object dtype) arrays can't use isnan - fall back to an @@ -33,7 +40,7 @@ def _masked_column_median(arr, masked_value): mask = _get_mask(arr, masked_value) if arr.size == 0: return cp.full(arr.shape[1], cp.nan) - if not cp.isnan(masked_value): + if not _is_na_sentinel(masked_value): arr_sorted = arr.copy() # If nan is not the missing value, any column with nans should # have a median of nan @@ -72,7 +79,7 @@ def _masked_column_mean(arr, masked_value): count_missing_values = mask.sum(axis=0) n_elems = arr.shape[0] - count_missing_values mean = cp.nansum(arr, axis=0) - if not cp.isnan(masked_value): + if not _is_na_sentinel(masked_value): mean -= count_missing_values * masked_value mean /= n_elems return mean diff --git a/python/cuml/tests/test_adapters.py b/python/cuml/tests/test_adapters.py index b59542c473..4f61243c2f 100644 --- a/python/cuml/tests/test_adapters.py +++ b/python/cuml/tests/test_adapters.py @@ -8,6 +8,7 @@ import cupy as cp import cupyx as cpx import numpy as np +import pandas as pd import pytest from scipy import stats from sklearn.utils._mask import _get_mask as sk_get_mask @@ -110,6 +111,30 @@ def test_get_mask(failure_logger, mask_dataset): assert_allclose(cu_mask, sk_mask) +def test_get_mask_nan_string_on_host_object_array(): + X = np.array([1.0, np.nan, "present"], dtype=object) + + np.testing.assert_array_equal( + cu_get_mask(X, value_to_mask="NaN"), [False, True, False] + ) + + +@pytest.mark.parametrize( + ("function", "expected"), + [ + (_masked_column_mean, [3.0, 5.0]), + (_masked_column_median, [3.0, 5.0]), + ], +) +@pytest.mark.parametrize("missing_value", [pd.NA, "NaN"]) +def test_masked_column_numeric_na_sentinel(function, expected, missing_value): + X = cp.array([[1.0, cp.nan], [3.0, 4.0], [5.0, 6.0]]) + + result = function(X, missing_value) + + np.testing.assert_allclose(result.get(), expected) + + def test_masked_column_median(failure_logger, mask_dataset): mask_value, X_np, X = mask_dataset median = _masked_column_median(X, mask_value).get() diff --git a/python/cuml/tests/test_compose.py b/python/cuml/tests/test_compose.py index 2084913d0d..ed8346c0a3 100644 --- a/python/cuml/tests/test_compose.py +++ b/python/cuml/tests/test_compose.py @@ -461,6 +461,16 @@ def test_simple_imputer_add_indicator_object_cudf(): np.testing.assert_array_equal(np.asarray(cu_result), np.asarray(sk_result)) +def test_simple_imputer_add_indicator_clone_params(): + imputer = cuSimpleImputer(add_indicator=True, missing_values=pd.NA) + + cloned = sk_clone(imputer) + + params = cloned.get_params() + assert params["add_indicator"] is True + assert params["missing_values"] is pd.NA + + def test_is_object_dtype_handles_series_and_extension_dtypes(): from cuml.internals.outputs import _is_object_dtype diff --git a/python/cuml/tests/test_reflection.py b/python/cuml/tests/test_reflection.py index a16851b6f6..3042953709 100644 --- a/python/cuml/tests/test_reflection.py +++ b/python/cuml/tests/test_reflection.py @@ -376,6 +376,54 @@ def test_convert_arrays_dataframe_with_index( ) +@pytest.mark.parametrize( + ("output_type", "expected_type"), + [ + ("pandas", pd.DataFrame), + ("cudf", cudf.DataFrame), + ], +) +def test_convert_arrays_object_array_dataframe_output_with_index( + output_type, expected_type +): + arr = np.array([["a", "x"], ["b", "y"]], dtype=object) + index = pd.Index(["first", "second"]) + + result = convert_arrays(arr, output_type, index=index) + + assert isinstance(result, expected_type) + cudf.testing.assert_frame_equal( + cudf.DataFrame(result), + cudf.from_pandas(pd.DataFrame(arr, index=index)), + ) + + +@pytest.mark.parametrize( + ("output_type", "expected_type"), + [ + ("series", cudf.Series), + ("dataframe", cudf.DataFrame), + ], +) +def test_convert_arrays_object_array_explicit_dataframe_outputs( + output_type, expected_type +): + arr = np.array(["a", "b"], dtype=object) + index = pd.Index(["first", "second"]) + + result = convert_arrays(arr, output_type, index=index) + + assert isinstance(result, expected_type) + if output_type == "series": + cudf.testing.assert_series_equal( + result, cudf.from_pandas(pd.Series(arr, index=index)) + ) + else: + cudf.testing.assert_frame_equal( + result, cudf.from_pandas(pd.DataFrame(arr, index=index)) + ) + + @pytest.mark.parametrize( "construct", [ From e841e5a57afd9b48436e05f231e14c7a9e9fb2d7 Mon Sep 17 00:00:00 2001 From: nethum529 Date: Mon, 20 Jul 2026 23:47:59 -0500 Subject: [PATCH 4/6] Fix copyright headers Signed-off-by: nethum529 --- python/cuml/cuml/internals/validation.py | 2 +- python/cuml/cuml/thirdparty_adapters/adapters.py | 2 +- python/cuml/tests/test_adapters.py | 2 +- python/cuml/tests/test_compose.py | 2 +- python/cuml/tests/test_validation.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/cuml/cuml/internals/validation.py b/python/cuml/cuml/internals/validation.py index d24c5e8540..3ab024cd13 100644 --- a/python/cuml/cuml/internals/validation.py +++ b/python/cuml/cuml/internals/validation.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # import numbers diff --git a/python/cuml/cuml/thirdparty_adapters/adapters.py b/python/cuml/cuml/thirdparty_adapters/adapters.py index 653f2fd8a6..f8e0ec57ca 100644 --- a/python/cuml/cuml/thirdparty_adapters/adapters.py +++ b/python/cuml/cuml/thirdparty_adapters/adapters.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # import cupy as cp diff --git a/python/cuml/tests/test_adapters.py b/python/cuml/tests/test_adapters.py index 4f61243c2f..5442bf6f14 100644 --- a/python/cuml/tests/test_adapters.py +++ b/python/cuml/tests/test_adapters.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # diff --git a/python/cuml/tests/test_compose.py b/python/cuml/tests/test_compose.py index ed8346c0a3..30be42ae3c 100644 --- a/python/cuml/tests/test_compose.py +++ b/python/cuml/tests/test_compose.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # diff --git a/python/cuml/tests/test_validation.py b/python/cuml/tests/test_validation.py index a7002882a7..eb9a6ae69b 100644 --- a/python/cuml/tests/test_validation.py +++ b/python/cuml/tests/test_validation.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import re import warnings From b20ad119a7447d39c4b3f0c089c655e6f46c47b8 Mon Sep 17 00:00:00 2001 From: nethum529 Date: Wed, 22 Jul 2026 19:11:03 -0500 Subject: [PATCH 5/6] Fix object output conversion Signed-off-by: nethum529 --- python/cuml/cuml/internals/outputs.py | 11 ++++++++--- python/cuml/tests/test_reflection.py | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index fa7fe15a65..e05773ee6e 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -529,14 +529,19 @@ def convert_arrays( return pd.Series(obj, index=index) elif _is_object_dtype(obj): # NumPy object arrays have no device representation. Preserve - # host arrays unless the requested output type wraps them. - if output_type not in ( + # host arrays for "array" or wrap them in a dataframe-like type. + if output_type == "array": + return obj + elif output_type not in ( "cudf", "df_obj", "dataframe", "series", ): - return obj + raise TypeError( + f"{output_type=!r} doesn't support outputs of dtype " + f"object and shape {obj.shape}" + ) if hasattr(index, "to_pandas"): index = index.to_pandas() diff --git a/python/cuml/tests/test_reflection.py b/python/cuml/tests/test_reflection.py index 3042953709..ad630abd6a 100644 --- a/python/cuml/tests/test_reflection.py +++ b/python/cuml/tests/test_reflection.py @@ -424,6 +424,23 @@ def test_convert_arrays_object_array_explicit_dataframe_outputs( ) +@pytest.mark.parametrize("output_type", ["cupy", "numba"]) +def test_convert_arrays_object_array_device_output_error(output_type): + arr = np.array(["a", "b"], dtype=object) + + with pytest.raises( + TypeError, + match=f"output_type={output_type!r} doesn't support outputs of dtype", + ): + convert_arrays(arr, output_type) + + +def test_convert_arrays_object_array_array_output(): + arr = np.array(["a", "b"], dtype=object) + + assert convert_arrays(arr, "array") is arr + + @pytest.mark.parametrize( "construct", [ From c9bab098ae4c7560415e731a7505aee09479bac8 Mon Sep 17 00:00:00 2001 From: nethum529 Date: Wed, 22 Jul 2026 19:13:11 -0500 Subject: [PATCH 6/6] Fix internal object output conversion Signed-off-by: nethum529 --- python/cuml/cuml/internals/outputs.py | 12 ++++-------- python/cuml/tests/test_reflection.py | 5 +++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index e05773ee6e..8c3887d964 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -529,15 +529,11 @@ def convert_arrays( return pd.Series(obj, index=index) elif _is_object_dtype(obj): # NumPy object arrays have no device representation. Preserve - # host arrays for "array" or wrap them in a dataframe-like type. - if output_type == "array": + # host arrays for "array" and internal "cuml" outputs, or wrap + # them in a dataframe-like type. + if output_type in ("array", "cuml"): return obj - elif output_type not in ( - "cudf", - "df_obj", - "dataframe", - "series", - ): + elif output_type in ("cupy", "numba"): raise TypeError( f"{output_type=!r} doesn't support outputs of dtype " f"object and shape {obj.shape}" diff --git a/python/cuml/tests/test_reflection.py b/python/cuml/tests/test_reflection.py index ad630abd6a..dcaeb9b8a6 100644 --- a/python/cuml/tests/test_reflection.py +++ b/python/cuml/tests/test_reflection.py @@ -435,10 +435,11 @@ def test_convert_arrays_object_array_device_output_error(output_type): convert_arrays(arr, output_type) -def test_convert_arrays_object_array_array_output(): +@pytest.mark.parametrize("output_type", ["array", "cuml"]) +def test_convert_arrays_object_array_array_output(output_type): arr = np.array(["a", "b"], dtype=object) - assert convert_arrays(arr, "array") is arr + assert convert_arrays(arr, output_type) is arr @pytest.mark.parametrize(