diff --git a/cpp/include/cudf_test/tdigest_utilities.hpp b/cpp/include/cudf_test/tdigest_utilities.hpp index dc252d14b08..2b8ca66fbb9 100644 --- a/cpp/include/cudf_test/tdigest_utilities.hpp +++ b/cpp/include/cudf_test/tdigest_utilities.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -65,12 +65,24 @@ inline T rand_range(T min, T max) return min + static_cast(frand() * (max - min)); } +/** + * @brief Generate a typed column from a bucketed percentile distribution. + * + * @param buckets Upper bound for each generated bucket + * @param sizes Number of values generated for each bucket + * @param t Data type of the returned column + * @param sorted Whether to sort the generated values before conversion + * @param mr Memory resources used for returned and temporary allocations + * @return Generated column + */ inline std::unique_ptr generate_typed_percentile_distribution( std::vector const& buckets, std::vector const& sizes, data_type t, - bool sorted = false) + bool sorted = false, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { + auto const temporary_mr = mr.get_temporary_mr(); srand(0); std::vector values; @@ -87,8 +99,8 @@ inline std::unique_ptr generate_typed_percentile_distribution( if (sorted) { std::sort(values.begin(), values.end()); } - cudf::test::fixed_width_column_wrapper src(values.begin(), values.end()); - return cudf::cast(src, t); + cudf::test::fixed_width_column_wrapper src(values.begin(), values.end(), temporary_mr); + return cudf::cast(src, t, cudf::get_default_stream(), mr.get_output_mr()); } // "standardized" means the parameters sent into generate_typed_percentile_distribution. the intent @@ -96,31 +108,53 @@ inline std::unique_ptr generate_typed_percentile_distribution( // percentile_approx tests. std::vector // buckets{10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0}; std::vector // sizes{50000, 50000, 50000, 50000, 50000, 100000, 100000, 100000, 100000, 100000}; +/** + * @brief Generate the standardized percentile distribution used by T-digest tests. + * + * @param t Data type of the returned column + * @param sorted Whether to sort the generated values before conversion + * @param mr Memory resources used for returned and temporary allocations + * @return Generated column + */ inline std::unique_ptr generate_standardized_percentile_distribution( - data_type t = data_type{type_id::FLOAT64}, bool sorted = false) + data_type t = data_type{type_id::FLOAT64}, + bool sorted = false, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { std::vector buckets{10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f, 70.0f, 80.0, 90.0f, 100.0f}; std::vector b_sizes{ 50000, 50000, 50000, 50000, 50000, 100000, 100000, 100000, 100000, 100000}; - return generate_typed_percentile_distribution(buckets, b_sizes, t, sorted); + return generate_typed_percentile_distribution(buckets, b_sizes, t, sorted, mr); } /** * @brief Compare a tdigest column against a sampling of expected values. + * + * @param tdv T-digest column to validate + * @param h_expected Expected centroid index, mean, and weight tuples + * @param mr Memory resources used for temporary device allocations */ void tdigest_sample_compare(cudf::tdigest::tdigest_column_view const& tdv, - std::vector const& h_expected); + std::vector const& h_expected, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); /** * @brief Compare the min/max values of a tdigest against inputs. + * + * @tparam T Input element type + * @param tdv T-digest column to validate + * @param input_values Values whose extrema are expected in the T-digest + * @param mr Memory resources used for temporary device allocations */ template void tdigest_minmax_compare(cudf::tdigest::tdigest_column_view const& tdv, - cudf::column_view const& input_values) + cudf::column_view const& input_values, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { - using ScalarType = cudf::scalar_type_t; + auto const temporary_mr = mr.get_temporary_mr(); + using ScalarType = cudf::scalar_type_t; - auto [col_min, col_max] = cudf::minmax(input_values); + auto [col_min, col_max] = cudf::minmax(input_values, cudf::get_default_stream(), temporary_mr); auto min_scalar = static_cast(col_min.get()); auto max_scalar = static_cast(col_max.get()); @@ -148,13 +182,22 @@ struct expected_tdigest { /** * @brief Create an expected tdigest column given component inputs. + * + * @param groups T-digest component values for each output row + * @param mr Memory resources used for returned and temporary allocations + * @return Expected T-digest column */ -std::unique_ptr make_expected_tdigest_column(std::vector const& groups); +std::unique_ptr make_expected_tdigest_column( + std::vector const& groups, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); // shared test for groupby/reduction. template -void tdigest_simple_aggregation(Func op) +void tdigest_simple_aggregation(Func op, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { + auto const temporary_mr = mr.get_temporary_mr(); + auto const temporary_resources = cudf::memory_resources{temporary_mr, temporary_mr}; bool is_cpu_cluster_computation_disabled[2] = {true, false}; for (int idx = 0; idx < 2; idx++) { cudf::tdigest::detail::is_cpu_cluster_computation_disabled = @@ -162,29 +205,35 @@ void tdigest_simple_aggregation(Func op) // create a tdigest that has far fewer values in it than the delta value. this should result // in every value remaining uncompressed - cudf::test::fixed_width_column_wrapper values{126, 15, 1, 99, 67}; + cudf::test::fixed_width_column_wrapper values({126, 15, 1, 99, 67}, temporary_resources); int const delta = 1000; auto result = cudf::type_dispatcher( static_cast(values).type(), tdigest_gen{}, op, values, delta); - cudf::test::fixed_width_column_wrapper raw_mean({1, 15, 67, 99, 126}); - cudf::test::fixed_width_column_wrapper weight{1, 1, 1, 1, 1}; - auto mean = cudf::cast(raw_mean, data_type{type_id::FLOAT64}); + cudf::test::fixed_width_column_wrapper raw_mean({1, 15, 67, 99, 126}, temporary_resources); + cudf::test::fixed_width_column_wrapper weight({1, 1, 1, 1, 1}, temporary_resources); + auto mean = + cudf::cast(raw_mean, data_type{type_id::FLOAT64}, cudf::get_default_stream(), temporary_mr); double const min = 1; double const max = 126; auto expected = make_expected_tdigest_column({{*mean, weight, static_cast(static_cast(min)), - static_cast(static_cast(max))}}); + static_cast(static_cast(max))}}, + temporary_resources); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, *expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + *result, *expected, cudf::test::debug_output_level::FIRST_ERROR, mr); } } // shared test for groupby/reduction. template -void tdigest_simple_with_nulls_aggregation(Func op) +void tdigest_simple_with_nulls_aggregation( + Func op, cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { + auto const temporary_mr = mr.get_temporary_mr(); + auto const temporary_resources = cudf::memory_resources{temporary_mr, temporary_mr}; bool is_cpu_cluster_computation_disabled[2] = {true, false}; for (int idx = 0; idx < 2; idx++) { cudf::tdigest::detail::is_cpu_cluster_computation_disabled = @@ -192,30 +241,37 @@ void tdigest_simple_with_nulls_aggregation(Func op) // create a tdigest that has far fewer values in it than the delta value. this should result // in every value remaining uncompressed - cudf::test::fixed_width_column_wrapper values{{122, 15, 1, 99, 67, 101, 100, 84, 44, 2}, - {1, 0, 1, 0, 1, 0, 1, 0, 1, 0}}; + cudf::test::fixed_width_column_wrapper values({122, 15, 1, 99, 67, 101, 100, 84, 44, 2}, + {1, 0, 1, 0, 1, 0, 1, 0, 1, 0}, + temporary_resources); int const delta = 1000; auto result = cudf::type_dispatcher( static_cast(values).type(), tdigest_gen{}, op, values, delta); - cudf::test::fixed_width_column_wrapper raw_mean({1, 44, 67, 100, 122}); - cudf::test::fixed_width_column_wrapper weight{1, 1, 1, 1, 1}; - auto mean = cudf::cast(raw_mean, data_type{type_id::FLOAT64}); + cudf::test::fixed_width_column_wrapper raw_mean({1, 44, 67, 100, 122}, temporary_resources); + cudf::test::fixed_width_column_wrapper weight({1, 1, 1, 1, 1}, temporary_resources); + auto mean = + cudf::cast(raw_mean, data_type{type_id::FLOAT64}, cudf::get_default_stream(), temporary_mr); double const min = 1; double const max = 122; auto expected = make_expected_tdigest_column({{*mean, weight, static_cast(static_cast(min)), - static_cast(static_cast(max))}}); + static_cast(static_cast(max))}}, + temporary_resources); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, *expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + *result, *expected, cudf::test::debug_output_level::FIRST_ERROR, mr); } } // shared test for groupby/reduction. template -void tdigest_simple_all_nulls_aggregation(Func op) +void tdigest_simple_all_nulls_aggregation( + Func op, cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { + auto const temporary_mr = mr.get_temporary_mr(); + auto const temporary_resources = cudf::memory_resources{temporary_mr, temporary_mr}; bool is_cpu_cluster_computation_disabled[2] = {true, false}; for (int idx = 0; idx < 2; idx++) { cudf::tdigest::detail::is_cpu_cluster_computation_disabled = @@ -223,31 +279,38 @@ void tdigest_simple_all_nulls_aggregation(Func op) // create a tdigest that has far fewer values in it than the delta value. this should result // in every value remaining uncompressed - cudf::test::fixed_width_column_wrapper values{{122, 15, 1, 99, 67, 101, 100, 84, 44, 2}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; + cudf::test::fixed_width_column_wrapper values({122, 15, 1, 99, 67, 101, 100, 84, 44, 2}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + temporary_resources); int const delta = 1000; auto result = cudf::type_dispatcher( static_cast(values).type(), tdigest_gen{}, op, values, delta); // NOTE: an empty tdigest column still has 1 row. auto expected = cudf::tdigest::detail::make_empty_tdigests_column( - 1, cudf::get_default_stream(), cudf::get_current_device_resource_ref()); + 1, cudf::get_default_stream(), temporary_mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, *expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + *result, *expected, cudf::test::debug_output_level::FIRST_ERROR, mr); } } // Note: there is no need to test different types here as the internals of a tdigest are always // the same regardless of input. template -void tdigest_merge_simple(Func op, MergeFunc merge_op) +void tdigest_merge_simple(Func op, + MergeFunc merge_op, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { + auto const temporary_mr = mr.get_temporary_mr(); + auto const temporary_resources = cudf::memory_resources{temporary_mr, temporary_mr}; bool is_cpu_cluster_computation_disabled[2] = {true, false}; for (int idx = 0; idx < 2; idx++) { cudf::tdigest::detail::is_cpu_cluster_computation_disabled = is_cpu_cluster_computation_disabled[idx]; - auto values = generate_standardized_percentile_distribution(data_type{type_id::FLOAT64}); + auto values = generate_standardized_percentile_distribution( + data_type{type_id::FLOAT64}, false, temporary_resources); CUDF_EXPECTS(values->size() == 750000, "Unexpected distribution size"); auto split_values = cudf::split(*values, {250000, 500000}); @@ -272,7 +335,7 @@ void tdigest_merge_simple(Func op, MergeFunc merge_op) int const merge_delta = 1000; // merge them - auto merge_input = cudf::concatenate(part_views); + auto merge_input = cudf::concatenate(part_views, cudf::get_default_stream(), temporary_mr); auto result = merge_op(*merge_input, merge_delta); cudf::tdigest::tdigest_column_view tdv(*result); @@ -290,18 +353,20 @@ void tdigest_merge_simple(Func op, MergeFunc merge_op) {625, 98.20470345147104751504, 405}, {700, 99.96818381983835877236, 56}, {711, 99.99970905482754801596, 1}}; - tdigest_sample_compare(tdv, expected); + tdigest_sample_compare(tdv, expected, mr); // verify min/max - tdigest_minmax_compare(tdv, *values); + tdigest_minmax_compare(tdv, *values, mr); } } } // shared test for groupby/reduction. template -void tdigest_merge_empty(MergeFunc merge_op) +void tdigest_merge_empty(MergeFunc merge_op, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { + auto const temporary_mr = mr.get_temporary_mr(); bool is_cpu_cluster_computation_disabled[2] = {true, false}; for (int idx = 0; idx < 2; idx++) { cudf::tdigest::detail::is_cpu_cluster_computation_disabled = @@ -309,24 +374,25 @@ void tdigest_merge_empty(MergeFunc merge_op) // 3 empty tdigests all in the same group auto a = cudf::tdigest::detail::make_empty_tdigests_column( - 1, cudf::get_default_stream(), cudf::get_current_device_resource_ref()); + 1, cudf::get_default_stream(), temporary_mr); auto b = cudf::tdigest::detail::make_empty_tdigests_column( - 1, cudf::get_default_stream(), cudf::get_current_device_resource_ref()); + 1, cudf::get_default_stream(), temporary_mr); auto c = cudf::tdigest::detail::make_empty_tdigests_column( - 1, cudf::get_default_stream(), cudf::get_current_device_resource_ref()); + 1, cudf::get_default_stream(), temporary_mr); std::vector cols; cols.push_back(*a); cols.push_back(*b); cols.push_back(*c); - auto values = cudf::concatenate(cols); + auto values = cudf::concatenate(cols, cudf::get_default_stream(), temporary_mr); auto const delta = 1000; auto result = merge_op(*values, delta); auto expected = cudf::tdigest::detail::make_empty_tdigests_column( - 1, cudf::get_default_stream(), cudf::get_current_device_resource_ref()); + 1, cudf::get_default_stream(), temporary_mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *result); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + *expected, *result, cudf::test::debug_output_level::FIRST_ERROR, mr); } } diff --git a/cpp/tests/reductions/tdigest_tests.cpp b/cpp/tests/reductions/tdigest_tests.cpp index 9ed6fd4eb95..f79af89c65b 100644 --- a/cpp/tests/reductions/tdigest_tests.cpp +++ b/cpp/tests/reductions/tdigest_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,8 @@ #include +#include + template struct ReductionTDigestAllTypes : public cudf::test::BaseFixture {}; TYPED_TEST_SUITE(ReductionTDigestAllTypes, cudf::test::NumericTypes); @@ -77,6 +79,51 @@ TEST_F(ReductionTDigestMerge, Simple) cudf::test::tdigest_merge_simple(reduce_op{}, reduce_merge_op{}); } +TEST_F(ReductionTDigestMerge, TestUtilityMemoryResourceControl) +{ + auto upstream = cudf::get_current_device_resource_ref(); + auto output_mr = rmm::mr::statistics_resource_adaptor(upstream); + auto temporary_mr = rmm::mr::statistics_resource_adaptor(upstream); + auto resources = cudf::memory_resources{output_mr, temporary_mr}; + + { + auto distribution = cudf::test::generate_typed_percentile_distribution( + {10.0}, {4}, cudf::data_type{cudf::type_id::FLOAT64}, false, resources); + cudf::test::get_default_stream().synchronize(); + EXPECT_GT(output_mr.get_bytes_counter().value, 0); + EXPECT_EQ(temporary_mr.get_bytes_counter().value, 0); + EXPECT_GT(temporary_mr.get_bytes_counter().total, 0); + } + cudf::test::get_default_stream().synchronize(); + EXPECT_EQ(output_mr.get_bytes_counter().value, 0); + + cudf::test::fixed_width_column_wrapper means{1.0, 2.0}; + cudf::test::fixed_width_column_wrapper weights{1.0, 1.0}; + auto validation_output_mr = rmm::mr::statistics_resource_adaptor(upstream); + auto validation_temporary_mr = rmm::mr::statistics_resource_adaptor(upstream); + auto validation_resources = cudf::memory_resources{validation_output_mr, validation_temporary_mr}; + auto const temporary_bytes_before = temporary_mr.get_bytes_counter().total; + + { + auto expected = + cudf::test::make_expected_tdigest_column({{means, weights, 1.0, 2.0}}, resources); + cudf::tdigest::tdigest_column_view tdv(*expected); + cudf::test::tdigest_sample_compare(tdv, {{0, 1.0, 1.0}, {1, 2.0, 1.0}}, validation_resources); + cudf::test::tdigest_minmax_compare(tdv, means, validation_resources); + + cudf::test::get_default_stream().synchronize(); + EXPECT_GT(output_mr.get_bytes_counter().value, 0); + EXPECT_EQ(temporary_mr.get_bytes_counter().value, 0); + EXPECT_GT(temporary_mr.get_bytes_counter().total, temporary_bytes_before); + EXPECT_EQ(validation_output_mr.get_bytes_counter().value, 0); + EXPECT_EQ(validation_output_mr.get_bytes_counter().total, 0); + EXPECT_EQ(validation_temporary_mr.get_bytes_counter().value, 0); + EXPECT_GT(validation_temporary_mr.get_bytes_counter().total, 0); + } + cudf::test::get_default_stream().synchronize(); + EXPECT_EQ(output_mr.get_bytes_counter().value, 0); +} + // tests an issue with the cluster generating code with a small number of centroids that have large // weights TEST_F(ReductionTDigestMerge, FewHeavyCentroids) diff --git a/cpp/tests/utilities/tdigest_utilities.cpp b/cpp/tests/utilities/tdigest_utilities.cpp index 0195edf91b7..b019201f198 100644 --- a/cpp/tests/utilities/tdigest_utilities.cpp +++ b/cpp/tests/utilities/tdigest_utilities.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -25,8 +25,10 @@ namespace cudf { namespace test { void tdigest_sample_compare(cudf::tdigest::tdigest_column_view const& tdv, - std::vector const& h_expected) + std::vector const& h_expected, + cudf::memory_resources mr) { + auto const temporary_mr = mr.get_temporary_mr(); column_view result_mean = tdv.means(); column_view result_weight = tdv.weights(); @@ -45,48 +47,73 @@ void tdigest_sample_compare(cudf::tdigest::tdigest_column_view const& tdv, }); auto d_expected_src = cudf::detail::make_device_uvector_async( - h_expected_src, cudf::get_default_stream(), cudf::get_current_device_resource_ref()); + h_expected_src, cudf::get_default_stream(), temporary_mr); auto d_expected_mean = cudf::detail::make_device_uvector_async( - h_expected_mean, cudf::get_default_stream(), cudf::get_current_device_resource_ref()); + h_expected_mean, cudf::get_default_stream(), temporary_mr); auto d_expected_weight = cudf::detail::make_device_uvector_async( - h_expected_weight, cudf::get_default_stream(), cudf::get_current_device_resource_ref()); - - auto map = cudf::device_span(d_expected_src); - auto sampled_result_mean = - std::move(cudf::gather(cudf::table_view({result_mean}), map)->release().front()); - auto sampled_result_weight = - std::move(cudf::gather(cudf::table_view({result_weight}), map)->release().front()); + h_expected_weight, cudf::get_default_stream(), temporary_mr); + + auto map = cudf::device_span(d_expected_src); + auto sampled_result_mean = std::move(cudf::gather(cudf::table_view({result_mean}), + map, + cudf::out_of_bounds_policy::DONT_CHECK, + cudf::get_default_stream(), + temporary_mr) + ->release() + .front()); + auto sampled_result_weight = std::move(cudf::gather(cudf::table_view({result_weight}), + map, + cudf::out_of_bounds_policy::DONT_CHECK, + cudf::get_default_stream(), + temporary_mr) + ->release() + .front()); auto expected_mean = cudf::device_span(d_expected_mean); auto expected_weight = cudf::device_span(d_expected_weight); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_mean, *sampled_result_mean); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_weight, *sampled_result_weight); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_mean, + *sampled_result_mean, + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected_weight, *sampled_result_weight, cudf::test::debug_output_level::FIRST_ERROR, mr); } -std::unique_ptr make_expected_tdigest_column(std::vector const& groups) +std::unique_ptr make_expected_tdigest_column(std::vector const& groups, + cudf::memory_resources mr) { + auto const temporary_mr = mr.get_temporary_mr(); std::vector> tdigests; // make an individual digest auto make_digest = [&](expected_tdigest const& tdigest) { std::vector> inner_children; - inner_children.push_back(std::make_unique(tdigest.mean)); - inner_children.push_back(std::make_unique(tdigest.weight)); + inner_children.push_back( + std::make_unique(tdigest.mean, cudf::get_default_stream(), temporary_mr)); + inner_children.push_back( + std::make_unique(tdigest.weight, cudf::get_default_stream(), temporary_mr)); // tdigest struct - auto tdigests = - cudf::make_structs_column(tdigest.mean.size(), std::move(inner_children), 0, {}); + auto tdigests = cudf::make_structs_column(tdigest.mean.size(), + std::move(inner_children), + 0, + {}, + cudf::get_default_stream(), + temporary_mr); - auto offsets = cudf::test::fixed_width_column_wrapper({0, tdigest.mean.size()}); - auto list = cudf::make_lists_column(1, offsets.release(), std::move(tdigests), 0, {}); + auto offsets = + cudf::test::fixed_width_column_wrapper({0, tdigest.mean.size()}, temporary_mr); + auto list = cudf::make_lists_column(1, offsets.release(), std::move(tdigests), 0, {}); - auto min_col = cudf::test::fixed_width_column_wrapper({tdigest.min}); - auto max_col = cudf::test::fixed_width_column_wrapper({tdigest.max}); + auto min_col = cudf::test::fixed_width_column_wrapper({tdigest.min}, temporary_mr); + auto max_col = cudf::test::fixed_width_column_wrapper({tdigest.max}, temporary_mr); std::vector> children; children.push_back(std::move(list)); children.push_back(min_col.release()); children.push_back(max_col.release()); - return make_structs_column(1, std::move(children), 0, {}); + return make_structs_column( + 1, std::move(children), 0, {}, cudf::get_default_stream(), temporary_mr); }; // build the individual digests @@ -99,7 +126,7 @@ std::unique_ptr make_expected_tdigest_column(std::vector const& c) { return c->view(); }); - return cudf::concatenate(views); + return cudf::concatenate(views, cudf::get_default_stream(), mr.get_output_mr()); } } // namespace test