Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -796,6 +796,7 @@ add_library(
src/io/parquet/experimental/hybrid_scan_preprocess.cu
src/io/parquet/experimental/page_index_filter.cu
src/io/parquet/experimental/page_index_filter_utils.cu
src/io/parquet/experimental/variant_encode.cu
src/io/parquet/experimental/variant_extract.cu
src/io/parquet/experimental/variant_path.cpp
src/io/parquet/expression_transform_helpers.cpp
Expand Down
34 changes: 34 additions & 0 deletions cpp/include/cudf/io/experimental/variant.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,16 @@

#include <cudf/column/column.hpp>
#include <cudf/column/column_view.hpp>
#include <cudf/strings/strings_column_view.hpp>
#include <cudf/types.hpp>
#include <cudf/utilities/default_stream.hpp>
#include <cudf/utilities/memory_resource.hpp>
#include <cudf/utilities/span.hpp>

#include <rmm/cuda_stream_view.hpp>

#include <memory>
#include <string>
#include <string_view>

/**
Expand Down Expand Up @@ -109,6 +112,37 @@ namespace io::parquet::experimental {
rmm::cuda_stream_view stream = cudf::get_default_stream(),
rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());

/**
* `@brief` Encode a strings column of flat JSON objects as Parquet VARIANT.
*
* Each row of `@p` input must be a non-nested JSON object string (e.g. `{"a":1,"b":"hi"}`).
* The function extracts the scalar fields named in `@p` column_names and encodes the result
* as a Parquet VARIANT column: a `struct<list<uint8> metadata, list<uint8> value>`.
*
* Supported JSON value types per field:
* - `null` → VARIANT null primitive
* - `true` / `false` → VARIANT boolean primitive
* - Integer literals → VARIANT INT64 primitive
* - Floating-point → VARIANT FLOAT64 primitive
* - Quoted strings → VARIANT short-string or long-string
*
* Fields absent from a row (or whose input row is null) are omitted from that row's
* VARIANT object; they do not appear in the value blob.
*
* `@param` input Strings column where each non-null row is a flat JSON object
* `@param` column_names Field names to encode; ordering need not be sorted. At most 255 names
* `@param` stream CUDA stream
* `@param` mr Device memory resource
* `@return` `struct<list<uint8> metadata, list<uint8> value>` VARIANT column
*
* `@throws` std::invalid_argument if `column_names` contains more than 255 names
*/
Comment thread
abigalekim marked this conversation as resolved.
[[nodiscard]] std::unique_ptr<column> encode_strings_to_variant(
cudf::strings_column_view const& input,
cudf::host_span<std::string const> column_names,
rmm::cuda_stream_view stream = cudf::get_default_stream(),
rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());

/** @} */
} // namespace io::parquet::experimental
} // namespace CUDF_EXPORT cudf
Loading
Loading