Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion python/cudf_polars/cudf_polars/dsl/ir.py
Original file line number Diff line number Diff line change
Expand Up @@ -2543,7 +2543,12 @@ class Join(IR):
"""A join of two dataframes."""

__slots__ = ("left_on", "options", "right_on")
_non_child = ("schema", "left_on", "right_on", "options")
_non_child: ClassVar[tuple[str, ...]] = (
"schema",
"left_on",
"right_on",
"options",
)
_n_non_child_args = 3
left_on: tuple[expr.NamedExpr, ...]
"""List of expressions used as keys in the left frame."""
Expand Down
9 changes: 9 additions & 0 deletions python/cudf_polars/cudf_polars/dsl/utils/column_domain.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
Slice,
Sort,
)
from cudf_polars.streaming.filter_hint import PushdownFilterHint

if TYPE_CHECKING:
from collections.abc import Mapping
Expand Down Expand Up @@ -141,3 +142,11 @@ def _(
return {
name: ColumnBinding(0, name) for name in node.schema if name in child.schema
}


@column_domain_bindings.register(PushdownFilterHint)
def _(node: PushdownFilterHint) -> Mapping[str, ColumnBinding]:
target = node.children[0]
return {
name: ColumnBinding(0, name) for name in node.schema if name in target.schema
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES.
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""RapidsMPF streaming-engine support."""
Expand All @@ -15,6 +15,7 @@
import cudf_polars.streaming.actor_graph.io
import cudf_polars.streaming.actor_graph.join
import cudf_polars.streaming.actor_graph.over
import cudf_polars.streaming.actor_graph.prefilter_actor
import cudf_polars.streaming.actor_graph.repartition
import cudf_polars.streaming.actor_graph.union # noqa: F401

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

from cudf_polars.dsl.ir import Distinct, GroupBy, Sort
from cudf_polars.dsl.traversal import traversal
from cudf_polars.streaming.filter_hint import PushdownFilterHint
from cudf_polars.streaming.io import StreamingSink
from cudf_polars.streaming.join import Join
from cudf_polars.streaming.over import Over
Expand Down Expand Up @@ -107,6 +108,7 @@ def __init__(
GroupBy,
Distinct,
Over,
PushdownFilterHint,
)

self.collective_nodes: list[IR] = [
Expand Down
8 changes: 5 additions & 3 deletions python/cudf_polars/cudf_polars/streaming/actor_graph/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
generate_ir_sub_network_wrapper,
metadata_drain_node,
)
from cudf_polars.streaming.filter_hint import PushdownFilterHint
from cudf_polars.streaming.io import StreamingScan
from cudf_polars.streaming.over import Over
from cudf_polars.utils.config import SPMDContext
Expand Down Expand Up @@ -178,11 +179,12 @@ def _mark_children_unbounded(node: IR) -> None:
for node in traversal([ir]):
if node in unbounded:
_mark_children_unbounded(node)
elif isinstance(node, (Union, Join, Over)):
elif isinstance(node, (Union, Join, Over, PushdownFilterHint)):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nitpick: the comment below describes why Union, Join, and Over need unbounded fanout.

Maybe it's just a person preference thing, but I'd prefer an IR.needs_unbounaded_fanout property / class variable, and then we can override that for specific IR nodes, and document the reasoning where we set the property.

But this is fine too.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Whether one needs unbounded fanout is a property of the implementation choice, not the IR node per-se. So I prefer it here.

# Union processes children sequentially; Join may broadcast one
# side; Over buffers (or samples-then-replays) its input before
# producing output. In every case the input source needs
# unbounded fanout so other consumers don't block it.
# producing output; PushdownFilterHint similarly might buffer
# then replay. In every case the input source needs unbounded
# fanout so other consumers don't block it.
_mark_children_unbounded(node)
elif len(node.children) > 1:
# Check if this node is doing any broadcasting.
Expand Down
Loading
Loading