From a663376d954fc950cc949d5f0a9d385cb262edfb Mon Sep 17 00:00:00 2001 From: Uros Bojanic <221401595+uros-b@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:32:57 +0000 Subject: [PATCH 01/11] [SPARK-58355][SS] Fix grammar in metadata log null-check message ### What changes were proposed in this pull request? Fixes the grammar of the null-metadata `require(...)` message in `HDFSMetadataLog` and `AsyncOffsetSeqLog`: "'null' metadata cannot written to a metadata log" becomes "... cannot be written ...". ### Why are the changes needed? The message was missing the word "be". The sibling `AsyncCommitLog` already uses the correct phrasing, so this brings the two outliers in line. The message is a plain `require()` string, not part of the error-condition framework, so it is not golden-tested. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Message-text-only change; no behavior change. No tests needed. ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 4.8) Closes #57546 from uros-b/error-metadatalog-cannot-be-written. Lead-authored-by: Uros Bojanic <221401595+uros-b@users.noreply.github.com> Co-authored-by: Uros Bojanic Signed-off-by: Uros Bojanic <221401595+uros-b@users.noreply.github.com> --- .../execution/streaming/checkpointing/AsyncOffsetSeqLog.scala | 2 +- .../sql/execution/streaming/checkpointing/HDFSMetadataLog.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/AsyncOffsetSeqLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/AsyncOffsetSeqLog.scala index 14fb4620527c0..c2ae9ac0fae6b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/AsyncOffsetSeqLog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/AsyncOffsetSeqLog.scala @@ -83,7 +83,7 @@ class AsyncOffsetSeqLog( * to indicate some write error. */ def addAsync(batchId: Long, metadata: OffsetSeqBase): CompletableFuture[(Long, Boolean)] = { - require(metadata != null, "'null' metadata cannot written to a metadata log") + require(metadata != null, "'null' metadata cannot be written to a metadata log") def issueAsyncWrite(batchId: Long): CompletableFuture[Long] = { lastCommitIssuedTimestampMs.set(clock.getTimeMillis()) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala index b06a90de44c48..3f2271ac2b2c2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala @@ -133,7 +133,7 @@ class HDFSMetadataLog[T <: AnyRef : ClassTag]( * metadata has already been stored, this method will return `false`. */ override def add(batchId: Long, metadata: T): Boolean = { - require(metadata != null, "'null' metadata cannot written to a metadata log") + require(metadata != null, "'null' metadata cannot be written to a metadata log") val res = addNewBatchByStream(batchId) { output => serialize(metadata, output) } if (metadataCacheEnabled && res) batchCache.put(batchId, metadata) res From effd582afa3980a6b674d212a705c0d5212f2948 Mon Sep 17 00:00:00 2001 From: YangJie Date: Mon, 27 Jul 2026 23:22:32 +0800 Subject: [PATCH 02/11] [SPARK-57635][SQL] Make Declarative Pipelines run-termination reason deterministic ### What changes were proposed in this pull request? `TriggeredGraphExecution.getRunTerminationReason` decided which failed flow's reason to report by calling `collectFirst` over `failureTracker`, a `ConcurrentHashMap` whose iteration order is unspecified. When more than one flow exhausts its retries (a non-retryable `StopFlowExecution`), the flow whose reason gets surfaced therefore varied from run to run. This extracts a small pure helper, `chooseRunTerminationReason`, that considers only the stopped flows and picks the earliest one by `(lastFailTimestamp, flowName)`, so the reported reason is stable across otherwise-identical runs. `getRunTerminationReason` now calls it and falls back to `UnexpectedRunFailure()`. The previous code also computed `graphForExecution.flow(...)` and `lastException` only to discard them; those are dropped. ### Why are the changes needed? Two runs that fail the same way could report different termination reasons (different flow name and cause), which is confusing in logs and events and makes the outcome non-reproducible. ### Does this PR introduce _any_ user-facing change? No. The reported reason was already one of the failing flows; it is now chosen deterministically. ### How was this patch tested? Added unit tests for `chooseRunTerminationReason` in `TriggeredGraphExecutionSuite`: the earliest failure wins regardless of iteration order, ties are broken by flow name, and retryable failures are ignored. They fail against the previous order-dependent selection and pass with this change. ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Code (Claude Opus 4.8) Closes #56698 from LuciferYang/sdp-deterministic-termination-reason. Authored-by: YangJie Signed-off-by: yangjie01 --- .../graph/TriggeredGraphExecution.scala | 35 +++++++------ .../graph/TriggeredGraphExecutionSuite.scala | 49 +++++++++++++++++++ 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala index 02871e64aa761..22b64798f598f 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala @@ -418,21 +418,9 @@ class TriggeredGraphExecution( return RunCompletion() } - val executionFailureOpt = failureTracker.iterator - .map { - case (flowIdentifier, failureInfo) => - ( - graphForExecution.flow(flowIdentifier), - failureInfo.lastException, - failureInfo.lastExceptionAction - ) - } - .collectFirst { - case (_, _, GraphExecution.StopFlowExecution(reason)) => - reason.runTerminationReason - } - - executionFailureOpt.getOrElse(UnexpectedRunFailure()) + TriggeredGraphExecution + .chooseRunTerminationReason(failureTracker.iterator) + .getOrElse(UnexpectedRunFailure()) } } @@ -450,6 +438,23 @@ case class TriggeredFailureInfo( object TriggeredGraphExecution { + /** + * Picks the run-termination reason from the flows whose execution was stopped because they + * exhausted their retries. Several flows can stop a run and `failures` comes from an unordered + * map, so the earliest failure is chosen - ties broken by flow name - to keep the reported + * reason stable across otherwise-identical runs. + */ + private[graph] def chooseRunTerminationReason( + failures: Iterator[(TableIdentifier, TriggeredFailureInfo)]): Option[RunTerminationReason] = { + failures + .collect { + case (id, TriggeredFailureInfo(ts, _, _, GraphExecution.StopFlowExecution(r))) => + (ts, id.unquotedString, r.runTerminationReason) + } + .minByOption { case (ts, flowName, _) => (ts, flowName) } + .map { case (_, _, reason) => reason } + } + // All possible states of a data stream for a flow sealed trait StreamState object StreamState { diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala index 57baf4c2d5b11..db8c368ca89df 100644 --- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala +++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala @@ -1060,4 +1060,53 @@ class TriggeredGraphExecutionSuite extends ExecutionTest with SharedSparkSession assert(warnCount == 2 && errorCount == 1) } + + /** A non-retryable (retries exhausted) failure, i.e. one that stops the run. */ + private def stopFailure(ts: Long, flowName: String, cause: Throwable): TriggeredFailureInfo = { + // currentNumTries > maxAllowedRetries yields a StopFlowExecution. + val action = GraphExecution.determineFlowExecutionActionFromError( + ex = cause, + flowDisplayName = flowName, + currentNumTries = 2, + maxAllowedRetries = 1) + TriggeredFailureInfo(ts, numFailures = 2, lastException = cause, lastExceptionAction = action) + } + + test("chooseRunTerminationReason surfaces the earliest non-retryable failure deterministically") { + val earliestCause = new RuntimeException("earliest") + // A retryable failure with an earlier timestamp must be ignored: it does not stop the run. + val retryable = { + val cause = new RuntimeException("retryable") + val action = GraphExecution.determineFlowExecutionActionFromError( + ex = cause, flowDisplayName = "flow_retry", currentNumTries = 1, maxAllowedRetries = 3) + TableIdentifier("flow_retry") -> TriggeredFailureInfo( + 50, numFailures = 1, lastException = cause, lastExceptionAction = action) + } + val entries = Seq( + TableIdentifier("flow_c") -> stopFailure(300, "flow_c", new RuntimeException("c")), + TableIdentifier("flow_b") -> stopFailure(100, "flow_b", earliestCause), + TableIdentifier("flow_a") -> stopFailure(200, "flow_a", new RuntimeException("a")), + retryable) + + // The earliest non-retryable failure (flow_b @ 100) wins regardless of iteration order. + Seq(entries, entries.reverse).foreach { ordered => + assert( + TriggeredGraphExecution.chooseRunTerminationReason(ordered.iterator) + .contains(QueryExecutionFailure("flow_b", 1, Some(earliestCause)))) + } + } + + test("chooseRunTerminationReason breaks ties between equal timestamps by flow name") { + val aCause = new RuntimeException("a") + val entries = Seq( + TableIdentifier("flow_z") -> stopFailure(100, "flow_z", new RuntimeException("z")), + TableIdentifier("flow_a") -> stopFailure(100, "flow_a", aCause)) + assert( + TriggeredGraphExecution.chooseRunTerminationReason(entries.iterator) + .contains(QueryExecutionFailure("flow_a", 1, Some(aCause)))) + } + + test("chooseRunTerminationReason has no reason when no flow stopped the run") { + assert(TriggeredGraphExecution.chooseRunTerminationReason(Iterator.empty).isEmpty) + } } From 0747e287579272958600e3db3ccf03044378576f Mon Sep 17 00:00:00 2001 From: YangJie Date: Mon, 27 Jul 2026 23:31:18 +0800 Subject: [PATCH 03/11] [SPARK-57638][SQL] Avoid busy-waiting in Declarative Pipelines flow resolution ### What changes were proposed in this pull request? `DataflowGraphTransformer.transformDownNodes` resolves flows on a bounded thread pool and drives them from a `while` loop that, each pass, partitioned the in-flight futures with the non-blocking `future.isDone`, reaped the completed ones, and scheduled a new flow if a slot was free. When all slots were in flight (or the queue was drained and only the last futures remained) and none had completed, the pass reaped nothing and scheduled nothing, then looped again immediately - busy-spinning on `isDone` and pinning a core for the duration of resolution. This drives the loop with an `ExecutorCompletionService` instead: completed tasks are drained with the non-blocking `poll()`, and when nothing can be scheduled but tasks are still running, the loop blocks on `take()` until the next one finishes rather than spinning. Behavior is otherwise unchanged - the same flows are scheduled in the same order, exceptions are still propagated via `Future.get()`, and an `outstanding` counter replaces the `ArrayBuffer[Future]` for slot bookkeeping. ### Why are the changes needed? Resolving a graph with more flows than the parallelism (10) kept one CPU core busy at 100% doing no useful work for the whole resolution, which is wasteful and shows up as unexplained driver CPU. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Two new cases in `ConnectValidPipelineSuite` cover the regime this PR changes - more flows than `parallelism` (10), so the slots fill and the loop reaches the blocking `take()` branch that replaces the busy-wait. The small graphs in the existing suites never get there. - `resolution terminates and resolves all flows when flow count exceeds parallelism` - 25 independent flows. - `resolution re-queues retryable flows under load when consumers exceed parallelism` - 20 consumers registered before their source `src`, so the first batch throws `TransformNodeRetryableException`, parks as dependents of `src`, and is re-queued once `src` resolves; this exercises the retryable re-queue path together with the blocking branch. Both assert only the outcome (every flow resolves and the call returns), so they are deterministic and have no timing dependence - a regression that deadlocked would hang until the suite times out. Asserting the absence of a busy-wait directly is not included, since that requires CPU-time or timing measurements that are flaky in CI. Existing graph-resolution suites (`ConnectValidPipelineSuite`, `ConnectInvalidPipelineSuite`, `SqlPipelineSuite`, `TriggeredGraphExecutionSuite`, `MaterializeTablesSuite`) still pass; the change only affects how the loop waits, not what it resolves. ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Code (Claude Opus 4.8) Closes #56700 from LuciferYang/sdp-resolution-busy-wait. Authored-by: YangJie Signed-off-by: yangjie01 --- .../graph/DataflowGraphTransformer.scala | 263 +++++++++--------- .../graph/ConnectValidPipelineSuite.scala | 58 ++++ 2 files changed, 194 insertions(+), 127 deletions(-) diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DataflowGraphTransformer.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DataflowGraphTransformer.scala index 2523c0ae5502a..f52d5572054d2 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DataflowGraphTransformer.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DataflowGraphTransformer.scala @@ -22,10 +22,10 @@ import java.util.concurrent.{ ConcurrentLinkedDeque, ConcurrentLinkedQueue, ExecutionException, + ExecutorCompletionService, Future } -import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ import scala.util.control.NoStackTrace @@ -134,159 +134,168 @@ class DataflowGraphTransformer(graph: DataflowGraph) extends AutoCloseable { val failedFlowsQueue = new ConcurrentLinkedQueue[ResolutionFailedFlow]() val failedDependentFlows = new ConcurrentHashMap[TableIdentifier, Seq[ResolutionFailedFlow]]() - var futures = ArrayBuffer[Future[Unit]]() + val completionService = new ExecutorCompletionService[Unit](executor) + var outstanding = 0 val toBeResolvedFlows = new ConcurrentLinkedDeque[Flow]() toBeResolvedFlows.addAll(flows.asJava) - while (futures.nonEmpty || toBeResolvedFlows.peekFirst() != null) { - val (done, notDone) = futures.partition(_.isDone) - // Explicitly call future.get() to propagate exceptions one by one if any + // Waits on a finished resolution task and propagates its exception, if any. + def reap(finished: Future[Unit]): Unit = { try { - done.foreach(_.get()) + finished.get() } catch { case exn: ExecutionException => // Computation threw the exception that is the cause of exn throw exn.getCause } - futures = notDone - val flowOpt = { - // We only schedule [[batchSize]] number of flows in parallel. - if (futures.size < batchSize) { - Option(toBeResolvedFlows.pollFirst()) - } else { - None - } + outstanding -= 1 + } + + while (outstanding > 0 || toBeResolvedFlows.peekFirst() != null) { + // Reap every resolution task that has already finished, without blocking. + var finished = completionService.poll() + while (finished != null) { + reap(finished) + finished = completionService.poll() } - flowOpt.foreach { flow => - futures.append( - executor.submit( - () => + // We only schedule [[batchSize]] number of flows in parallel. + if (outstanding < batchSize && toBeResolvedFlows.peekFirst() != null) { + val flow = toBeResolvedFlows.pollFirst() + outstanding += 1 + completionService.submit( + () => + try { try { - try { - // Note: Flow don't need their inputs passed, so for now we send empty Seq. - val result = transformer(flow, Seq.empty) - require( - result.forall(_.isInstanceOf[ResolvedFlow]), - "transformer must return a Seq[Flow]" - ) + // Note: Flow don't need their inputs passed, so for now we send empty Seq. + val result = transformer(flow, Seq.empty) + require( + result.forall(_.isInstanceOf[ResolvedFlow]), + "transformer must return a Seq[Flow]" + ) - val transformedFlows = result.map(_.asInstanceOf[ResolvedFlow]) - resolvedFlowsMap.put(flow.identifier, transformedFlows) - resolvedFlows.addAll(transformedFlows.asJava) - } catch { - case e: TransformNodeRetryableException => - val datasetIdentifier = e.datasetIdentifier - failedDependentFlows.compute( - datasetIdentifier, - (_, flows) => { - // Don't add the input flow back but the failed flow object - // back which has relevant failure information. - val failedFlow = e.failedNode - if (flows == null) { - Seq(failedFlow) - } else { - flows :+ failedFlow - } + val transformedFlows = result.map(_.asInstanceOf[ResolvedFlow]) + resolvedFlowsMap.put(flow.identifier, transformedFlows) + resolvedFlows.addAll(transformedFlows.asJava) + } catch { + case e: TransformNodeRetryableException => + val datasetIdentifier = e.datasetIdentifier + failedDependentFlows.compute( + datasetIdentifier, + (_, flows) => { + // Don't add the input flow back but the failed flow object + // back which has relevant failure information. + val failedFlow = e.failedNode + if (flows == null) { + Seq(failedFlow) + } else { + flows :+ failedFlow } - ) - // Between the time the flow started and finished resolving, perhaps the - // dependent dataset was resolved - resolvedFlowDestinationsMap.computeIfPresent( - datasetIdentifier, - (_, resolved) => { - if (resolved) { - // Check if the dataset that the flow is dependent on has been resolved - // and if so, remove all dependent flows from the failedDependentFlows and - // add them to the toBeResolvedFlows queue for retry. - failedDependentFlows.computeIfPresent( - datasetIdentifier, - (_, toRetryFlows) => { - toRetryFlows.foreach(toBeResolvedFlows.addFirst(_)) - null - } - ) - } - resolved + } + ) + // Between the time the flow started and finished resolving, perhaps the + // dependent dataset was resolved + resolvedFlowDestinationsMap.computeIfPresent( + datasetIdentifier, + (_, resolved) => { + if (resolved) { + // Check if the dataset that the flow is dependent on has been resolved + // and if so, remove all dependent flows from the failedDependentFlows and + // add them to the toBeResolvedFlows queue for retry. + failedDependentFlows.computeIfPresent( + datasetIdentifier, + (_, toRetryFlows) => { + toRetryFlows.foreach(toBeResolvedFlows.addFirst(_)) + null + } + ) } + resolved + } + ) + case other: Throwable => throw other + } + // If all flows to this particular destination are resolved, move to the destination + // node transformer + if (flowsTo(flow.destinationIdentifier).forall({ flowToDestination => + resolvedFlowsMap.containsKey(flowToDestination.identifier) + })) { + // If multiple flows completed in parallel, ensure we resolve the destination only + // once by electing a leader via computeIfAbsent + var isCurrentThreadLeader = false + resolvedFlowDestinationsMap.computeIfAbsent(flow.destinationIdentifier, _ => { + isCurrentThreadLeader = true + // Set initial value as false as flow destination is not resolved yet. + false + }) + if (isCurrentThreadLeader) { + if (tableMap.contains(flow.destinationIdentifier)) { + val transformed = + transformer( + tableMap(flow.destinationIdentifier), + flowsTo(flow.destinationIdentifier) + ) + resolvedTables.addAll( + transformed.collect { case t: Table => t }.asJava ) - case other: Throwable => throw other - } - // If all flows to this particular destination are resolved, move to the destination - // node transformer - if (flowsTo(flow.destinationIdentifier).forall({ flowToDestination => - resolvedFlowsMap.containsKey(flowToDestination.identifier) - })) { - // If multiple flows completed in parallel, ensure we resolve the destination only - // once by electing a leader via computeIfAbsent - var isCurrentThreadLeader = false - resolvedFlowDestinationsMap.computeIfAbsent(flow.destinationIdentifier, _ => { - isCurrentThreadLeader = true - // Set initial value as false as flow destination is not resolved yet. - false - }) - if (isCurrentThreadLeader) { - if (tableMap.contains(flow.destinationIdentifier)) { + resolvedFlows.addAll( + transformed.collect { case f: ResolvedFlow => f }.asJava + ) + } else if (viewMap.contains(flow.destinationIdentifier)) { + resolvedViews.addAll { val transformed = transformer( - tableMap(flow.destinationIdentifier), + viewMap(flow.destinationIdentifier), flowsTo(flow.destinationIdentifier) ) - resolvedTables.addAll( - transformed.collect { case t: Table => t }.asJava - ) - resolvedFlows.addAll( - transformed.collect { case f: ResolvedFlow => f }.asJava - ) - } else if (viewMap.contains(flow.destinationIdentifier)) { - resolvedViews.addAll { - val transformed = - transformer( - viewMap(flow.destinationIdentifier), - flowsTo(flow.destinationIdentifier) - ) - transformed.map(_.asInstanceOf[View]).asJava - } - } else if (sinkMap.contains(flow.destinationIdentifier)) { - resolvedSinks.addAll { - val transformed = - transformer( - sinkMap(flow.destinationIdentifier), flowsTo(flow.destinationIdentifier) - ) - require( - transformed.forall(_.isInstanceOf[Sink]), - "transformer must return a Seq[Sink]" + transformed.map(_.asInstanceOf[View]).asJava + } + } else if (sinkMap.contains(flow.destinationIdentifier)) { + resolvedSinks.addAll { + val transformed = + transformer( + sinkMap(flow.destinationIdentifier), flowsTo(flow.destinationIdentifier) ) - transformed.map(_.asInstanceOf[Sink]).asJava - } - } else { - throw new IllegalArgumentException( - s"Unsupported destination ${flow.destinationIdentifier.unquotedString}" + - s" in flow: ${flow.displayName} at transformDownNodes" + require( + transformed.forall(_.isInstanceOf[Sink]), + "transformer must return a Seq[Sink]" ) + transformed.map(_.asInstanceOf[Sink]).asJava } - // Set flow destination as resolved now. - resolvedFlowDestinationsMap.computeIfPresent( - flow.destinationIdentifier, - (_, _) => { - // If there are any other node failures dependent on this destination, retry - // them - failedDependentFlows.computeIfPresent( - flow.destinationIdentifier, - (_, toRetryFlows) => { - toRetryFlows.foreach(toBeResolvedFlows.addFirst(_)) - null - } - ) - true - } + } else { + throw new IllegalArgumentException( + s"Unsupported destination ${flow.destinationIdentifier.unquotedString}" + + s" in flow: ${flow.displayName} at transformDownNodes" ) } + // Set flow destination as resolved now. + resolvedFlowDestinationsMap.computeIfPresent( + flow.destinationIdentifier, + (_, _) => { + // If there are any other node failures dependent on this destination, retry + // them + failedDependentFlows.computeIfPresent( + flow.destinationIdentifier, + (_, toRetryFlows) => { + toRetryFlows.foreach(toBeResolvedFlows.addFirst(_)) + null + } + ) + true + } + ) } - } catch { - case ex: TransformNodeFailedException => failedFlowsQueue.add(ex.failedNode) } - ) + } catch { + case ex: TransformNodeFailedException => failedFlowsQueue.add(ex.failedNode) + } ) + } else if (outstanding > 0) { + // Nothing could be scheduled (slots full, or the queue is drained) but tasks are still + // running: block until the next finishes instead of busy-spinning on Future.isDone. The + // outstanding > 0 guard is required, not redundant: the poll() drain above can take + // outstanding to 0 with an empty queue, and then there is nothing to wait for - the loop + // should just exit rather than block forever in take(). + reap(completionService.take()) } } diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/ConnectValidPipelineSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/ConnectValidPipelineSuite.scala index 014e9e1d5a60d..ea6d3202ba868 100644 --- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/ConnectValidPipelineSuite.scala +++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/ConnectValidPipelineSuite.scala @@ -544,6 +544,64 @@ class ConnectValidPipelineSuite extends PipelineTest with SharedSparkSession { ) } + test("resolution terminates and resolves all flows when flow count exceeds parallelism") { + val session = spark + import session.implicits._ + + // DataflowGraphTransformer caps in-flight resolutions at `parallelism` (10). With more + // independent flows than that, the slots fill and the scheduler blocks on a finished task + // (the `take()` branch) once `parallelism` tasks are outstanding - the path the busy-wait + // rewrite changes. Small graphs in the other suites never reach this regime. This asserts the + // outcome only (all flows resolve and the call returns), so it is deterministic and has no + // timing dependence; a regression that deadlocked would hang here until the suite times out. + val numFlows = 25 + class P extends TestGraphRegistrationContext(spark) { + (0 until numFlows).foreach { i => + registerPersistedView(s"v$i", query = dfFlowFunc(Seq(i).toDF("x"))) + } + } + val p = new P().resolveToDataflowGraph() + + assert(p.resolved, "all flows should resolve when their count exceeds parallelism") + (0 until numFlows).foreach { i => + assert( + p.resolvedFlow.contains(fullyQualifiedIdentifier(s"v$i")), + s"flow v$i was not resolved") + } + } + + test("resolution re-queues retryable flows under load when consumers exceed parallelism") { + val session = spark + import session.implicits._ + + // A wide fan-out: many consumers reading from one source view, with the consumer count above + // `parallelism` (10), so slots fill and the loop blocks on take(). The consumers are + // registered (and therefore scheduled) before `src`, so the first batch resolves consumers + // whose `src` input is not yet available: each throws TransformNodeRetryableException and is + // parked as a dependent of `src`; once `src` resolves they are re-queued onto the deque and + // retried, re-driving the loop until every consumer resolves. This deterministically exercises + // the retryable re-queue path together with the blocking branch. Asserts only that everything + // resolves and the call returns - no timing assertions. + val numConsumers = 20 + class P extends TestGraphRegistrationContext(spark) { + (0 until numConsumers).foreach { i => + registerPersistedView(s"c$i", query = sqlFlowFunc(spark, "SELECT x FROM src")) + } + registerPersistedView("src", query = dfFlowFunc(Seq(1, 2, 3).toDF("x"))) + } + val p = new P().resolveToDataflowGraph() + + assert(p.resolved, "source and all consumers should resolve under load") + assert( + p.resolvedFlow.contains(fullyQualifiedIdentifier("src")), + "source flow was not resolved") + (0 until numConsumers).foreach { i => + assert( + p.resolvedFlow.contains(fullyQualifiedIdentifier(s"c$i")), + s"consumer flow c$i was not resolved") + } + } + /** Verifies the [[DataflowGraph]] has the specified [[Flow]] with the specified schema. */ private def verifyFlowSchema( pipeline: DataflowGraph, From 54d5f5125972fc0f358a9716c30ab8e82bd53b40 Mon Sep 17 00:00:00 2001 From: YangJie Date: Mon, 27 Jul 2026 23:36:57 +0800 Subject: [PATCH 04/11] [SPARK-58363][SQL] Assign a name to the error condition _LEGACY_ERROR_TEMP_2214-2219 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What changes were proposed in this pull request? The catalog-plugin loading failures in `Catalogs.load` were reported with six placeholder error conditions `_LEGACY_ERROR_TEMP_2214` through `_LEGACY_ERROR_TEMP_2219`, none of which carried a SQLSTATE. Group the six into a single umbrella error condition `CANNOT_LOAD_CATALOG` (SQLSTATE `46103`), with one subclass per failure mode in the `Catalogs.load` try/catch: - `NOT_A_CATALOG_PLUGIN` — the class does not implement `CatalogPlugin` - `PLUGIN_CLASS_NOT_FOUND` — `ClassNotFoundException` - `CONSTRUCTOR_NOT_FOUND` — `NoSuchMethodException` - `CONSTRUCTOR_NOT_ACCESSIBLE` — `IllegalAccessException` - `ABSTRACT_CLASS` — `InstantiationException` - `CONSTRUCTOR_FAILURE` — `InvocationTargetException` This also drops a stray trailing `)` in the legacy `_2216`/`_2217` messages. ### Why are the changes needed? The error-conditions README disallows new `_LEGACY_ERROR_TEMP_*` entries and asks existing ones to be resolved. This resolves six of them. SQLSTATE `46103` ("Java Error") is consistent with the sibling `CANNOT_LOAD_FUNCTION_CLASS`; the failures are triggered by a user-supplied `spark.sql.catalog.` plugin class, so they are user-actionable rather than system errors. ### Does this PR introduce _any_ user-facing change? No. The `_LEGACY_ERROR_TEMP_*` names are not part of the public API. Message text is preserved (except the stray `)` typo removal), now rendered under the umbrella prefix. ### How was this patch tested? Updated the existing `checkError`/condition assertions in `CatalogLoadingSuite` and `SupportsCatalogOptionsSuite`, and added tests for the previously-uncovered `CONSTRUCTOR_NOT_FOUND` and `ABSTRACT_CLASS` subclasses. `build/sbt "catalyst/testOnly *CatalogLoadingSuite" "core/testOnly org.apache.spark.SparkThrowableSuite"` passes. ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 4.8) Closes #57556 from LuciferYang/assign-name-legacy-2214-2219. Authored-by: YangJie Signed-off-by: yangjie01 --- .../resources/error/error-conditions.json | 68 +++++++++++-------- .../sql/errors/QueryExecutionErrors.scala | 12 ++-- .../catalog/CatalogLoadingSuite.java | 63 +++++++++++++++-- .../SupportsCatalogOptionsSuite.scala | 2 +- 4 files changed, 102 insertions(+), 43 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index eca32c03d3e0f..e4e91cb28760e 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -557,6 +557,44 @@ ], "sqlState" : "0A000" }, + "CANNOT_LOAD_CATALOG" : { + "message" : [ + "Cannot load catalog '' with the plugin class '':" + ], + "subClass" : { + "ABSTRACT_CLASS" : { + "message" : [ + "the class is abstract and cannot be instantiated." + ] + }, + "CONSTRUCTOR_FAILURE" : { + "message" : [ + "the constructor threw an exception during instantiation." + ] + }, + "CONSTRUCTOR_NOT_ACCESSIBLE" : { + "message" : [ + "failed to call the public no-arg constructor." + ] + }, + "CONSTRUCTOR_NOT_FOUND" : { + "message" : [ + "failed to find the public no-arg constructor." + ] + }, + "NOT_A_CATALOG_PLUGIN" : { + "message" : [ + "the class does not implement CatalogPlugin." + ] + }, + "PLUGIN_CLASS_NOT_FOUND" : { + "message" : [ + "cannot find the plugin class." + ] + } + }, + "sqlState" : "46103" + }, "CANNOT_LOAD_CHECKPOINT_FILE_MANAGER" : { "message" : [ "Error loading streaming checkpoint file manager for path=." @@ -10957,36 +10995,6 @@ "Invalid catalog name: ." ] }, - "_LEGACY_ERROR_TEMP_2214" : { - "message" : [ - "Plugin class for catalog '' does not implement CatalogPlugin: ." - ] - }, - "_LEGACY_ERROR_TEMP_2215" : { - "message" : [ - "Cannot find catalog plugin class for catalog '': ." - ] - }, - "_LEGACY_ERROR_TEMP_2216" : { - "message" : [ - "Failed to find public no-arg constructor for catalog '': )." - ] - }, - "_LEGACY_ERROR_TEMP_2217" : { - "message" : [ - "Failed to call public no-arg constructor for catalog '': )." - ] - }, - "_LEGACY_ERROR_TEMP_2218" : { - "message" : [ - "Cannot instantiate abstract catalog plugin class for catalog '': ." - ] - }, - "_LEGACY_ERROR_TEMP_2219" : { - "message" : [ - "Failed during instantiating constructor for catalog '': ." - ] - }, "_LEGACY_ERROR_TEMP_2220" : { "message" : [ "" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index a5d64ddd314ed..a9ea3f9f1c26c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -1920,7 +1920,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE def catalogPluginClassNotImplementedError(name: String, pluginClassName: String): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2214", + errorClass = "CANNOT_LOAD_CATALOG.NOT_A_CATALOG_PLUGIN", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), @@ -1932,7 +1932,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE pluginClassName: String, e: Exception): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2215", + errorClass = "CANNOT_LOAD_CATALOG.PLUGIN_CLASS_NOT_FOUND", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), @@ -1944,7 +1944,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE pluginClassName: String, e: Exception): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2216", + errorClass = "CANNOT_LOAD_CATALOG.CONSTRUCTOR_NOT_FOUND", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), @@ -1956,7 +1956,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE pluginClassName: String, e: Exception): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2217", + errorClass = "CANNOT_LOAD_CATALOG.CONSTRUCTOR_NOT_ACCESSIBLE", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), @@ -1968,7 +1968,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE pluginClassName: String, e: Exception): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2218", + errorClass = "CANNOT_LOAD_CATALOG.ABSTRACT_CLASS", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), @@ -1980,7 +1980,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE pluginClassName: String, e: Exception): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2219", + errorClass = "CANNOT_LOAD_CATALOG.CONSTRUCTOR_FAILURE", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), diff --git a/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java b/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java index c7e8d7b0f7f30..655cb55852a7a 100644 --- a/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java +++ b/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java @@ -97,7 +97,7 @@ public void testLoadMissingClass() { SparkException exc = Assertions.assertThrows(SparkException.class, () -> Catalogs.load("missing", conf)); - Assertions.assertTrue(exc.getMessage().contains("Cannot find catalog plugin class"), + Assertions.assertEquals("CANNOT_LOAD_CATALOG.PLUGIN_CLASS_NOT_FOUND", exc.getCondition(), "Should complain that the class is not found"); Assertions.assertTrue(exc.getMessage().contains("missing"), "Should identify the catalog by name"); @@ -127,7 +127,7 @@ public void testLoadNonCatalogPlugin() { SparkException exc = Assertions.assertThrows(SparkException.class, () -> Catalogs.load("invalid", conf)); - Assertions.assertTrue(exc.getMessage().contains("does not implement CatalogPlugin"), + Assertions.assertEquals("CANNOT_LOAD_CATALOG.NOT_A_CATALOG_PLUGIN", exc.getCondition(), "Should complain that class does not implement CatalogPlugin"); Assertions.assertTrue(exc.getMessage().contains("invalid"), "Should identify the catalog by name"); @@ -144,8 +144,7 @@ public void testLoadConstructorFailureCatalogPlugin() { SparkException exc = Assertions.assertThrows(SparkException.class, () -> Catalogs.load("invalid", conf)); - Assertions.assertTrue( - exc.getMessage().contains("Failed during instantiating constructor for catalog"), + Assertions.assertEquals("CANNOT_LOAD_CATALOG.CONSTRUCTOR_FAILURE", exc.getCondition(), "Should identify the constructor error"); Assertions.assertTrue(exc.getCause().getMessage().contains("Expected failure"), "Should have expected error message"); @@ -160,14 +159,47 @@ public void testLoadAccessErrorCatalogPlugin() { SparkException exc = Assertions.assertThrows(SparkException.class, () -> Catalogs.load("invalid", conf)); - Assertions.assertTrue( - exc.getMessage().contains("Failed to call public no-arg constructor for catalog"), + Assertions.assertEquals("CANNOT_LOAD_CATALOG.CONSTRUCTOR_NOT_ACCESSIBLE", exc.getCondition(), "Should complain that no public constructor is provided"); Assertions.assertTrue(exc.getMessage().contains("invalid"), "Should identify the catalog by name"); Assertions.assertTrue(exc.getMessage().contains(invalidClassName), "Should identify the class"); } + + @Test + public void testLoadNoNoArgConstructorCatalogPlugin() { + SQLConf conf = new SQLConf(); + String invalidClassName = NoNoArgConstructorCatalogPlugin.class.getCanonicalName(); + conf.setConfString("spark.sql.catalog.invalid", invalidClassName); + + SparkException exc = Assertions.assertThrows(SparkException.class, + () -> Catalogs.load("invalid", conf)); + + Assertions.assertEquals("CANNOT_LOAD_CATALOG.CONSTRUCTOR_NOT_FOUND", exc.getCondition(), + "Should complain that no public no-arg constructor is found"); + Assertions.assertTrue(exc.getMessage().contains("invalid"), + "Should identify the catalog by name"); + Assertions.assertTrue(exc.getMessage().contains(invalidClassName), + "Should identify the class"); + } + + @Test + public void testLoadAbstractCatalogPlugin() { + SQLConf conf = new SQLConf(); + String invalidClassName = AbstractCatalogPlugin.class.getCanonicalName(); + conf.setConfString("spark.sql.catalog.invalid", invalidClassName); + + SparkException exc = Assertions.assertThrows(SparkException.class, + () -> Catalogs.load("invalid", conf)); + + Assertions.assertEquals("CANNOT_LOAD_CATALOG.ABSTRACT_CLASS", exc.getCondition(), + "Should complain that the class is abstract"); + Assertions.assertTrue(exc.getMessage().contains("invalid"), + "Should identify the catalog by name"); + Assertions.assertTrue(exc.getMessage().contains(invalidClassName), + "Should identify the class"); + } } class TestCatalogPlugin implements CatalogPlugin { @@ -218,6 +250,25 @@ public String name() { } } +class NoNoArgConstructorCatalogPlugin implements CatalogPlugin { // no public no-arg constructor + NoNoArgConstructorCatalogPlugin(String arg) { + } + + @Override + public void initialize(String name, CaseInsensitiveStringMap options) { + } + + @Override + public String name() { + return null; + } +} + +abstract class AbstractCatalogPlugin implements CatalogPlugin { // abstract, cannot be instantiated + AbstractCatalogPlugin() { + } +} + class InvalidCatalogPlugin { // doesn't implement CatalogPlugin public void initialize(CaseInsensitiveStringMap options) { } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala index 98904e6976074..bcd8ba185d1dd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala @@ -285,7 +285,7 @@ class SupportsCatalogOptionsSuite extends SharedSparkSession with BeforeAndAfter sql(s"create table t1 (id bigint) using $format") } - assert(e.getMessage.contains("Cannot find catalog plugin class")) + assert(e.getMessage.contains("cannot find the plugin class")) assert(e.getMessage.contains("InvalidCatalogClass")) } finally { spark.sessionState.catalogManager.reset() From 014c53b454c416d93993497b20a35ce4602068ee Mon Sep 17 00:00:00 2001 From: YangJie Date: Mon, 27 Jul 2026 23:40:33 +0800 Subject: [PATCH 05/11] [SPARK-58371][DOC] Update `json` gem version to 2.21.1 ### What changes were proposed in this pull request? This PR upgrades the `json` gem from 2.12.2 to 2.21.1 in `docs/Gemfile.lock`. It is a transitive dependency (pulled in by `jekyll`, which requires `json (~> 2.6)`), and 2.21.1 satisfies that constraint, so only the locked spec version changes. `json` has no runtime dependencies, so no other lock entries change and `docs/Gemfile` does not need to be touched. ### Why are the changes needed? 2.21.1 includes the fix for a security advisory that affects all versions in `>= 2.9.0, < 2.19.9`: - [GHSA-x2f5-4prf-w687](https://github.com/advisories/GHSA-x2f5-4prf-w687) / CVE-2026-54696 (low): heap out-of-bounds write in the JSON generator when streaming to an IO. On the IO path, `fbuffer_do_inc_capa()` compared the requested size against the buffer's total capacity instead of its remaining capacity, so `JSON.dump(obj, io)` or `JSON::State#generate(obj, io)` could write past the buffer when serializing a string near 16 KB. Fixed in 2.19.9. Following the same pattern as SPARK-57633 (`concurrent-ruby` 1.3.7), this picks up the latest release rather than the minimum patched version. ### Does this PR introduce _any_ user-facing change? No. This only affects the documentation build toolchain. ### How was this patch tested? Manually verified in a scratch copy of `docs/Gemfile` and `docs/Gemfile.lock`: 1. Lock resolution matches. Running `bundle lock --update=json` produces a lock file byte-identical to the hand-edited one, confirming 2.21.1 satisfies jekyll's `json (~> 2.6)` and that no other locked spec is affected. 2. Frozen install succeeds. `BUNDLE_FROZEN=true bundle install` installs all 36 gems without modifying the lock file, and `bundle list` reports `json (2.21.1)`. 3. The advisory's PoC no longer crashes: ```ruby require "json" require "stringio" io = StringIO.new big = "a" * 16385 big[16382] = '"' # escapable byte near the buffer boundary JSON.dump([big], io) ``` ```text json version: 2.21.1 PoC (JSON.dump with IO): no crash, output bytes = 16390 roundtrip ok: true ``` 4. The docs site builds: ``` $ cd docs && SKIP_API=1 bundle exec jekyll build Configuration file: .../docs/_config.yml ************************ * Building error docs. * ************************ Generated: docs/_generated/error-conditions.html Source: .../docs Destination: .../docs/_site Incremental build: disabled. Enable with --incremental Generating... Warning: Tolerating missing API files because the following skip flags are set: SKIP_API done in 3.125 seconds. Auto-regeneration: disabled. Use --watch to enable. ``` ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 5) Closes #57565 from LuciferYang/docs-json-gem-2.21.1. Authored-by: YangJie Signed-off-by: yangjie01 --- docs/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 25a847198bcfc..30851e513a8cb 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -45,7 +45,7 @@ GEM sass-embedded (~> 1.75) jekyll-watch (2.2.1) listen (~> 3.0) - json (2.12.2) + json (2.21.1) kramdown (2.5.1) rexml (>= 3.3.9) kramdown-parser-gfm (1.1.0) From 2d5e1b65350d0ae3d61c94c034a5207221a88950 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Mon, 27 Jul 2026 23:45:45 +0800 Subject: [PATCH 06/11] [SPARK-56460][CORE] Define configs in text files ### What changes were proposed in this pull request? This PR adds a new module `common/config` which introduces a framework to define configs in prototext files. When the Spark application starts, we load config definitions from all prototext files and put them in a map with config name as the key, and the proto binary of the config definition as the value. These proto-backed configs are also registered in `ConfigEntry.knownConfigs` via a wrapper `ProtoBackedConfigEntry`, together with existing Spark configs. The proto schema defines each config entry with: key, value type, default value, scope (cluster vs session), mutability (static vs dynamic, i.e. whether the config can be changed after system initialization), visibility (public vs internal), binding policy (session/persisted/not_applicable for SQL views, UDFs, and procedures), documentation, and version. Enum values follow proto3 naming conventions with type-name prefixes (e.g. `SCOPE_CLUSTER`, `VALUE_TYPE_BOOL`, `VISIBILITY_PUBLIC`, `BINDING_POLICY_SESSION`, `MUTABILITY_STATIC`) to avoid namespace collisions. The `BindingPolicy` field maps to the existing Scala `ConfigBindingPolicy` enum. The `Mutability` field is used by `SQLConf.isStaticConfigKey` to determine whether a config can be changed at runtime. The new framework will co-exist with the existing config framework, until we migrate all existing configs to this new framework. For simple configs that are only accessed in one place, we can hardcode the config name in the place that accesses it, with a new API `SQLConf#getConfByKeyStrict` to avoid typo. Example in this PR: `spark.sql.optimizer.datasourceV2ExprFolding` For configs that are accessed in multiple places and we want to avoid hardcoding config name, or configs that need custom validation, we can still have an entry in `object SQLConf` to reference the config definition. Examples in this PR: `spark.sql.optimizer.maxIterations` and `spark.sql.shuffledHashJoinFactor`. As part of the migration, the 4 example configs are removed from the binding-policy exceptions allowlist (`configs-without-binding-policy-exceptions`) and assigned `NOT_APPLICABLE`, aligning them with standard binding-policy handling. Since `NOT_APPLICABLE` configs do not affect view/UDF/procedure resolution results, no resolved plans change. ### Why are the changes needed? Defining configs in various Scala objects is a bad practice: - Configs are registered when the Scala objects are loaded. To list all configs we must know all these Scala objects and load them. - It's hard to audit configs as they spread all over the codebase. - We will hit JVM limitation one day sooner or later, as defining configs in a Scala object is basically doing heavy work in the constructor, which has limitation of 64 kb bytecode size. ### Does this PR introduce _any_ user-facing change? No, it's developer facing ### How was this patch tested? new tests ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Code Closes #53488 from cloud-fan/conf. Authored-by: Wenchen Fan Signed-off-by: Wenchen Fan --- common/config/README.md | 122 ++++++ common/config/pom.xml | 187 +++++++++ .../apache/spark/config/ConfigRegistry.java | 210 ++++++++++ common/config/src/main/protobuf/buf.yaml | 24 ++ .../apache/spark/config/config_schema.proto | 108 +++++ .../config/cluster_configs/sql.textproto | 28 ++ .../config/session_configs/sql.textproto | 54 +++ .../spark/config/ConfigRegistrySuite.java | 382 ++++++++++++++++++ .../config/duplicate_key_file2.textproto | 28 ++ .../invalid_missing_binding_policy.textproto | 27 ++ .../config/invalid_missing_doc.textproto | 27 ++ .../config/invalid_missing_key.textproto | 27 ++ .../invalid_missing_mutability.textproto | 27 ++ .../config/invalid_missing_scope.textproto | 27 ++ .../invalid_missing_value_type.textproto | 27 ++ .../invalid_missing_visibility.textproto | 27 ++ ...alid_testdefault_without_default.textproto | 28 ++ .../spark/config/invalid_unordered.textproto | 40 ++ .../spark/config/test_configs.textproto | 91 +++++ common/utils/pom.xml | 5 + .../spark/internal/config/ConfigEntry.scala | 189 ++++++++- dev/mima | 10 +- pom.xml | 1 + project/SparkBuild.scala | 36 +- .../catalyst/util/V2ExpressionBuilder.scala | 2 +- .../apache/spark/sql/internal/SQLConf.scala | 84 ++-- .../spark/sql/internal/StaticSQLConf.scala | 8 +- .../apache/spark/sql/RuntimeConfigSuite.scala | 2 + .../v2/DataSourceV2StrategySuite.scala | 2 +- .../spark/sql/internal/SQLConfSuite.scala | 83 ++++ .../configs-without-binding-policy-exceptions | 4 - 31 files changed, 1873 insertions(+), 44 deletions(-) create mode 100644 common/config/README.md create mode 100644 common/config/pom.xml create mode 100644 common/config/src/main/java/org/apache/spark/config/ConfigRegistry.java create mode 100644 common/config/src/main/protobuf/buf.yaml create mode 100644 common/config/src/main/protobuf/org/apache/spark/config/config_schema.proto create mode 100644 common/config/src/main/resources/org/apache/spark/config/cluster_configs/sql.textproto create mode 100644 common/config/src/main/resources/org/apache/spark/config/session_configs/sql.textproto create mode 100644 common/config/src/test/java/org/apache/spark/config/ConfigRegistrySuite.java create mode 100644 common/config/src/test/resources/org/apache/spark/config/duplicate_key_file2.textproto create mode 100644 common/config/src/test/resources/org/apache/spark/config/invalid_missing_binding_policy.textproto create mode 100644 common/config/src/test/resources/org/apache/spark/config/invalid_missing_doc.textproto create mode 100644 common/config/src/test/resources/org/apache/spark/config/invalid_missing_key.textproto create mode 100644 common/config/src/test/resources/org/apache/spark/config/invalid_missing_mutability.textproto create mode 100644 common/config/src/test/resources/org/apache/spark/config/invalid_missing_scope.textproto create mode 100644 common/config/src/test/resources/org/apache/spark/config/invalid_missing_value_type.textproto create mode 100644 common/config/src/test/resources/org/apache/spark/config/invalid_missing_visibility.textproto create mode 100644 common/config/src/test/resources/org/apache/spark/config/invalid_testdefault_without_default.textproto create mode 100644 common/config/src/test/resources/org/apache/spark/config/invalid_unordered.textproto create mode 100644 common/config/src/test/resources/org/apache/spark/config/test_configs.textproto diff --git a/common/config/README.md b/common/config/README.md new file mode 100644 index 0000000000000..67c74ac48be28 --- /dev/null +++ b/common/config/README.md @@ -0,0 +1,122 @@ +# Spark Configuration Module + +This module provides a proto-based configuration system for defining Spark configurations. Configurations are defined using Protocol Buffers text format (`.textproto` files) and loaded at runtime by `ConfigRegistry`. + +## Directory Structure + +``` +common/config/src/main/ +├── java/org/apache/spark/config/ +│ └── ConfigRegistry.java # Loads and queries configs +├── protobuf/org/apache/spark/config/ +│ └── config_schema.proto # Defines the config schema +└── resources/org/apache/spark/config/ + ├── cluster_configs/ # CLUSTER scope configs + │ └── sql.textproto + └── session_configs/ # SESSION scope configs + └── sql.textproto +``` + +## How to Define a New Config + +### Step 1: Choose the Right File + +- **CLUSTER scope** (`MUTABILITY_STATIC`): If the config applies to the entire Spark cluster and cannot be changed at runtime, add to a file in `cluster_configs/` directory. +- **SESSION scope** (`MUTABILITY_DYNAMIC`): If the config applies to a Spark session and can be changed at runtime (e.g., via `SET`), add to a file in `session_configs/` directory. + +Currently, SQL configs go in `sql.textproto`. New categories can have their own files. + +### Step 2: Add the Config Entry + +Add a new `configs` block to the appropriate `.textproto` file: + +```protobuf +configs { + key: "spark.sql.myFeature.enabled" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_SESSION + doc: "When true, enables my new feature." + version: "4.0.0" +} +``` + +**Notes**: +- See `config_schema.proto` for field details and requirements +- Configs within each file must be ordered alphabetically by key +- A config that sets `test_default` must also set `default_value` (a test-only override needs a + production default to override); this is validated at load time +- These constraints are validated by tests + +### Step 3: Register the Config File (if new) + +If you created a new `.textproto` file, add it to `ConfigRegistry.java`: + +```java +private static final String[] DEFAULT_CONFIG_FILES = { + "org/apache/spark/config/cluster_configs/sql.textproto", + "org/apache/spark/config/session_configs/sql.textproto", + "org/apache/spark/config/session_configs/your_new_file.textproto" // Add here +}; +``` + +## Multi-line Documentation + +For long documentation strings, use protobuf string concatenation: + +```protobuf +configs { + key: "spark.sql.myFeature.threshold" + value_type: VALUE_TYPE_INT + default_value: "100" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "This is a long documentation string that spans multiple lines. " + "Simply place multiple quoted strings adjacent to each other " + "and protobuf will concatenate them automatically." + version: "4.0.0" +} +``` + +## Using Configs in Spark + +### Reading Config Values + +For configs accessed in only one or a few places, use `getConfByKeyStrict` with the config key directly. This will fail at runtime if the key is not defined in a `.textproto` file: + +```scala +def myFunc(conf: SQLConf): Unit = { + if (conf.getConfByKeyStrict[Boolean]("spark.sql.myFeature.enabled")) { + doSomething() + } +} +``` + +For configs accessed in many places, create a `ConfigEntry` in `object SQLConf` (or its friends) to avoid hardcoding keys: + +```scala +object SQLConf { + val MY_FEATURE_ENABLED = buildConfFromConfigFile[Boolean]("spark.sql.myFeature.enabled") +} + +def myFunc(conf: SQLConf): Unit = { + if (conf.getConf(SQLConf.MY_FEATURE_ENABLED)) { + doSomething() + } +} +``` + +### Adding Config Value Validation + +For configs that need to validate their values, create a `ConfigEntry` and use `checkValue`: + +```scala +val MY_THRESHOLD = buildConfFromConfigFile[Int]("spark.sql.myFeature.threshold") + .checkValue(_ > 0, "Threshold must be positive") +``` + diff --git a/common/config/pom.xml b/common/config/pom.xml new file mode 100644 index 0000000000000..b5dba2fcf9d9e --- /dev/null +++ b/common/config/pom.xml @@ -0,0 +1,187 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.13 + 5.0.0-SNAPSHOT + ../../pom.xml + + + spark-config_2.13 + jar + Spark Project Config + https://spark.apache.org/ + + config + + + + + org.apache.spark + spark-tags_${scala.binary.version} + + + + com.google.protobuf + protobuf-java + compile + + + + + org.apache.spark + spark-tags_${scala.binary.version} + test-jar + test + + + + org.junit.jupiter + junit-jupiter + test + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + + org.apache.maven.plugins + maven-shade-plugin + + + false + false + + + com.google.protobuf:* + + + + + com.google.protobuf + ${spark.shade.packageName}.protobuf + + com.google.protobuf.** + + + + + + *:* + + google/protobuf/** + + + + + + + package + + shade + + + + + + + + + + default-protoc + + + !skipDefaultProtoc + + + + + + com.github.os72 + protoc-jar-maven-plugin + ${protoc-jar-maven-plugin.version} + + + generate-sources + + run + + + com.google.protobuf:protoc:${protobuf.version} + ${protobuf.version} + + src/main/protobuf + + + + + + + + + + user-defined-protoc + + ${env.SPARK_PROTOC_EXEC_PATH} + + + + + com.github.os72 + protoc-jar-maven-plugin + ${protoc-jar-maven-plugin.version} + + + generate-sources + + run + + + com.google.protobuf:protoc:${protobuf.version} + ${protobuf.version} + ${spark.protoc.executable.path} + + src/main/protobuf + + + + + + + + + + diff --git a/common/config/src/main/java/org/apache/spark/config/ConfigRegistry.java b/common/config/src/main/java/org/apache/spark/config/ConfigRegistry.java new file mode 100644 index 0000000000000..08080081762aa --- /dev/null +++ b/common/config/src/main/java/org/apache/spark/config/ConfigRegistry.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.config; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.*; + +import com.google.protobuf.TextFormat; + +import org.apache.spark.config.protobuf.ConfigEntries; +import org.apache.spark.config.protobuf.ConfigEntry; + +/** + * A registry for Spark configuration entries. + * + *

This class loads configuration entries from .textproto files on the classpath + * and provides methods to query them by key. + * + *

For production use, use the static methods which access the default singleton instance. + * For testing, create a new instance with custom config file paths using the constructor. + * + *

Config files are organized into two directories: + *

    + *
  • cluster_configs/: Contains configs with CLUSTER scope
    • + *
  • session_configs/: Contains configs with SESSION scope
    • + *
+ */ +public class ConfigRegistry { + + // Explicit list of config files to load for production (package-private for testing) + static final String[] DEFAULT_CONFIG_FILES = { + "org/apache/spark/config/cluster_configs/sql.textproto", + "org/apache/spark/config/session_configs/sql.textproto" + }; + + // Lazy initialization holder class idiom for thread-safe lazy loading of default instance + private static class DefaultInstanceHolder { + static final ConfigRegistry INSTANCE = new ConfigRegistry(DEFAULT_CONFIG_FILES); + } + + // Instance field: map of config key -> ConfigEntry + private final Map configMap; + + /** + * Create a ConfigRegistry with custom config file paths. + * + * @param configFiles the resource paths of .textproto files to load + */ + public ConfigRegistry(String... configFiles) { + this.configMap = loadConfigs(configFiles); + } + + /** + * Get a config entry by its key. + * + * @param key the config key (e.g., "spark.sql.optimizer.maxIterations") + * @return the ConfigEntry if found, null otherwise + */ + public ConfigEntry get(String key) { + return configMap.get(key); + } + + /** + * Check if a config key exists in the registry. + * + * @param key the config key + * @return true if the key exists, false otherwise + */ + public boolean contains(String key) { + return configMap.containsKey(key); + } + + /** + * Get all registered config keys. + * + * @return an unmodifiable set of all config keys + */ + public Set keys() { + return configMap.keySet(); + } + + /** + * Get all registered config entries. + * + * @return an unmodifiable collection of all config entries + */ + public Collection all() { + return configMap.values(); + } + + // ========================================================================== + // Static methods for production use (delegate to default singleton instance) + // ========================================================================== + + /** + * Get the default singleton instance. + * + * @return the default ConfigRegistry instance + */ + public static ConfigRegistry getInstance() { + return DefaultInstanceHolder.INSTANCE; + } + + /** + * Get a config entry by its key from the default registry. + * + * @param key the config key (e.g., "spark.sql.optimizer.maxIterations") + * @return the ConfigEntry if found, null otherwise + */ + public static ConfigEntry getConfig(String key) { + return getInstance().get(key); + } + + /** + * Check if a config key exists in the default registry. + * + * @param key the config key + * @return true if the key exists, false otherwise + */ + public static boolean containsConfig(String key) { + return getInstance().contains(key); + } + + /** + * Get all registered config keys from the default registry. + * + * @return an unmodifiable set of all config keys + */ + public static Set allKeys() { + return getInstance().keys(); + } + + /** + * Get all registered config entries from the default registry. + * + * @return an unmodifiable collection of all config entries + */ + public static Collection allConfigs() { + return getInstance().all(); + } + + // ========================================================================== + // Private helper methods + // ========================================================================== + + /** + * Load configs from .textproto files. + */ + private static Map loadConfigs(String[] configFiles) { + Map result = new HashMap<>(); + ClassLoader classLoader = ConfigRegistry.class.getClassLoader(); + + for (String file : configFiles) { + List configs = loadConfigFile(classLoader, file); + for (ConfigEntry config : configs) { + result.put(config.getKey(), config); + } + } + + return Collections.unmodifiableMap(result); + } + + /** + * Load configs from a single .textproto file. + * Package-private for testing. + */ + static List loadConfigFile( + ClassLoader classLoader, + String resourcePath) { + InputStream inputStream = classLoader.getResourceAsStream(resourcePath); + if (inputStream == null) { + throw internalError("Config file not found on classpath: " + resourcePath); + } + + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + ConfigEntries.Builder builder = ConfigEntries.newBuilder(); + TextFormat.getParser().merge(reader, builder); + return builder.build().getConfigsList(); + } catch (Exception e) { + throw internalError("Failed to load config file: " + resourcePath, e); + } + } + + private static AssertionError internalError(String message) { + return new AssertionError(message); + } + + private static AssertionError internalError(String message, Throwable cause) { + return new AssertionError(message, cause); + } +} diff --git a/common/config/src/main/protobuf/buf.yaml b/common/config/src/main/protobuf/buf.yaml new file mode 100644 index 0000000000000..3b3903fc97c9f --- /dev/null +++ b/common/config/src/main/protobuf/buf.yaml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: v1 +breaking: + use: + - FILE +lint: + use: + - BASIC + diff --git a/common/config/src/main/protobuf/org/apache/spark/config/config_schema.proto b/common/config/src/main/protobuf/org/apache/spark/config/config_schema.proto new file mode 100644 index 0000000000000..c46dbdbb97d4d --- /dev/null +++ b/common/config/src/main/protobuf/org/apache/spark/config/config_schema.proto @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +package org.apache.spark.config; + +option java_multiple_files = true; +option java_package = "org.apache.spark.config.protobuf"; + +// The scope of the config +enum Scope { + SCOPE_UNSPECIFIED = 0; + // The config is effective for the entire Spark cluster. + SCOPE_CLUSTER = 1; + // The config is effective for the current session. + SCOPE_SESSION = 2; +} + +// The type of the config value +enum ValueType { + VALUE_TYPE_UNSPECIFIED = 0; + VALUE_TYPE_BOOL = 1; + VALUE_TYPE_INT = 2; + VALUE_TYPE_LONG = 3; + VALUE_TYPE_DOUBLE = 4; + VALUE_TYPE_STRING = 5; +} + +// The visibility of the config +enum Visibility { + VISIBILITY_UNSPECIFIED = 0; + VISIBILITY_PUBLIC = 1; + VISIBILITY_INTERNAL = 2; +} + +// The binding policy of the config for SQL views, UDFs, and procedures. +enum BindingPolicy { + BINDING_POLICY_UNSPECIFIED = 0; + // The config value propagates from the active session to views/UDFs/procedures. + BINDING_POLICY_SESSION = 1; + // The config uses the value saved at view/UDF/procedure creation time, + // or the Spark default if none was saved. + BINDING_POLICY_PERSISTED = 2; + // The config does not apply to views/UDFs/procedures. At runtime, behaves like SESSION. + BINDING_POLICY_NOT_APPLICABLE = 3; +} + +// Whether the config can be changed after system initialization. +enum Mutability { + MUTABILITY_UNSPECIFIED = 0; + // The config is fixed once the Spark application starts. + MUTABILITY_STATIC = 1; + // The config can be changed at runtime (e.g., via SET). + MUTABILITY_DYNAMIC = 2; +} + +// A single config entry +message ConfigEntry { + // [Required] The config key. + optional string key = 1; + + // [Required] The type of the config value + optional ValueType value_type = 2; + + // The default value as a string representation + optional string default_value = 3; + + // The default value to use during testing, if different from default_value. + // Requires default_value to be set, since it only overrides the production default in tests. + optional string test_default = 4; + + // [Required] The scope of the config (CLUSTER or SESSION). + optional Scope scope = 5; + + // [Required] Whether the config can be changed after system initialization. + optional Mutability mutability = 6; + + // [Required] The visibility of the config (PUBLIC or INTERNAL). + optional Visibility visibility = 7; + + // [Required] The binding policy for SQL views, UDFs, and procedures. + optional BindingPolicy binding_policy = 8; + + // [Required] Documentation for the config. + optional string doc = 9; + + // The Spark version when this config was introduced. + optional string version = 10; +} + +// A collection of config entries +message ConfigEntries { + repeated ConfigEntry configs = 1; +} diff --git a/common/config/src/main/resources/org/apache/spark/config/cluster_configs/sql.textproto b/common/config/src/main/resources/org/apache/spark/config/cluster_configs/sql.textproto new file mode 100644 index 0000000000000..f599ed52d794b --- /dev/null +++ b/common/config/src/main/resources/org/apache/spark/config/cluster_configs/sql.textproto @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SQL configuration entries (CLUSTER scope) + +configs { + key: "spark.sql.ui.retainedExecutions" + value_type: VALUE_TYPE_INT + default_value: "1000" + scope: SCOPE_CLUSTER + mutability: MUTABILITY_STATIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "Number of executions to retain in the Spark UI." + version: "1.5.0" +} diff --git a/common/config/src/main/resources/org/apache/spark/config/session_configs/sql.textproto b/common/config/src/main/resources/org/apache/spark/config/session_configs/sql.textproto new file mode 100644 index 0000000000000..06eaf7ee02779 --- /dev/null +++ b/common/config/src/main/resources/org/apache/spark/config/session_configs/sql.textproto @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SQL configuration entries (SESSION scope) + +configs { + key: "spark.sql.optimizer.datasourceV2ExprFolding" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_INTERNAL + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "When this config is set to true, do safe constant folding for the " + "expressions before translation and pushdown." + version: "4.1.0" +} + +configs { + key: "spark.sql.optimizer.maxIterations" + value_type: VALUE_TYPE_INT + default_value: "100" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_INTERNAL + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "The max number of iterations the optimizer runs." + version: "2.0.0" +} + +configs { + key: "spark.sql.shuffledHashJoinFactor" + value_type: VALUE_TYPE_INT + default_value: "3" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "The shuffle hash join can be selected if the data size of small " + "side multiplied by this factor is still smaller than the large side." + version: "3.3.0" +} diff --git a/common/config/src/test/java/org/apache/spark/config/ConfigRegistrySuite.java b/common/config/src/test/java/org/apache/spark/config/ConfigRegistrySuite.java new file mode 100644 index 0000000000000..9ec05c1a83287 --- /dev/null +++ b/common/config/src/test/java/org/apache/spark/config/ConfigRegistrySuite.java @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.config; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import org.apache.spark.config.protobuf.BindingPolicy; +import org.apache.spark.config.protobuf.ConfigEntry; +import org.apache.spark.config.protobuf.Mutability; +import org.apache.spark.config.protobuf.Scope; +import org.apache.spark.config.protobuf.ValueType; +import org.apache.spark.config.protobuf.Visibility; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for {@link ConfigRegistry}. + */ +public class ConfigRegistrySuite { + + private static final String TEST_CONFIG_FILE = + "org/apache/spark/config/test_configs.textproto"; + + private static final ConfigRegistry registry = new ConfigRegistry(TEST_CONFIG_FILE); + + private static final ClassLoader CLASS_LOADER = ConfigRegistrySuite.class.getClassLoader(); + + // ========================================================================== + // Validation helpers + // ========================================================================== + + /** + * Validate configs in one file: required fields and alphabetical ordering. + * Convenience method that loads the file first. + * + * @param file the resource path of the .textproto file + */ + private static void validateConfigsInOneFile(String file) { + List configs = ConfigRegistry.loadConfigFile(CLASS_LOADER, file); + validateConfigsInOneFile(configs, file); + } + + /** + * Validate configs: required fields and alphabetical ordering. + * + * @param configs the list of config entries + * @param file the file path (for error messages) + */ + private static void validateConfigsInOneFile(List configs, String file) { + String previousKey = null; + for (ConfigEntry config : configs) { + // Validate required fields + List missingFields = new ArrayList<>(); + if (config.getKey().isEmpty()) { + missingFields.add("key"); + } + if (config.getValueType() == ValueType.VALUE_TYPE_UNSPECIFIED) { + missingFields.add("value_type"); + } + if (config.getScope() == Scope.SCOPE_UNSPECIFIED) { + missingFields.add("scope"); + } + if (config.getVisibility() == Visibility.VISIBILITY_UNSPECIFIED) { + missingFields.add("visibility"); + } + if (config.getBindingPolicy() == BindingPolicy.BINDING_POLICY_UNSPECIFIED) { + missingFields.add("binding_policy"); + } + if (config.getMutability() == Mutability.MUTABILITY_UNSPECIFIED) { + missingFields.add("mutability"); + } + if (config.getDoc().isEmpty()) { + missingFields.add("doc"); + } + if (!missingFields.isEmpty()) { + String keyInfo = config.getKey().isEmpty() ? "" : " for key '" + config.getKey() + "'"; + fail("Missing required fields " + missingFields + keyInfo + " in " + file); + } + + // TODO: Relax this constraint once the config framework fully supports + // cluster-scoped dynamic configs and session-scoped static configs. + if (config.getScope() == Scope.SCOPE_CLUSTER + && config.getMutability() != Mutability.MUTABILITY_STATIC) { + fail("CLUSTER scope config must be STATIC" + + " for key '" + config.getKey() + "' in " + file); + } + if (config.getScope() == Scope.SCOPE_SESSION + && config.getMutability() != Mutability.MUTABILITY_DYNAMIC) { + fail("SESSION scope config must be DYNAMIC" + + " for key '" + config.getKey() + "' in " + file); + } + + // A test_default only overrides default_value in test environments, so it requires a + // default_value to override. Otherwise the entry would have a default under test but none + // in production, diverging its shape between the two. + if (config.hasTestDefault() && !config.hasDefaultValue()) { + fail("test_default requires default_value" + + " for key '" + config.getKey() + "' in " + file); + } + + // Validate alphabetical ordering + String key = config.getKey(); + if (previousKey != null && key.compareTo(previousKey) < 0) { + fail("Config keys must be ordered alphabetically in " + file + + ": '" + key + "' should come before '" + previousKey + "'"); + } + previousKey = key; + } + } + + /** + * Validate all config files: required fields, ordering, and no duplicate keys. + * Each file is loaded only once. + * + * @param configFiles the list of config file paths to check + */ + private static void validateAllConfigFiles(String[] configFiles) { + Map keyToFile = new HashMap<>(); + for (String file : configFiles) { + List configs = ConfigRegistry.loadConfigFile(CLASS_LOADER, file); + // Validate required fields and ordering + validateConfigsInOneFile(configs, file); + // Validate no duplicate keys across files + for (ConfigEntry config : configs) { + String key = config.getKey(); + if (keyToFile.containsKey(key)) { + fail("Duplicate config key '" + key + "': " + + "first defined in " + keyToFile.get(key) + ", duplicated in " + file); + } + keyToFile.put(key, file); + } + } + } + + // ========================================================================== + // Basic functionality tests + // ========================================================================== + + @Test + public void testLoadConfigs() { + assertEquals(6, registry.keys().size()); + assertTrue(registry.contains("spark.test.bool.config")); + assertTrue(registry.contains("spark.test.int.config")); + assertTrue(registry.contains("spark.test.string.config")); + assertTrue(registry.contains("spark.test.long.doc.config")); + assertTrue(registry.contains("spark.test.optional.config")); + assertTrue(registry.contains("spark.test.testdefault.config")); + } + + @Test + public void testBoolConfig() { + ConfigEntry config = registry.get("spark.test.bool.config"); + assertNotNull(config); + assertEquals("spark.test.bool.config", config.getKey()); + assertEquals(ValueType.VALUE_TYPE_BOOL, config.getValueType()); + assertEquals("true", config.getDefaultValue()); + assertEquals(Scope.SCOPE_SESSION, config.getScope()); + assertEquals(Visibility.VISIBILITY_PUBLIC, config.getVisibility()); + assertEquals("A test boolean config", config.getDoc()); + assertEquals("4.0.0", config.getVersion()); + assertEquals(BindingPolicy.BINDING_POLICY_SESSION, config.getBindingPolicy()); + } + + @Test + public void testIntConfig() { + ConfigEntry config = registry.get("spark.test.int.config"); + assertNotNull(config); + assertEquals("spark.test.int.config", config.getKey()); + assertEquals(ValueType.VALUE_TYPE_INT, config.getValueType()); + assertEquals("42", config.getDefaultValue()); + assertEquals(Scope.SCOPE_CLUSTER, config.getScope()); + assertEquals(Visibility.VISIBILITY_INTERNAL, config.getVisibility()); + assertEquals("A test integer config", config.getDoc()); + assertEquals("4.0.0", config.getVersion()); + assertEquals(BindingPolicy.BINDING_POLICY_NOT_APPLICABLE, config.getBindingPolicy()); + } + + @Test + public void testStringConfig() { + ConfigEntry config = registry.get("spark.test.string.config"); + assertNotNull(config); + assertEquals("spark.test.string.config", config.getKey()); + assertEquals(ValueType.VALUE_TYPE_STRING, config.getValueType()); + assertEquals("default_value", config.getDefaultValue()); + assertEquals(Scope.SCOPE_SESSION, config.getScope()); + assertEquals(Visibility.VISIBILITY_PUBLIC, config.getVisibility()); + assertEquals("A test string config", config.getDoc()); + assertEquals("4.0.0", config.getVersion()); + assertEquals(BindingPolicy.BINDING_POLICY_NOT_APPLICABLE, config.getBindingPolicy()); + } + + @Test + public void testGetNonExistent() { + assertNull(registry.get("spark.nonexistent.config")); + assertFalse(registry.contains("spark.nonexistent.config")); + } + + @Test + public void testAll() { + assertEquals(6, registry.all().size()); + } + + @Test + public void testOptionalConfig() { + // A config without a default_value: hasDefaultValue is false and the field is empty. + ConfigEntry config = registry.get("spark.test.optional.config"); + assertNotNull(config); + assertEquals("spark.test.optional.config", config.getKey()); + assertEquals(ValueType.VALUE_TYPE_STRING, config.getValueType()); + assertFalse(config.hasDefaultValue()); + assertEquals("", config.getDefaultValue()); + } + + @Test + public void testTestDefaultConfig() { + // A config with a test_default distinct from default_value. + ConfigEntry config = registry.get("spark.test.testdefault.config"); + assertNotNull(config); + assertEquals("spark.test.testdefault.config", config.getKey()); + assertEquals("10", config.getDefaultValue()); + assertTrue(config.hasTestDefault()); + assertEquals("20", config.getTestDefault()); + } + + @Test + public void testMultiLineDoc() { + ConfigEntry config = registry.get("spark.test.long.doc.config"); + assertNotNull(config); + assertEquals("spark.test.long.doc.config", config.getKey()); + assertEquals(ValueType.VALUE_TYPE_STRING, config.getValueType()); + assertEquals("test", config.getDefaultValue()); + assertEquals(Scope.SCOPE_SESSION, config.getScope()); + assertEquals(Visibility.VISIBILITY_PUBLIC, config.getVisibility()); + // Verify multi-line string concatenation works + String expectedDoc = "This is a very long documentation string that spans multiple lines. " + + "It demonstrates the prototext multi-line string concatenation feature. " + + "Each quoted segment will be concatenated together into a single string, " + + "which is useful for configs with lengthy descriptions."; + assertEquals(expectedDoc, config.getDoc()); + assertEquals("4.0.0", config.getVersion()); + } + + // ========================================================================== + // Validation of production config files + // ========================================================================== + + @Test + public void testProductionConfigsAreValid() { + validateAllConfigFiles(ConfigRegistry.DEFAULT_CONFIG_FILES); + } + + // ========================================================================== + // Tests for validation logic using invalid config files + // ========================================================================== + + @Test + public void testValidationMissingKey() { + String file = "org/apache/spark/config/invalid_missing_key.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("key")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingValueType() { + String file = "org/apache/spark/config/invalid_missing_value_type.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("value_type")); + assertTrue(error.getMessage().contains("spark.test.missing.value.type")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingScope() { + String file = "org/apache/spark/config/invalid_missing_scope.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("scope")); + assertTrue(error.getMessage().contains("spark.test.missing.scope")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingVisibility() { + String file = "org/apache/spark/config/invalid_missing_visibility.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("visibility")); + assertTrue(error.getMessage().contains("spark.test.missing.visibility")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingDoc() { + String file = "org/apache/spark/config/invalid_missing_doc.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("doc")); + assertTrue(error.getMessage().contains("spark.test.missing.doc")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingBindingPolicy() { + String file = "org/apache/spark/config/invalid_missing_binding_policy.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("binding_policy")); + assertTrue(error.getMessage().contains("spark.test.missing.binding.policy")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingMutability() { + String file = "org/apache/spark/config/invalid_missing_mutability.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("mutability")); + assertTrue(error.getMessage().contains("spark.test.missing.mutability")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationTestDefaultWithoutDefault() { + String file = "org/apache/spark/config/invalid_testdefault_without_default.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("test_default requires default_value")); + assertTrue(error.getMessage().contains("spark.test.testdefault.without.default")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationUnorderedConfigs() { + String file = "org/apache/spark/config/invalid_unordered.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("must be ordered alphabetically")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationDuplicateKeys() { + String file1 = TEST_CONFIG_FILE; + String file2 = "org/apache/spark/config/duplicate_key_file2.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateAllConfigFiles(new String[]{file1, file2})); + assertTrue(error.getMessage().contains("Duplicate config key")); + assertTrue(error.getMessage().contains("spark.test.bool.config")); + assertTrue(error.getMessage().contains(file1)); + assertTrue(error.getMessage().contains(file2)); + } +} diff --git a/common/config/src/test/resources/org/apache/spark/config/duplicate_key_file2.textproto b/common/config/src/test/resources/org/apache/spark/config/duplicate_key_file2.textproto new file mode 100644 index 0000000000000..5221f87af8839 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/duplicate_key_file2.textproto @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Second test file with a key that duplicates one in test_configs.textproto + +configs { + key: "spark.test.bool.config" + value_type: VALUE_TYPE_BOOL + default_value: "false" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_SESSION + doc: "Duplicate of spark.test.bool.config" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_binding_policy.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_binding_policy.textproto new file mode 100644 index 0000000000000..3f99ec3facb6f --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_binding_policy.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing binding_policy field + +configs { + key: "spark.test.missing.binding.policy" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + doc: "A config with missing binding_policy" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_doc.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_doc.textproto new file mode 100644 index 0000000000000..a0dcbfd6afb64 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_doc.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing doc field + +configs { + key: "spark.test.missing.doc" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_key.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_key.textproto new file mode 100644 index 0000000000000..14f561da7c501 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_key.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing key field + +configs { + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config with missing key" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_mutability.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_mutability.textproto new file mode 100644 index 0000000000000..e8b759941515b --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_mutability.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing mutability field + +configs { + key: "spark.test.missing.mutability" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config with missing mutability" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_scope.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_scope.textproto new file mode 100644 index 0000000000000..56f3661f4d032 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_scope.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing scope field + +configs { + key: "spark.test.missing.scope" + value_type: VALUE_TYPE_BOOL + default_value: "true" + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config with missing scope" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_value_type.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_value_type.textproto new file mode 100644 index 0000000000000..7652ac680f66d --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_value_type.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing value_type field + +configs { + key: "spark.test.missing.value.type" + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config with missing value_type" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_visibility.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_visibility.textproto new file mode 100644 index 0000000000000..e1a5a731e6167 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_visibility.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing visibility field + +configs { + key: "spark.test.missing.visibility" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config with missing visibility" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_testdefault_without_default.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_testdefault_without_default.textproto new file mode 100644 index 0000000000000..a9abe3b60a133 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_testdefault_without_default.textproto @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with test_default but no default_value + +configs { + key: "spark.test.testdefault.without.default" + value_type: VALUE_TYPE_INT + test_default: "20" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A test config with a test default but no default value" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_unordered.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_unordered.textproto new file mode 100644 index 0000000000000..99a6c983717d6 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_unordered.textproto @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with unordered config keys (z comes before a) + +configs { + key: "spark.test.z.config" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "Z config" + version: "4.0.0" +} + +configs { + key: "spark.test.a.config" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config - should come before Z" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/test_configs.textproto b/common/config/src/test/resources/org/apache/spark/config/test_configs.textproto new file mode 100644 index 0000000000000..3ce51f2d0b4c6 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/test_configs.textproto @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test configuration entries + +configs { + key: "spark.test.bool.config" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_SESSION + doc: "A test boolean config" + version: "4.0.0" +} + +configs { + key: "spark.test.int.config" + value_type: VALUE_TYPE_INT + default_value: "42" + scope: SCOPE_CLUSTER + mutability: MUTABILITY_STATIC + visibility: VISIBILITY_INTERNAL + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A test integer config" + version: "4.0.0" +} + +configs { + key: "spark.test.long.doc.config" + value_type: VALUE_TYPE_STRING + default_value: "test" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_PERSISTED + doc: "This is a very long documentation string that spans multiple lines. " + "It demonstrates the prototext multi-line string concatenation feature. " + "Each quoted segment will be concatenated together into a single string, " + "which is useful for configs with lengthy descriptions." + version: "4.0.0" +} + +configs { + key: "spark.test.optional.config" + value_type: VALUE_TYPE_STRING + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A test config without a default value" + version: "4.0.0" +} + +configs { + key: "spark.test.string.config" + value_type: VALUE_TYPE_STRING + default_value: "default_value" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A test string config" + version: "4.0.0" +} + +configs { + key: "spark.test.testdefault.config" + value_type: VALUE_TYPE_INT + default_value: "10" + test_default: "20" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A test config with a distinct test default" + version: "4.0.0" +} diff --git a/common/utils/pom.xml b/common/utils/pom.xml index e09467141c700..17d1186201e71 100644 --- a/common/utils/pom.xml +++ b/common/utils/pom.xml @@ -39,6 +39,11 @@ org.apache.spark spark-tags_${scala.binary.version} + + org.apache.spark + spark-config_${scala.binary.version} + ${project.version} + org.apache.spark spark-common-utils-java_${scala.binary.version} diff --git a/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala b/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala index 97fddf632ed23..abede2d57d519 100644 --- a/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala +++ b/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala @@ -17,6 +17,11 @@ package org.apache.spark.internal.config +import org.apache.spark.SparkException +import org.apache.spark.config.ConfigRegistry +import org.apache.spark.config.protobuf.{BindingPolicy, ConfigEntry => ProtoConfigEntry, ValueType, Visibility} +import org.apache.spark.util.SparkEnvUtils + // ==================================================================================== // The guideline for naming configurations // ==================================================================================== @@ -283,6 +288,107 @@ private[spark] class FallbackConfigEntry[T] ( } } +/** + * Marker trait for config entries backed by proto definitions. + */ +private[spark] sealed trait ProtoBackedBase + +/** + * A proto-backed config entry with a default value. + */ +private[spark] class ProtoBackedConfigEntry[T]( + protoEntry: ProtoConfigEntry, + valueConverter: String => T, + stringConverter: T => String) + extends ConfigEntry[T]( + key = protoEntry.getKey, + prependedKey = None, + prependSeparator = "", + alternatives = Nil, + valueConverter = valueConverter, + stringConverter = stringConverter, + doc = protoEntry.getDoc, + isPublic = protoEntry.getVisibility != Visibility.VISIBILITY_INTERNAL, + version = protoEntry.getVersion, + bindingPolicy = ProtoBackedConfigEntry.toBindingPolicy(protoEntry) + ) with ProtoBackedBase { + + override def defaultValueString: String = + defaultValue.map(stringConverter).getOrElse(ConfigEntry.UNDEFINED) + + // The default is derived from immutable proto fields (and the fixed testing flag), so convert it + // once and cache rather than re-parsing the string on every default-fallback read. + private lazy val cachedDefaultValue: Option[T] = { + val defaultStrOpt = if (SparkEnvUtils.isTesting && protoEntry.hasTestDefault) { + Some(protoEntry.getTestDefault) + } else if (protoEntry.hasDefaultValue) { + Some(protoEntry.getDefaultValue) + } else { + None + } + defaultStrOpt.map(valueConverter) + } + + override def defaultValue: Option[T] = cachedDefaultValue + + override def readFrom(reader: ConfigReader): T = { + readString(reader).map(valueConverter).getOrElse(defaultValue.get) + } + + def checkValue(validator: T => Boolean, errorMsg: String): ProtoBackedConfigEntry[T] = { + new ProtoBackedConfigEntry[T]( + protoEntry, + str => { + val v = valueConverter(str) + if (!validator(v)) { + throw ConfigHelpers.configRequirementError(key, str, errorMsg) + } + v + }, + stringConverter + ) + } +} + +/** + * A proto-backed config entry that does not have a default value. + */ +private[spark] class ProtoBackedOptionalConfigEntry[T]( + protoEntry: ProtoConfigEntry, + rawValueConverter: String => T, + rawStringConverter: T => String) + extends ConfigEntry[Option[T]]( + key = protoEntry.getKey, + prependedKey = None, + prependSeparator = "", + alternatives = Nil, + valueConverter = s => Some(rawValueConverter(s)), + stringConverter = v => v.map(rawStringConverter).orNull, + doc = protoEntry.getDoc, + isPublic = protoEntry.getVisibility != Visibility.VISIBILITY_INTERNAL, + version = protoEntry.getVersion, + bindingPolicy = ProtoBackedConfigEntry.toBindingPolicy(protoEntry) + ) with ProtoBackedBase { + + override def defaultValueString: String = ConfigEntry.UNDEFINED + + override def readFrom(reader: ConfigReader): Option[T] = { + readString(reader).map(rawValueConverter) + } +} + +private[spark] object ProtoBackedConfigEntry { + def toBindingPolicy( + protoEntry: ProtoConfigEntry): Option[ConfigBindingPolicy.Value] = { + protoEntry.getBindingPolicy match { + case BindingPolicy.BINDING_POLICY_SESSION => Some(ConfigBindingPolicy.SESSION) + case BindingPolicy.BINDING_POLICY_PERSISTED => Some(ConfigBindingPolicy.PERSISTED) + case BindingPolicy.BINDING_POLICY_NOT_APPLICABLE => Some(ConfigBindingPolicy.NOT_APPLICABLE) + case _ => None + } + } +} + private[spark] object ConfigEntry { val UNDEFINED = "" @@ -290,13 +396,94 @@ private[spark] object ConfigEntry { private[spark] val knownConfigs = new java.util.concurrent.ConcurrentHashMap[String, ConfigEntry[_]]() + // Register all proto-backed configs at object initialization. + // These can be overwritten later by modules that need to add validation. + ConfigRegistry.allConfigs().forEach { protoEntry => + createProtoBackedConfigEntry(protoEntry) + } + def registerEntry(entry: ConfigEntry[_]): Unit = { val existing = knownConfigs.putIfAbsent(entry.key, entry) - require(existing == null, s"Config entry ${entry.key} already registered!") + if (existing != null) { + // A key registered twice is normally a bug (typo or accidental duplicate), so we fail loudly. + // The one exception is the enhancement pattern: a proto-backed config is registered eagerly + // at object init, then a Scala entry built via `buildConfFromConfigFile` (e.g. to add + // `checkValue`) may intentionally replace it. We allow the overwrite only when the existing + // entry is proto-backed; this deliberately trades the duplicate-detection net for that key, + // so the Scala side must still reference a key that is genuinely defined in a .textproto + // file. + require(existing.isInstanceOf[ProtoBackedBase] && entry.isInstanceOf[ProtoBackedBase], + s"Config entry ${entry.key} already registered!") + knownConfigs.put(entry.key, entry) + } } def findEntry(key: String): ConfigEntry[_] = knownConfigs.get(key) def listAllEntries(): java.util.Collection[ConfigEntry[_]] = knownConfigs.values() + def findProtoBackedEntry(key: String): ProtoBackedConfigEntry[_] = { + knownConfigs.get(key) match { + case entry: ProtoBackedConfigEntry[_] => entry + case _ => null + } + } + + def findProtoDefinedEntry(key: String): ConfigEntry[_] = { + if (ConfigRegistry.containsConfig(key)) knownConfigs.get(key) else null + } + + def listAllProtoDefinedConfigs(): java.util.Collection[ConfigEntry[_]] = { + new java.util.AbstractCollection[ConfigEntry[_]]() { + override def iterator(): java.util.Iterator[ConfigEntry[_]] = { + val keys = ConfigRegistry.allKeys().iterator() + new java.util.Iterator[ConfigEntry[_]]() { + override def hasNext: Boolean = keys.hasNext + override def next(): ConfigEntry[_] = knownConfigs.get(keys.next()) + } + } + + override def size(): Int = ConfigRegistry.allKeys().size() + } + } + + private def createProtoBackedConfigEntry( + protoEntry: ProtoConfigEntry): ConfigEntry[_] = { + // `test_default` only overrides `default_value` in test environments; a config that sets + // `test_default` without `default_value` would be a has-default entry under test but an + // optional entry in production, diverging the entry's shape (and `buildConfFromConfigFile`'s + // success/failure) between the two. Reject it here so the misconfiguration fails consistently. + require(!protoEntry.hasTestDefault || protoEntry.hasDefaultValue, + s"Config entry ${protoEntry.getKey} sets test_default without default_value; " + + "a config with a test default must also declare a default value.") + // With the invariant above, `test_default` implies `default_value`, so having a default is + // equivalent to having a `default_value` in every environment. + val hasDefault = protoEntry.hasDefaultValue + protoEntry.getValueType match { + case ValueType.VALUE_TYPE_BOOL => + createTypedEntry[Boolean](protoEntry, hasDefault, _.toBoolean, _.toString) + case ValueType.VALUE_TYPE_INT => + createTypedEntry[Int](protoEntry, hasDefault, _.toInt, _.toString) + case ValueType.VALUE_TYPE_LONG => + createTypedEntry[Long](protoEntry, hasDefault, _.toLong, _.toString) + case ValueType.VALUE_TYPE_DOUBLE => + createTypedEntry[Double](protoEntry, hasDefault, _.toDouble, _.toString) + case ValueType.VALUE_TYPE_STRING => + createTypedEntry[String](protoEntry, hasDefault, identity[String], identity[String]) + case other => + throw SparkException.internalError(s"Unsupported value type: $other") + } + } + + private def createTypedEntry[T]( + protoEntry: ProtoConfigEntry, + hasDefault: Boolean, + valueConverter: String => T, + stringConverter: T => String): ConfigEntry[_] = { + if (hasDefault) { + new ProtoBackedConfigEntry[T](protoEntry, valueConverter, stringConverter) + } else { + new ProtoBackedOptionalConfigEntry[T](protoEntry, valueConverter, stringConverter) + } + } } diff --git a/dev/mima b/dev/mima index 17558be37fc8e..39be02a1dd557 100755 --- a/dev/mima +++ b/dev/mima @@ -25,8 +25,16 @@ FWDIR="$(cd "`dirname "$0"`"/..; pwd)" cd "$FWDIR" SPARK_PROFILES=${1:-"-Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} + +# Capture sbt output to show errors if the command fails +OLD_DEPS_OUTPUT="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" 2>&1)" || { + echo "ERROR: Failed to export oldDeps classpath:" + echo "$OLD_DEPS_OUTPUT" | grep -E "^\[error\]" + exit 1 +} +OLD_DEPS_CLASSPATH="$(echo "$OLD_DEPS_OUTPUT" | grep jar | tail -n1)" + TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)" -OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)" rm -f .generated-mima* diff --git a/pom.xml b/pom.xml index f6354961cf292..0eaf37497f683 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,7 @@ common/utils-java common/variant common/tags + common/config sql/connect/shims core graphx diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 53412c20d8313..ef027de5699f6 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -64,10 +64,10 @@ object BuildCommons { val allProjects@Seq( core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, kvstore, - commonUtils, commonUtilsJava, variant, pipelines, _* + commonUtils, commonUtilsJava, variant, pipelines, sparkConfig, _* ) = Seq( "core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe", - "tags", "sketch", "kvstore", "common-utils", "common-utils-java", "variant", "pipelines" + "tags", "sketch", "kvstore", "common-utils", "common-utils-java", "variant", "pipelines", "config" ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ connectProjects ++ udfWorkerProjects @@ -373,6 +373,7 @@ object SparkBuild extends PomBuild { "-groups", "-skip-packages", Seq( "org.apache.spark.api.python", + "org.apache.spark.config", "org.apache.spark.deploy", "org.apache.spark.kafka010", "org.apache.spark.network", @@ -415,7 +416,7 @@ object SparkBuild extends PomBuild { Seq( spark, hive, hiveThriftServer, repl, networkCommon, networkShuffle, networkYarn, unsafe, tags, tokenProviderKafka010, sqlKafka010, pipelines, connectCommon, connect, - connectJdbc, connectClient, variant, connectShims, profiler, commonUtilsJava, + connectJdbc, connectClient, variant, connectShims, profiler, commonUtilsJava, sparkConfig, udfWorkerProto, udfWorkerCore, udfWorkerGrpc ).contains(x) } @@ -475,6 +476,9 @@ object SparkBuild extends PomBuild { /* UDF Worker gRPC settings */ enable(UDFWorkerGrpc.settings)(udfWorkerGrpc) + /* Config module protobuf settings */ + enable(SparkConfig.settings)(sparkConfig) + enable(DockerIntegrationTests.settings)(dockerIntegrationTests) enable(KubernetesIntegrationTests.settings)(kubernetesIntegrationTests) @@ -682,6 +686,31 @@ object Core { } } +object SparkConfig { + import BuildCommons.protoVersion + lazy val settings = Seq( + // Setting version for the protobuf compiler. + PB.protocVersion := BuildCommons.protoVersion, + libraryDependencies ++= { + Seq( + "com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf" + ) + }, + (Compile / PB.targets) := Seq( + PB.gens.java -> (Compile / sourceManaged).value + ) + ) ++ { + val sparkProtocExecPath = sys.props.get("spark.protoc.executable.path") + if (sparkProtocExecPath.isDefined) { + Seq( + PB.protocExecutable := file(sparkProtocExecPath.get) + ) + } else { + Seq.empty + } + } +} + object SparkConnectCommon { import BuildCommons.protoVersion @@ -1642,6 +1671,7 @@ object Unidoc { protected def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = { packages .map(_.filterNot(_.getName.contains("$"))) + .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/config"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/deploy"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/examples"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/internal"))) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala index e40a6e6df88b5..3fb20dcd6420d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala @@ -95,7 +95,7 @@ class V2ExpressionBuilder(e: Expression, isPredicate: Boolean = false) extends L expr: Expression, isPredicate: Boolean = false): Option[V2Expression] = expr match { case literal: Literal => Some(translateLiteral(literal)) case _ if expr.contextIndependentFoldable - && SQLConf.get.getConf(SQLConf.DATA_SOURCE_V2_EXPR_FOLDING) => + && SQLConf.get.getConfByKeyStrict[Boolean]("spark.sql.optimizer.datasourceV2ExprFolding") => // If the expression is context independent foldable, we can convert it to a literal. // This is useful for increasing the coverage of V2 expressions. val constantExpr = ConstantFolding.constantFolding(expr) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 77c69494a15a5..c6e0c9bc60bf7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -35,6 +35,8 @@ import org.apache.hadoop.mapreduce.OutputCommitter import org.slf4j.event.Level import org.apache.spark.{ErrorMessageFormat, SparkConf, SparkContext, SparkException, TaskContext} +import org.apache.spark.config.ConfigRegistry +import org.apache.spark.config.protobuf.Mutability import org.apache.spark.internal.Logging import org.apache.spark.internal.config.{ConfigBindingPolicy, _} import org.apache.spark.internal.io.FileCommitProtocol @@ -75,6 +77,9 @@ object SQLConf { private def register(entry: ConfigEntry[_]): Unit = sqlConfEntriesUpdateLock.synchronized { require(!sqlConfEntries.containsKey(entry.key), s"Duplicate SQLConfigEntry. ${entry.key} has been registered") + require(!ConfigRegistry.containsConfig(entry.key), + s"${entry.key} is defined in a .textproto config file. " + + s"Use buildConfFromConfigFile instead of buildConf.") val updatedMap = new java.util.HashMap[String, ConfigEntry[_]](sqlConfEntries) updatedMap.put(entry.key, entry) sqlConfEntries = updatedMap @@ -88,11 +93,28 @@ object SQLConf { } private[internal] def getConfigEntry(key: String): ConfigEntry[_] = { - sqlConfEntries.get(key) + Option(sqlConfEntries.get(key)).getOrElse(ConfigEntry.findProtoDefinedEntry(key)) } + // TODO: once all configs are migrated to textproto, this can be replaced by + // ConfigEntry.listAllEntries() and callers can filter by config properties. private[sql] def getConfigEntries(): util.Collection[ConfigEntry[_]] = { - sqlConfEntries.values() + // Lazy concatenating view - no intermediate allocation + new util.AbstractCollection[ConfigEntry[_]]() { + private lazy val protoConfigs = ConfigEntry.listAllProtoDefinedConfigs() + + override def iterator(): util.Iterator[ConfigEntry[_]] = { + val first = sqlConfEntries.values().iterator() + val second = protoConfigs.iterator() + new util.Iterator[ConfigEntry[_]]() { + override def hasNext: Boolean = first.hasNext || second.hasNext + override def next(): ConfigEntry[_] = + if (first.hasNext) first.next() else second.next() + } + } + + override def size(): Int = sqlConfEntries.size() + protoConfigs.size() + } } private[internal] def containsConfigEntry(entry: ConfigEntry[_]): Boolean = { @@ -100,7 +122,7 @@ object SQLConf { } private[sql] def containsConfigKey(key: String): Boolean = { - sqlConfEntries.containsKey(key) + sqlConfEntries.containsKey(key) || ConfigRegistry.containsConfig(key) } def registerStaticConfigKey(key: String): Unit = staticConfKeysUpdateLock.synchronized { @@ -109,10 +131,27 @@ object SQLConf { staticConfKeys = updated } - def isStaticConfigKey(key: String): Boolean = staticConfKeys.contains(key) + def isStaticConfigKey(key: String): Boolean = { + staticConfKeys.contains(key) || + Option(ConfigRegistry.getConfig(key)).exists(_.getMutability == Mutability.MUTABILITY_STATIC) + } def buildConf(key: String): ConfigBuilder = ConfigBuilder(key).onCreate(register) + def buildConfFromConfigFile[T](key: String): ProtoBackedConfigEntry[T] = { + Option(ConfigEntry.findProtoBackedEntry(key)).map(_.asInstanceOf[ProtoBackedConfigEntry[T]]) + .getOrElse { + if (ConfigRegistry.containsConfig(key)) { + // The key is defined in a .textproto file but has no default value, so it is backed by + // a ProtoBackedOptionalConfigEntry. Such configs must be accessed via getConfByKeyStrict. + throw SparkException.internalError( + s"Config entry $key has no default value; use getConfByKeyStrict to read it.") + } else { + throw SparkException.internalError(s"Config entry $key not found in ConfigRegistry") + } + } + } + def buildStaticConf(key: String): ConfigBuilder = { ConfigBuilder(key).onCreate { entry => SQLConf.registerStaticConfigKey(entry.key) @@ -129,7 +168,7 @@ object SQLConf { private[sql] def mergeNonStaticSQLConfigs( sqlConf: SQLConf, configs: Map[String, String]): Unit = { - for ((k, v) <- configs if !staticConfKeys.contains(k)) { + for ((k, v) <- configs if !isStaticConfigKey(k)) { sqlConf.setConfString(k, v) } } @@ -570,12 +609,8 @@ object SQLConf { .stringConf .createOptional - val OPTIMIZER_MAX_ITERATIONS = buildConf("spark.sql.optimizer.maxIterations") - .internal() - .doc("The max number of iterations the optimizer runs.") - .version("2.0.0") - .intConf - .createWithDefault(100) + val OPTIMIZER_MAX_ITERATIONS = + buildConfFromConfigFile[Int]("spark.sql.optimizer.maxIterations") val OPTIMIZER_INSET_CONVERSION_THRESHOLD = buildConf("spark.sql.optimizer.inSetConversionThreshold") @@ -1001,13 +1036,9 @@ object SQLConf { .bytesConf(ByteUnit.BYTE) .createWithDefaultString("10MB") - val SHUFFLE_HASH_JOIN_FACTOR = buildConf("spark.sql.shuffledHashJoinFactor") - .doc("The shuffle hash join can be selected if the data size of small" + - " side multiplied by this factor is still smaller than the large side.") - .version("3.3.0") - .intConf - .checkValue(_ >= 1, "The shuffle hash join factor cannot be negative.") - .createWithDefault(3) + val SHUFFLE_HASH_JOIN_FACTOR = + buildConfFromConfigFile[Int]("spark.sql.shuffledHashJoinFactor") + .checkValue(_ >= 1, "The shuffle hash join factor must be at least 1.") val LIMIT_INITIAL_NUM_PARTITIONS = buildConf("spark.sql.limit.initialNumPartitions") .internal() @@ -2217,15 +2248,6 @@ object SQLConf { .booleanConf .createWithDefault(false) - val DATA_SOURCE_V2_EXPR_FOLDING = - buildConf("spark.sql.optimizer.datasourceV2ExprFolding") - .internal() - .version("4.1.0") - .doc("When this config is set to true, do safe constant folding for the " + - "expressions before translation and pushdown.") - .booleanConf - .createWithDefault(true) - // This is used to set the default data source val DEFAULT_DATA_SOURCE_NAME = buildConf("spark.sql.sources.default") .doc("The default data source to use in input/output.") @@ -9475,6 +9497,14 @@ class SQLConf extends Serializable with Logging with SqlApiConf { } } + def getConfByKeyStrict[T](key: String): T = { + Option(ConfigEntry.findProtoDefinedEntry(key)) + .map(_.asInstanceOf[ConfigEntry[T]].readFrom(reader)) + .getOrElse { + throw SparkException.internalError(s"Config entry $key not found in ConfigRegistry") + } + } + private var definedConfsLoaded = false /** * Init [[StaticSQLConf]] and [[org.apache.spark.sql.hive.HiveUtils]] so that all the defined diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala index 658ab5f5312d5..72e36250b068a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala @@ -33,7 +33,7 @@ import org.apache.spark.util.Utils */ object StaticSQLConf { - import SQLConf.buildStaticConf + import SQLConf.{buildConfFromConfigFile, buildStaticConf} val WAREHOUSE_PATH = buildStaticConf("spark.sql.warehouse.dir") .doc("The default location for managed databases and tables.") @@ -198,11 +198,7 @@ object StaticSQLConf { .createOptional val UI_RETAINED_EXECUTIONS = - buildStaticConf("spark.sql.ui.retainedExecutions") - .doc("Number of executions to retain in the Spark UI.") - .version("1.5.0") - .intConf - .createWithDefault(1000) + buildConfFromConfigFile[Int]("spark.sql.ui.retainedExecutions") val SHUFFLE_EXCHANGE_MAX_THREAD_THRESHOLD = buildStaticConf("spark.sql.shuffleExchange.maxThreadThreshold") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala index 8c9e3eae49816..1ca88ce408c80 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala @@ -67,6 +67,8 @@ class RuntimeConfigSuite extends SparkFunSuite { // SQL configs assert(!conf.isModifiable(GLOBAL_TEMP_DATABASE.key)) assert(conf.isModifiable(CHECKPOINT_LOCATION.key)) + // Proto-backed cluster config is not modifiable + assert(!conf.isModifiable("spark.sql.ui.retainedExecutions")) // Core configs assert(!conf.isModifiable(config.CPUS_PER_TASK.key)) assert(!conf.isModifiable("spark.executor.cores")) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala index 0301c1d0f5baa..4af5a32515349 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala @@ -1026,7 +1026,7 @@ class DataSourceV2StrategySuite extends SharedSparkSession { val expr = Abs(Literal(-5), failOnError = true) checkV2Conversion(expr, LiteralValue(5, IntegerType)) - withSQLConf(SQLConf.DATA_SOURCE_V2_EXPR_FOLDING.key -> "false") { + withSQLConf("spark.sql.optimizer.datasourceV2ExprFolding" -> "false") { // when spark.sql.optimizer.datasourceV2ExprFolding = false // expression will be converted to V2 expressions, but not folded checkV2Conversion(expr, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala index 94d860174cece..f29d91ba7bdf4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path import org.apache.logging.log4j.Level import org.apache.spark.{SPARK_DOC_ROOT, SparkIllegalArgumentException, SparkNoSuchElementException} +import org.apache.spark.internal.config.{ConfigBuilder, ConfigEntry} import org.apache.spark.network.util.ByteUnit import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.catalyst.parser.ParseException @@ -183,6 +184,88 @@ class SQLConfSuite extends SharedSparkSession { } } + test("proto-backed config - OPTIMIZER_MAX_ITERATIONS") { + // Verify that the config is loaded from the prototext file with correct properties + val config = SQLConf.OPTIMIZER_MAX_ITERATIONS + assert(config.key === "spark.sql.optimizer.maxIterations") + assert(config.defaultValueString === "100") + assert(config.doc === "The max number of iterations the optimizer runs.") + assert(config.version === "2.0.0") + // This config is internal (not public) + assert(!config.isPublic) + + // Verify the config works correctly at runtime + sqlConf.clear() + assert(sqlConf.getConf(config) === 100) + sql(s"set ${config.key}=50") + assert(sqlConf.getConf(config) === 50) + sqlConf.clear() + } + + test("proto-backed config cannot be replaced by an ordinary config entry") { + val key = SQLConf.OPTIMIZER_MAX_ITERATIONS.key + val original = ConfigEntry.findEntry(key) + + intercept[IllegalArgumentException] { + ConfigBuilder(key).intConf.createWithDefault(1) + } + assert(ConfigEntry.findEntry(key) eq original) + } + + test("proto-backed config with checkValue - SHUFFLE_HASH_JOIN_FACTOR") { + val config = SQLConf.SHUFFLE_HASH_JOIN_FACTOR + assert(config.key === "spark.sql.shuffledHashJoinFactor") + assert(config.defaultValueString === "3") + assert(config.version === "3.3.0") + assert(config.isPublic) + + // Verify the config works correctly at runtime + sqlConf.clear() + assert(sqlConf.getConf(config) === 3) + sql(s"set ${config.key}=5") + assert(sqlConf.getConf(config) === 5) + + // Verify the checkValue validation works - value must be >= 1 + checkError( + exception = intercept[SparkIllegalArgumentException] { + spark.conf.set(config.key, 0) + }, + condition = "INVALID_CONF_VALUE.REQUIREMENT", + parameters = Map( + "confName" -> config.key, + "confValue" -> "0", + "confRequirement" -> "The shuffle hash join factor must be at least 1.")) + + sqlConf.clear() + } + + test("getConfByKeyStrict - read proto-backed config by key") { + sqlConf.clear() + + // Test reading default value + assert(sqlConf.getConfByKeyStrict[Int]("spark.sql.optimizer.maxIterations") === 100) + assert(sqlConf.getConfByKeyStrict[Boolean]( + "spark.sql.optimizer.datasourceV2ExprFolding") === true) + + // Test reading configured value + sql("set spark.sql.optimizer.maxIterations=50") + assert(sqlConf.getConfByKeyStrict[Int]("spark.sql.optimizer.maxIterations") === 50) + + // Test that non-existent key throws error + val e = intercept[Exception] { + sqlConf.getConfByKeyStrict[Int]("spark.nonexistent.config") + } + assert(e.getMessage.contains("not found in ConfigRegistry")) + + // Test that type mismatch throws ClassCastException + // spark.sql.optimizer.maxIterations is an INT config, reading as Boolean should fail + intercept[ClassCastException] { + sqlConf.getConfByKeyStrict[Boolean]("spark.sql.optimizer.maxIterations") + } + + sqlConf.clear() + } + test("reset - user-defined conf") { sqlConf.clear() val userDefinedConf = "x.y.z.reset" diff --git a/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions b/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions index 75a905223e67a..37598f184c021 100644 --- a/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions +++ b/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions @@ -786,7 +786,6 @@ spark.sql.optimizer.avoidCollapseUDFWithExpensiveExpr spark.sql.optimizer.avoidDoubleFilterEval spark.sql.optimizer.canChangeCachedPlanOutputPartitioning spark.sql.optimizer.collapseProjectAlwaysInline -spark.sql.optimizer.datasourceV2ExprFolding spark.sql.optimizer.datasourceV2JoinPushdown spark.sql.optimizer.decorrelateExistsSubqueryLegacyIncorrectCountHandling.enabled spark.sql.optimizer.decorrelateInnerQuery.enabled @@ -811,7 +810,6 @@ spark.sql.optimizer.expression.nestedPruning.enabled spark.sql.optimizer.expressionProjectionCandidateLimit spark.sql.optimizer.inSetConversionThreshold spark.sql.optimizer.inSetSwitchThreshold -spark.sql.optimizer.maxIterations spark.sql.optimizer.metadataOnly spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources spark.sql.optimizer.nestedSchemaPruning.enabled @@ -947,7 +945,6 @@ spark.sql.shuffle.partitions spark.sql.shuffleDependency.fileCleanup.enabled spark.sql.shuffleDependency.skipMigration.enabled spark.sql.shuffleExchange.maxThreadThreshold -spark.sql.shuffledHashJoinFactor spark.sql.sort.enableRadixSort spark.sql.sortMergeJoinExec.buffer.in.memory.threshold spark.sql.sortMergeJoinExec.buffer.spill.size.threshold @@ -1103,7 +1100,6 @@ spark.sql.transposeMaxValues spark.sql.truncateTable.ignorePermissionAcl.enabled spark.sql.tvf.allowMultipleTableArguments.enabled spark.sql.ui.explainMode -spark.sql.ui.retainedExecutions spark.sql.unionOutputPartitioning spark.sql.useCommonExprIdForAlias spark.sql.variable.substitute From 90b1f8984ea8868bb1ea3edb9c66382de2930495 Mon Sep 17 00:00:00 2001 From: Andreas Neumann Date: Mon, 27 Jul 2026 09:03:05 -0700 Subject: [PATCH 07/11] [SPARK-58313][SDP] Validate SCD2 track-history columns at AutoCDC flow construction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What changes were proposed in this pull request? This PR replaces https://github.com/apache/spark/pull/57490, which had to be abandoned due to severe merge conflicts. It is an exact cherry-pick of the previous commits, plus a fix for the test cases that failed due to the merge conflicts. An SCD2 AutoCDC flow can restrict which columns define a "run" via TRACK HISTORY ON (...), which populate ChangeArgs.trackHistorySelection. Until now, that selection was only resolved when the first microbatch ran, inside Scd2BatchProcessor.computeTrackedHistoryColumns during reconciliation. An unresolvable or ineligible tracking column — one that is absent from the source, is a key, is a reserved framework column, or was dropped by the flow's column_list — therefore surfaced mid-stream rather than at flow construction, unlike every other AutoCDC misconfiguration (keys, column selection, reserved names), which fail eagerly. This PR validates trackHistorySelection at AutoCdcMergeFlow construction time, mirroring the existing requireKeysPresentInSelectedSchema check: - The eligibility + resolution logic is extracted from Scd2BatchProcessor.computeTrackedHistoryColumns into a schema-based companion helper Scd2BatchProcessor.computeTrackedHistoryColumns(schema, changeArgs, caseSensitive). Both the per-microbatch runtime path and the new construction-time validator call it, so the two can never diverge. The refactor is behavior-preserving. - AutoCdcMergeFlow gains requireTrackHistoryColumnsResolvableInSelectedSchema, invoked when deriving the user-selected schema (right after the key-presence check). It runs before the flow's schema is forced, so the actionable error surfaces ahead of the temporary AUTOCDC_SCD2_NOT_SUPPORTED gate and remains correct once SCD2 support lands. - No new error condition: an unresolvable selection reuses the existing AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA (schema name trackHistorySelection). - The check is a no-op when trackHistorySelection is None, which covers all SCD1 flows (enforced by ChangeArgs) and SCD2 flows that do not restrict tracking. ### Why are the changes needed? Deferring this validation to reconciliation means a simple typo or misconfiguration (TRACK HISTORY ON (typo), or tracking a key/excluded column) is not reported at graph analysis time; it only fails once data flows, with an error raised deep in the SCD2 batch processor. Validating at flow construction gives a fail-fast, user-actionable error consistent with the rest of the AutoCDC configuration surface (keys, column_list, reserved names). ### Does this PR introduce any user-facing change? Yes. An AutoCDC SCD2 flow whose TRACK HISTORY ON (...) references a column that is not an eligible history-tracking column (absent, a key, a framework column, or excluded by column_list) now fails at flow construction with AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA instead of failing when the first microbatch runs. Valid selections are unaffected, and there is no change for SCD1 flows. (Note: SCD2 AutoCDC flows are not yet generally supported on master — still gated by AUTOCDC_SCD2_NOT_SUPPORTED — so no released behavior changes.) ### How was this patch tested? New unit tests in AutoCdcFlowSuite covering: an SCD2 flow tracking a non-existent column, a key column (ineligible), and a column dropped by columnSelection are each rejected at construction with AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA; a resolvable selection passes the check (falling through to the SCD2-not-supported gate); and case-sensitive/insensitive resolution behavior. ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 4.8) Closes #57524 from anew/spark-58313-track-history-v2. Authored-by: Andreas Neumann Signed-off-by: Jose Torres --- .../autocdc/Scd2BatchProcessor.scala | 66 +++++--- .../spark/sql/pipelines/graph/Flow.scala | 25 +++ .../pipelines/autocdc/AutoCdcFlowSuite.scala | 153 +++++++++++++++++- 3 files changed, 220 insertions(+), 24 deletions(-) diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala index 157d2d5788b6d..fbf9bc15127ec 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala @@ -21,6 +21,7 @@ import org.apache.spark.SparkException import org.apache.spark.sql.{functions => F} import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.{caseInsensitiveResolution, caseSensitiveResolution} import org.apache.spark.sql.catalyst.expressions.{CreateMap, If, Literal, RaiseError} import org.apache.spark.sql.catalyst.util.QuotingUtils import org.apache.spark.sql.classic.{DataFrame, ExpressionUtils} @@ -922,28 +923,12 @@ case class Scd2BatchProcessor( * the eligible user-data columns (those not in [[ChangeArgs.keys]] or the framework * reserved set) filtered through [[ChangeArgs.trackHistorySelection]]. */ - private def computeTrackedHistoryColumns(df: DataFrame): Seq[String] = { - val conf = df.sparkSession.sessionState.conf - val resolver = conf.resolver - - val keyColNames = changeArgs.keys.map(_.name) - val reservedColNames = Scd2BatchProcessor.reservedFrameworkColNames - - val eligibleSchema = StructType(df.schema.fields.filterNot { field => - reservedColNames.exists(resolver(_, field.name)) || - keyColNames.exists(resolver(_, field.name)) - }) - - ColumnSelection - .applyToSchema( - schemaName = "trackHistorySelection", - schema = eligibleSchema, - columnSelection = changeArgs.trackHistorySelection, - caseSensitive = conf.caseSensitiveAnalysis - ) - .fieldNames - .toImmutableArraySeq - } + private def computeTrackedHistoryColumns(df: DataFrame): Seq[String] = + Scd2BatchProcessor.computeTrackedHistoryColumns( + schema = df.schema, + changeArgs = changeArgs, + caseSensitive = df.sparkSession.sessionState.conf.caseSensitiveAnalysis + ) /** * Tag each post-reconciliation row with [[Scd2BatchProcessor.shouldRouteToAuxTableColName]]: @@ -1410,6 +1395,43 @@ object Scd2BatchProcessor { AutoCdcReservedNames.cdcMetadataColName ) + /** + * Resolve [[ChangeArgs.trackHistorySelection]] against `schema` and return the field names of + * the history-tracking columns: the eligible user-data columns (those that are neither + * [[ChangeArgs.keys]] nor framework reserved columns) filtered through the selection. + * + * This is the single source of truth for which columns define an SCD2 run. It is called both + * per-microbatch (against the reconciled dataframe's schema) and at + * [[org.apache.spark.sql.pipelines.graph.AutoCdcMergeFlow]] construction time (against the + * user-selected source schema), so an unresolvable or ineligible selection fails fast with a + * user-actionable [[org.apache.spark.sql.AnalysisException]] instead of surfacing mid-stream. + * + * Throws `AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA` if the selection references a column that is not + * an eligible history-tracking column in `schema` (i.e. absent, or a key/framework column). + */ + private[pipelines] def computeTrackedHistoryColumns( + schema: StructType, + changeArgs: ChangeArgs, + caseSensitive: Boolean): Seq[String] = { + val resolver = if (caseSensitive) caseSensitiveResolution else caseInsensitiveResolution + val keyColNames = changeArgs.keys.map(_.name) + + val eligibleSchema = StructType(schema.fields.filterNot { field => + reservedFrameworkColNames.exists(resolver(_, field.name)) || + keyColNames.exists(resolver(_, field.name)) + }) + + ColumnSelection + .applyToSchema( + schemaName = "trackHistorySelection", + schema = eligibleSchema, + columnSelection = changeArgs.trackHistorySelection, + caseSensitive = caseSensitive + ) + .fieldNames + .toImmutableArraySeq + } + /** * Name of temporary column projected onto microbatch to compute the min sequencing value per * key within the microbatch. diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala index 31444f0c15f3d..410b0486c787b 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala @@ -267,6 +267,9 @@ class AutoCdcMergeFlow( // AutoCDC flows require all key columns to be present in the user-selected source schema, // so that they survive into the target table where SCD reconciliation needs them. requireKeysPresentInSelectedSchema(selectedSchema) + // SCD2 flows may specify history-tracking columns; validate they resolve to eligible columns + // of the selected schema at construction time, rather than failing mid-stream on first batch. + requireTrackHistoryColumnsResolvableInSelectedSchema(selectedSchema) selectedSchema } @@ -417,4 +420,26 @@ class AutoCdcMergeFlow( ) } } + + /** + * Validate that this flow's [[ChangeArgs.trackHistorySelection]] (SCD2 `TRACK HISTORY ON ...`) + * resolves against the user-selected source schema at construction time. Without this, an + * unresolvable or ineligible (key/framework) tracking column would only surface when the first + * microbatch runs reconciliation, deep inside the SCD2 batch processor. + * + * Delegates to [[Scd2BatchProcessor.computeTrackedHistoryColumns]] -- the same resolution used at + * runtime -- so the two can never diverge; it throws `AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA` on an + * unresolvable selection. `trackHistorySelection` is `None` for SCD1 (enforced by [[ChangeArgs]]) + * and for SCD2 flows that do not restrict tracking, in which case resolution is a no-op. + */ + private def requireTrackHistoryColumnsResolvableInSelectedSchema( + selectedSchema: StructType): Unit = { + if (changeArgs.trackHistorySelection.isDefined) { + Scd2BatchProcessor.computeTrackedHistoryColumns( + schema = selectedSchema, + changeArgs = changeArgs, + caseSensitive = spark.sessionState.conf.caseSensitiveAnalysis + ) + } + } } diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala index 98c31a3020ef5..5769b264993bf 100644 --- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala +++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala @@ -165,13 +165,15 @@ class AutoCdcFlowSuite extends QueryTest with SharedSparkSession { keys: Seq[UnqualifiedColumnName] = Seq(UnqualifiedColumnName("id")), sequencing: Column = F.col("seq"), storedAsScdType: ScdType = ScdType.Type1, - columnSelection: Option[ColumnSelection] = None): AutoCdcMergeFlow = { + columnSelection: Option[ColumnSelection] = None, + trackHistorySelection: Option[ColumnSelection] = None): AutoCdcMergeFlow = { val flow = newAutoCdcFlow( changeArgs = ChangeArgs( keys = keys, sequencing = sequencing, storedAsScdType = storedAsScdType, - columnSelection = columnSelection + columnSelection = columnSelection, + trackHistorySelection = trackHistorySelection ) ) new AutoCdcMergeFlow(flow, successfulFuncResult(sourceDf)) @@ -767,4 +769,151 @@ class AutoCdcFlowSuite extends QueryTest with SharedSparkSession { ) ) } + + // =========================================================================================== + // AutoCdcMergeFlow track-history validation tests + // + // SCD2 `TRACK HISTORY ON (...)` populates trackHistorySelection. These tests lock in that an + // unresolvable or ineligible (key / dropped-by-column-selection) tracking column is rejected at + // flow construction rather than deferring to the first microbatch's reconciliation, mirroring + // the keys-presence validator above. A resolvable selection passes the check and the flow + // constructs successfully. + // =========================================================================================== + + test( + "an SCD2 flow tracking a non-existent column is rejected at construction" + ) { + // Eligible tracking columns from the 3-column source (id, name, seq), less the key `id`, are + // {name, seq}. `missing` is absent, so resolution against trackHistorySelection fails. + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("missing"))) + ) + ) + }, + condition = "AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "schemaName" -> "trackHistorySelection", + "missingColumns" -> "missing", + "availableColumns" -> "name, seq" + ) + ) + } + + test( + "an SCD2 flow tracking a key column is rejected at construction (ineligible)" + ) { + // A key is never an eligible history-tracking column, so it is absent from the eligible + // schema {name, seq} and resolution fails -- surfacing the misconfiguration eagerly. + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("id"))) + ) + ) + }, + condition = "AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "schemaName" -> "trackHistorySelection", + "missingColumns" -> "id", + "availableColumns" -> "name, seq" + ) + ) + } + + test( + "an SCD2 flow tracking a column dropped by columnSelection is rejected" + ) { + // `name` exists in the source but is excluded from the selected schema, so it is not an + // eligible tracking column. Eligible columns are then just {seq}. + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + columnSelection = Some( + ColumnSelection.ExcludeColumns(Seq(UnqualifiedColumnName("name"))) + ), + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("name"))) + ) + ) + }, + condition = "AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "schemaName" -> "trackHistorySelection", + "missingColumns" -> "name", + "availableColumns" -> "seq" + ) + ) + } + + test( + "an SCD2 flow with a resolvable track-history selection passes the check" + ) { + // `name` is an eligible tracking column, so the construction-time check passes and the flow + // constructs successfully, resolving its SCD2 schema (no AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA). + val resolvedFlow = newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("name"))) + ) + ) + assert(resolvedFlow.schema.fieldNames.contains(AutoCdcReservedNames.cdcMetadataColName)) + } + + test( + "track-history validation respects case-insensitive analysis" + ) { + // With caseSensitive=false, `NAME` resolves to the eligible `name`, so the check passes and + // the flow constructs successfully. + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { + val resolvedFlow = newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("NAME"))) + ) + ) + assert(resolvedFlow.schema.fieldNames.contains(AutoCdcReservedNames.cdcMetadataColName)) + } + } + + test( + "track-history validation respects case-sensitive analysis" + ) { + // With caseSensitive=true, `NAME` is a distinct identifier from the eligible `name` and does + // not resolve, so the construction-time check rejects it. + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("NAME"))) + ) + ) + }, + condition = "AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseSensitive, + "schemaName" -> "trackHistorySelection", + "missingColumns" -> "NAME", + "availableColumns" -> "name, seq" + ) + ) + } + } } From adad83831f22076a855456aeae7ea62b3895e851 Mon Sep 17 00:00:00 2001 From: Qiegang Long Date: Tue, 28 Jul 2026 00:04:08 +0800 Subject: [PATCH 08/11] [SPARK-58089][SQL] Push variant extractions through Aggregate/Sort/Join MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What changes were proposed in this pull request? Introduce a new optimizer rule `PullOutVariantExtractions` that hoists `variant_get` / `Cast(variant)` extractions out of three operator types that the existing `PushVariantIntoScan` / `V2ScanRelationPushDown` rules cannot see through: - **Aggregate function arguments** – e.g. `max(variant_get(v, '$.price', 'int'))`: the extraction is moved into a `Project` directly below the `Aggregate` and the aggregate references the resulting alias. The bare variant column is suppressed unless it is also needed raw (e.g. as a `GROUP BY` key), so no redundant full-variant slot is generated. - **Sort order keys** – e.g. `ORDER BY variant_get(v, '$.price', 'int')`: matched as `Project → Sort`; the extraction is hoisted below the `Sort` and the original `Project` is reproduced to prevent the alias from leaking into the output. - **Join conditions and projections above joins** – matched as `Project → Join`; extractions in both the join condition and the outer `Project` are routed to the owning join side. A `pushSideAliases` helper then pushes the aliases *through* any depth of chained joins so they land in a `Project` directly above the scan (where `PhysicalOperation` collapses them with the scan, making them visible to the pushdown). This is necessary because `PhysicalOperation` stops at a `Join` node. A `Sort` sitting over a `Join` is handled by fusing the two cases: the order-key aliases are pushed through the join tree, not left in a `Project` above it. The rule is gated by a new internal config `spark.sql.variant.pushVariantIntoScan.pullOutExtractions` (default `true`) and is a no-op unless `spark.sql.variant.pushVariantIntoScan` is also enabled. Non-variant plans are untouched. The rule is registered as the first rule in `SparkOptimizer.earlyScanPushDownRules`, before `SchemaPruning` and the V2 scan pushdown rules. ### Why are the changes needed? Before this change, a `variant_get` inside an aggregate function argument, sort key, or join condition caused the whole variant column to be read raw (or shredded with a redundant full-variant slot). For example: ```sql SELECT name, max(variant_get(v, '$.price', 'int')) FROM T GROUP BY name ``` read the entire v column even though only the price field was needed. After this change, Spark shreds only the requested typed fields, avoiding the full-variant I/O. The change improves query performance for variant referenced in aggregrate, join, sort. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Added new units. Also run correctness tests against some known workload. #### Performance result Test framework: https://github.com/cloudera-labs/variant-conformance-benchmark Dataset: tpc-ds dataset (SF=5), spark native parquet table, payload in variant or json. Run setup: pre-warm jvm, three runs, median query timing reported by spark. Run A with pullout rule enabled vs B with the rule disabled. ``` Compare: tpcds-flat-pullout-20260714 vs tpcds-flat-no-pullout-20260714 (metric: query_median) Run A: /Users/qlong/opensources/variant-conformance-benchmark/results/tpcds-flat-pullout-20260714/tpcds-flat/timings-variant.csv Run B: /Users/qlong/opensources/variant-conformance-benchmark/results/tpcds-flat-no-pullout-20260714/tpcds-flat/timings-variant.csv query median_A median_B delta_s delta_% ------ -------- -------- ------- ------- q07 1.40 3.28 -1.88 -57.2% q12 0.07 0.07 -0.00 -2.8% q19 0.06 0.07 -0.01 -9.9% q26 1.14 3.40 -2.26 -66.4% q42 0.56 2.59 -2.03 -78.2% q52 0.61 3.04 -2.43 -80.0% q55 0.54 2.68 -2.14 -79.9% q63 0.58 2.95 -2.37 -80.4% q68 1.00 3.42 -2.42 -70.8% q73 0.57 2.88 -2.31 -80.1% q79 0.85 3.26 -2.41 -73.9% q98 0.65 3.32 -2.67 -80.5% Summary: 12 queries, 12 comparable Geo-mean delta: -69.5% (Run A faster) Total (query_median): 8.0s vs 31.0s ``` Run A with pullout rule enabled vs B using json for payload, this test shows the **performance advantage of variant over json**. ``` Compare: tpcds-flat-pullout-20260714 vs tpcds-flat-pullout-20260714 (metric: query_median) Run A: /Users/qlong/opensources/variant-conformance-benchmark/results/tpcds-flat-pullout-20260714/tpcds-flat/timings-variant.csv Run B: /Users/qlong/opensources/variant-conformance-benchmark/results/tpcds-flat-pullout-20260714/tpcds-flat/timings-string_json.csv query median_A median_B delta_s delta_% ------ -------- -------- ------- ------- q07 1.40 3.26 -1.85 -56.9% q12 0.07 0.13 -0.06 -47.0% q19 0.06 0.23 -0.17 -72.2% q26 1.14 2.17 -1.03 -47.5% q42 0.56 0.56 +0.01 +1.1% q52 0.61 1.09 -0.48 -44.1% q55 0.54 0.47 +0.07 +15.2% q63 0.58 0.88 -0.30 -34.2% q68 1.00 1.34 -0.34 -25.1% q73 0.57 0.70 -0.13 -18.0% q79 0.85 1.62 -0.77 -47.4% q98 0.65 1.19 -0.54 -45.4% Summary: 12 queries, 12 comparable Geo-mean delta: -39.3% (Run A faster) Total (query_median): 8.0s vs 13.6s ``` ### Was this patch authored or co-authored using generative AI tooling? Co-authored with Claude Code Closes #57190 from qlong/variant-pushout-aggregate. Authored-by: Qiegang Long Signed-off-by: Wenchen Fan --- .../apache/spark/sql/internal/SQLConf.scala | 13 + .../spark/sql/execution/SparkOptimizer.scala | 6 +- .../PullOutVariantExtractions.scala | 487 ++++++++ .../PushVariantIntoScanSuite.scala | 1039 ++++++++++++++++- 4 files changed, 1511 insertions(+), 34 deletions(-) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PullOutVariantExtractions.scala diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index c6e0c9bc60bf7..97594802e258d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -6697,6 +6697,19 @@ object SQLConf { .booleanConf .createWithDefault(true) + val PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS = + buildConf("spark.sql.variant.pushVariantIntoScan.pullOutExtractions") + .internal() + .doc("When true, extend variant field pushdown to extractions used inside aggregate " + + "functions, join conditions, sort keys, and projections above joins (including through " + + "chained joins), so that only the requested fields are read from the scan for these " + + "cases too instead of the whole variant column. Has no effect unless " + + "spark.sql.variant.pushVariantIntoScan is also true.") + .version("4.3.0") + .withBindingPolicy(ConfigBindingPolicy.SESSION) + .booleanConf + .createWithDefault(true) + val PUSH_VARIANT_INTO_SCAN_DEFER_CAST_ERROR = buildConf("spark.sql.variant.pushVariantIntoScan.deferCastError") .internal() diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala index 54158d5bb4a94..0994220afe0c1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.optimizer._ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.connector.catalog.CatalogManager -import org.apache.spark.sql.execution.datasources.{MarkSingleTaskExecution, PruneFileSourcePartitions, PushVariantIntoScan, SchemaPruning, V1Writes} +import org.apache.spark.sql.execution.datasources.{MarkSingleTaskExecution, PruneFileSourcePartitions, PullOutVariantExtractions, PushVariantIntoScan, SchemaPruning, V1Writes} import org.apache.spark.sql.execution.datasources.v2.{GroupBasedRowLevelOperationScanPlanning, OptimizeMetadataOnlyDeleteFromTable, V2ScanPartitioningAndOrdering, V2ScanRelationPushDown, V2Writes} import org.apache.spark.sql.execution.dynamicpruning.{CleanupDynamicPruningFilters, PartitionPruning, RowLevelOperationRuntimeGroupFiltering} import org.apache.spark.sql.execution.python.{ExtractGroupingPythonUDFFromAggregate, ExtractPythonUDFFromAggregate, ExtractPythonUDFs, ExtractPythonUDTFs} @@ -37,6 +37,10 @@ class SparkOptimizer( override def earlyScanPushDownRules: Seq[Rule[LogicalPlan]] = // TODO: move SchemaPruning into catalyst Seq( + // Hoist variant extractions out of operators that variant-into-scan pushdown cannot see + // through (aggregate function arguments, etc.) into a Project above the scan, so the + // extractions below become visible to V2ScanRelationPushDown and PushVariantIntoScan. + PullOutVariantExtractions, SchemaPruning, GroupBasedRowLevelOperationScanPlanning, V1Writes, diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PullOutVariantExtractions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PullOutVariantExtractions.scala new file mode 100644 index 0000000000000..6695e24ad6186 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PullOutVariantExtractions.scala @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources + +import scala.collection.mutable + +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeSet, Cast, Expression, + NamedExpression, SortOrder} +import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression +import org.apache.spark.sql.catalyst.expressions.variant.VariantGet +import org.apache.spark.sql.catalyst.plans.LeftExistence +import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Join, LogicalPlan, Project, Sort} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.catalyst.trees.TreePattern.{AGGREGATE, JOIN, SORT} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.VariantType + +/** + * Hoists variant extractions out of operators that variant-into-scan pushdown cannot see through, + * into a [[Project]] directly below that operator, so the extraction becomes visible to the + * pushdown rules ([[PushVariantIntoScan]] for v1 and + * [[org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown]] for v2). The rule + * applies uniformly to all three table read paths: DS1 (e.g. `spark.read.parquet`), DS2 (e.g. + * `spark.read.parquet` with `spark.sql.sources.useV1SourceList` cleared), and Spark native + * catalog tables (managed or external tables created with `CREATE TABLE ... USING PARQUET`). In + * every case the downstream pushdown rule sees the hoisted extraction in the + * `Project`/`Filter` chain that [[org.apache.spark.sql.catalyst.planning.PhysicalOperation]] + * collapses above the scan; an extraction embedded in an `Aggregate`/`Join`/`Sort` is invisible + * to them and the whole variant column is read raw (or, when the column is also read raw for + * pass-through, shredded with a wasteful full-variant slot). + * + * This complements the rules that already relocate variant extractions into such a `Project`: + * `PhysicalOperation` (Project/Filter), `PullOutGroupingExpressions` (GROUP BY / DISTINCT keys), + * and `ExtractWindowExpressions` (window partition/order keys and window function arguments). The + * residual, un-relocated extractions live in aggregate function arguments, join conditions, and + * sort orders. This rule handles those three: + * + * - Aggregate: `Aggregate([name], [name, max(variant_get(v, '$.p'))], child)` becomes + * `Aggregate([name], [name, max(_ve0)], Project([name, variant_get(v, '$.p') AS _ve0], child))`. + * The Aggregate defines its own output, so the raw `v` is simply not passed through. + * + * - Sort / Join: these operators pass their child's columns through, so a naive hoist would keep + * a bare `v` alongside the hoisted extraction and the pushdown would still request the full + * variant. To avoid that we match the `Project` sitting above the barrier -- its `references` + * are exactly the columns still live above the barrier -- and drop a variant column that is no + * longer referenced once its extraction has been hoisted. A variant genuinely needed raw above + * the barrier stays in `references` and is preserved (keeping its full-variant slot). + * + * A `Join`, and a `Sort` directly under a narrowing `Project`, are handled with that Project's + * `references` telling us which columns remain live: a variant used only via the hoisted + * extraction is dropped (no redundant full-variant slot). This is the common `Sort` shape when + * only a subset of columns is selected (e.g. `SELECT name ... ORDER BY variant_get(v, ...)`), + * where the analyzer adds a `Project` above the `Sort` to project away the order-only columns. + * + * A bare/root `Sort` -- one with no narrowing `Project` above it, as in + * `SELECT * FROM t ORDER BY variant_get(v, '$.b')` (all order columns are already selected, so + * the analyzer adds no narrowing `Project`) -- is also handled, by a second `transformUp` pass + * (see `apply`). Its output must equal its child's output exactly, so ALL child columns stay + * live: a variant present in the output stays raw AND gains an extra `_ve` slot, producing a + * multi-slot shredded struct with a full-variant slot -- the same shape as a `Sort`-under- + * `Project` whose `v` is also selected. `Sort`s under a `GlobalLimit`/`LocalLimit` (a + * `SELECT * ... ORDER BY ... LIMIT n`) are covered by the same second pass. A `Sort`/`Join` in + * any other shape -- under a `Filter`/`Aggregate`/another non-`Project` operator that is not a + * limit wrapper -- is left unchanged: the extraction stays in place, the variant is read raw, + * and the plan is correct (just not shredded). When the `Sort` itself sits over a `Join`, the + * hoisted order-key aliases are additionally pushed *through* the join to the scan side (see the + * through-a-`Join` note below), not merely parked in a `Project` above it. + * + * Pushing extractions *through* a `Join`: hoisting an extraction into a `Project`/child directly + * above a `Join` is not enough on its own, because `PhysicalOperation` (which both pushdown rules + * rely on) collapses only a contiguous `Project`/`Filter` chain and stops at the `Join`. A variant + * column `v` that reaches the barrier only via a hoisted `variant_get` would still flow up through + * the join as a bare attribute and be read raw at the scan. So once an extraction is hoisted above + * a `Join` (by the `Aggregate` or `Sort` case, or by the projection/condition handling of the + * `Project`-over-`Join` case), `pushSideAliases` pushes the resulting `_ve` aliases *down* through + * the join tree, routing each to the side whose output owns the referenced attribute, until it + * lands in a `Project` directly above a non-join child (the scan side). This descends any depth of + * chained joins in a single pass, so a variant fact-table column joined to several dimensions + * shreds to just the requested typed fields. Correctness is per join type: `Inner`/`Cross` and the + * outer joins (`LeftOuter`/`RightOuter`/`FullOuter`) push to either side -- the outer nullable side + * is value-preserving because `variant_get(NULL) = NULL` and null-padding commutes with the + * extraction. `LeftSemi`/`LeftAnti`/`ExistenceJoin` push only left-side aliases (their right side + * is not in the join output); a right-side alias on such a join is not pushed and the parent keeps + * a `Project` referencing it above the join. A variant still needed raw above the join stays live + * and keeps its full-variant slot, exactly as in the non-join cases. + * + * Every rewrite is a plain alias/`Project` introduction and is semantics-preserving; the pushdown + * shred/rewrite machinery is unchanged. The rule is gated on + * [[SQLConf.PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS]] (and is a no-op unless + * [[SQLConf.PUSH_VARIANT_INTO_SCAN]] is also enabled) and only fires when a hoistable extraction is + * present, so non-variant plans are untouched. + * + * Cast-error surface: relocating a strict extraction (`variant_get(..., failOnError = true)` or a + * strict `Cast`) below a `Join` means it is evaluated at the scan on rows the join later eliminates + * -- so a cast failure can surface for a row the un-hoisted plan would never have cast (here the + * eliminating rows come from the *other* joined table). This is the same pre-existing trade-off as + * [[PushVariantIntoScan]] pushing casts below a `Filter`, not a new error class: in both, the + * strict cast runs before the operator that would have discarded the failing row. This rule only + * relocates the extraction into a `Project`; [[PushVariantIntoScan]] still does the scan-level + * materialization and, when [[SQLConf.PUSH_VARIANT_INTO_SCAN_DEFER_CAST_ERROR]] is set (default + * false), wraps the cast with a per-row cast-error companion slot so the error is only raised when + * the original expression consumes the failing row. That deferral is provenance-agnostic -- it acts + * on the relocated extraction regardless of how it reached the `Project` -- so enabling the flag + * suppresses the join-eliminated-row error exactly as it does the filter-eliminated-row case. With + * the flag off (the default), the strict cast raises immediately on any failing scanned row, as + * documented for below-`Filter` pushdown. + */ +object PullOutVariantExtractions extends Rule[LogicalPlan] { + + override def apply(plan: LogicalPlan): LogicalPlan = { + // Hoisting is only useful when variant-into-scan pushdown is enabled to consume the relocated + // extractions, and is independently gated so it can be turned off on its own. + if (!SQLConf.get.getConf(SQLConf.PUSH_VARIANT_INTO_SCAN) || + !SQLConf.get.getConf(SQLConf.PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS)) { + plan + } else { + // First pass: handle operators where a narrowing Project above the barrier tells us which + // columns remain live. `transformUp` so inner operators are rewritten before their parents. + // Sort/Join are matched together with the Project above them (see class doc). Prune to + // subtrees that actually contain one of the matched operators so plans with none (and the + // Once-batch idempotence re-run) skip the traversal. + val afterProjectCases = plan.transformUpWithPruning( + _.containsAnyPattern(AGGREGATE, SORT, JOIN)) { + case agg: Aggregate => rewriteAggregate(agg) + case p @ Project(projectList, s: Sort) => rewriteSortUnderProject(p, projectList, s) + case p @ Project(projectList, j: Join) => rewriteJoinUnderProject(p, projectList, j) + } + // Second pass: handle a Sort NOT under a narrowing Project -- a bare/root Sort (as in + // `SELECT * ORDER BY variant_get(...)`), or one under a GlobalLimit/LocalLimit. This must be + // a separate pass, not another arm in the first `transformUp`: that traversal is bottom-up, + // so a bare-Sort arm would fire on the inner Sort of a `Project(_, Sort)` shape before the + // parent Project is visited, keeping v raw and pre-empting `rewriteSortUnderProject`. After + // the first pass, any Sort that was under a Project has its order rewritten to non-hoistable + // `_ve` attribute references, so `rewriteBareSort` is a no-op on it (idempotent). Prune to + // Sort-bearing subtrees so Sort-free plans skip this pass entirely. + afterProjectCases.transformUpWithPruning(_.containsPattern(SORT)) { + case s: Sort => rewriteBareSort(s) + } + } + } + + // A variant column path: an `Attribute` (possibly through `GetStructField`) typed `VariantType`. + // Used by `isHoistable` to verify the extraction's source is a scan column, not a computed + // expression. Non-column variants (e.g. `variant_get(parse_json(x), ...)`) are excluded. + private def isVariantColumnPath(e: Expression): Boolean = + e.dataType.isInstanceOf[VariantType] && StructPath.unapply(e).isDefined + + private def isHoistable(e: Expression): Boolean = e match { + // Exclude the two-arg form variant_get(v, path) whose targetType is VariantType: it produces a + // variant-typed alias that the pushdown drops (see SPARK-57499), so hoisting adds a Project for + // no benefit. It would also break Once-batch idempotence: the hoisted `_ve0` is a VariantType + // attribute, so a wrapping `cast(_ve0 as string)` (e.g. from `v:a::string`, which desugars to + // `cast(variant_get(v, '$.a') as string)`) would match the Cast branch below on the next pass. + case g: VariantGet => + !g.targetType.isInstanceOf[VariantType] && g.path.foldable && isVariantColumnPath(g.child) + // A variant-to-variant `Cast` (e.g. `cast(v as variant)`) is a whole-variant read: its sole + // requested field targets `VariantType`, which the pushdown drops (see SPARK-57499) because a + // lone variant-typed slot saves no I/O and is mishandled by the reader. Hoisting it would add a + // Project + alias for no shredding benefit and leave the column raw anyway, so exclude it. + // A hoistable `Cast` yields a non-variant-typed alias, so it is never re-hoisted (idempotent). + case c: Cast => !c.dataType.isInstanceOf[VariantType] && isVariantColumnPath(c.child) + case _ => false + } + + /** + * Collects hoisted extractions as aliases, de-duplicated by canonical form so a repeated + * extraction maps to a single output slot. + */ + private class ExtractionHoister { + private val extracted = mutable.LinkedHashMap.empty[Expression, Alias] + + def aliases: Seq[NamedExpression] = extracted.values.toSeq + + def isEmpty: Boolean = extracted.isEmpty + + /** Returns (creating if needed) the alias attribute for a single hoistable extraction. */ + def aliasFor(ex: Expression): Attribute = + extracted.getOrElseUpdate(ex.canonicalized, Alias(ex, s"_ve${extracted.size}")()).toAttribute + + /** Replaces every hoistable extraction in `e` with a reference to its (new) alias. */ + def hoist(e: Expression): Expression = e.transformDown { + case ex if isHoistable(ex) => aliasFor(ex) + } + } + + // Recursively pushes hoisted extraction aliases down through a join tree until each lands in a + // `Project` directly above a non-join child (the scan side, or the `Filter`/`Project` chain above + // it that `PhysicalOperation` collapses). Hoisting an extraction into a `Project` above a `Join` + // is not sufficient on its own, because `PhysicalOperation` stops at the `Join`: the variant + // would still be read raw at the scan. This descends any depth of chained joins in a single call. + // + // `aliases` are the extraction aliases still to be placed. Each is routed to the join side whose + // output owns its referenced attribute. `needed` is the set of attributes that must remain in the + // pushed-into child's output -- the columns referenced above the push point plus the + // join-condition references accumulated while recursing, so join keys are never dropped. A + // variant consumed solely via a hoisted extraction is thereby not passed through (no redundant + // full-variant slot); one used raw elsewhere stays in `needed` and keeps its full-variant slot. + private def pushSideAliases( + child: LogicalPlan, + aliases: Seq[NamedExpression], + needed: AttributeSet): LogicalPlan = { + if (aliases.isEmpty) { + child + } else { + child match { + case join: Join => + val leftOutput = join.left.outputSet + val rightOutput = join.right.outputSet + // For LeftSemi/LeftAnti/ExistenceJoin (all matched by `LeftExistence`) the right side is + // not in the join output, so a right-side alias could never be referenced above the join. + // Route only left-side aliases there; leave any right-side alias in place (it will not + // have been produced for such joins in practice). + val rightEligible = join.joinType match { + case LeftExistence(_) => false + case _ => true + } + val (leftAliases, rest) = + aliases.partition(_.references.subsetOf(leftOutput)) + val (rightAliases, unrouted) = + if (rightEligible) rest.partition(_.references.subsetOf(rightOutput)) + else (Seq.empty[NamedExpression], rest) + if (leftAliases.isEmpty && rightAliases.isEmpty) { + // Nothing cleanly routable (e.g. an ExistenceJoin, or an alias not confined to one + // side). Fall back to a Project above the join keeping the live columns. + val keep = join.output.filter(needed.contains) + Project(keep ++ aliases, join) + } else { + val neededHere = + needed ++ join.condition.map(_.references).getOrElse(AttributeSet.empty) + val newLeft = pushSideAliases(join.left, leftAliases, neededHere) + val newRight = pushSideAliases(join.right, rightAliases, neededHere) + val newJoin = join.copy(left = newLeft, right = newRight) + // An alias that could not be routed to a single side stays above the join. + if (unrouted.isEmpty) { + newJoin + } else { + val keep = newJoin.output.filter(needed.contains) + Project(keep ++ unrouted, newJoin) + } + } + // Descend through a pass-through `Project` (as `PhysicalOperation` does) to reach the + // join/scan beneath the intermediate Projects the optimizer leaves between chained joins. + // A `pushable` alias resolves against the grandchild and is pushed there; a `stay` alias + // references a column this Project introduces and cannot be pushed. + case project @ Project(projectList, grandChild) => + val (pushable, stay) = + aliases.partition(_.references.subsetOf(grandChild.outputSet)) + if (pushable.isEmpty) { + val keep = project.output.filter(needed.contains) + Project(keep ++ aliases, project) + } else { + // Prune to the still-live columns before re-adding the pushed alias references, else a + // bare variant consumed only via a now-pushed extraction would stay live and be read + // raw. Retain any projection a `stay` alias references so it stays resolvable. + val stayRefs = AttributeSet(stay.flatMap(_.references)) + val keptProjections = projectList.filter { e => + needed.contains(e.toAttribute) || stayRefs.contains(e.toAttribute) + } + val newGrandChild = pushSideAliases( + grandChild, pushable, needed ++ AttributeSet(keptProjections.flatMap(_.references))) + Project(keptProjections ++ pushable.map(_.toAttribute) ++ stay, newGrandChild) + } + // Any other operator -- notably a `Filter` above the scan: wrap a `Project` above it. We do + // NOT descend through a `Filter`, as that would force its variant predicate reference into + // `needed` and pass the raw variant through. The Filter and this Project collapse into the + // same `PhysicalOperation` chain, so the hoisted extraction is still seen at the scan. + case other => + val keep = other.output.filter(needed.contains) + Project(keep ++ aliases, other) + } + } + } + + // Routes hoistable extractions found in `projectList` to the owning-side hoister and returns the + // projectList with those extractions replaced by references to their (new) alias attributes. + // Mirrors the join-condition routing in `rewriteJoinUnderProject`: a single variant extraction + // references exactly one attribute, hence one join side. Anything not cleanly on one side is left + // in place. + private def hoistProjectListExtractions( + projectList: Seq[NamedExpression], + leftOutput: AttributeSet, + rightOutput: AttributeSet, + leftHoister: ExtractionHoister, + rightHoister: ExtractionHoister): Seq[NamedExpression] = { + projectList.map { e => + e.transformDown { + case ex if isHoistable(ex) => + if (ex.references.subsetOf(leftOutput)) { + leftHoister.aliasFor(ex) + } else if (ex.references.subsetOf(rightOutput)) { + rightHoister.aliasFor(ex) + } else { + ex + } + }.asInstanceOf[NamedExpression] + } + } + + private def rewriteAggregate(agg: Aggregate): LogicalPlan = { + val hoister = new ExtractionHoister + // Only hoist extractions that sit inside an aggregate function's arguments (or filter). A + // top-level extraction in `aggregateExpressions` is a grouping-key reference (grouping keys are + // already pulled out by `PullOutGroupingExpressions`); hoisting it would leave the `Aggregate` + // referencing a column that is neither grouped nor aggregated. + val newAggExprs = agg.aggregateExpressions.map { + _.transform { + case ae: AggregateExpression => ae.withNewChildren(ae.children.map(hoister.hoist)) + }.asInstanceOf[NamedExpression] + } + + if (hoister.isEmpty) { + agg + } else { + // `referenced` is what the rewritten Aggregate still needs directly (aggregate-function args + // after hoisting, plus grouping keys). A variant consumed solely via a hoisted extraction is + // not in it and so is dropped rather than passed through raw -- see the class doc. + val referenced = AttributeSet( + newAggExprs.flatMap(_.references) ++ agg.groupingExpressions.flatMap(_.references)) + val newChild = agg.child match { + // Fuse into the Project that PullOutGroupingExpressions/PhysicalOperation already placed + // below the Aggregate; the Once-batch earlyScanPushDownRules has no CollapseProject to + // flatten a second stacked Project. Sound only when the fused Project still resolves + // against `grandChild.output` (see `canFuseIntoChildProject`); else keep the nested one. + case Project(projectList, grandChild) + if canFuseIntoChildProject(projectList, referenced, hoister, grandChild) => + val keptProjections = projectList.filter(e => referenced.contains(e.toAttribute)) + val fused = Project(keptProjections ++ hoister.aliases, grandChild) + grandChild match { + // The fused Project sits directly above a Join, which `PhysicalOperation` cannot see + // through -- and this newly created node will not be revisited by the outer + // `transformUp` in this pass. Push the hoisted aliases the rest of the way down through + // the join immediately, reusing the Project-over-Join handling. + case grandChildJoin: Join => + rewriteJoinUnderProject(fused, fused.projectList, grandChildJoin) + case _ => fused + } + // No fusable child Project. Only the still-referenced columns plus the hoisted aliases + // reach the child, so a variant used solely via a hoisted extraction is dropped. + case join: Join => + // The Aggregate's child is a Join. `pushSideAliases` pushes the hoisted aliases down + // through the join tree onto the owning side(s), where they land directly above the scan; + // the Aggregate references the `_ve` aliases, which the join propagates up through its + // output. Any variant still needed raw (in `referenced`) stays live. A `Project` wrapper + // above the join would leave the aliases invisible to the pushdown (`PhysicalOperation` + // stops at the join) and would not be revisited by the outer `transformUp`, so we push + // down instead. + pushSideAliases(join, hoister.aliases, referenced) + case other => + val passthrough = other.output.filter(referenced.contains) + Project(passthrough ++ hoister.aliases, other) + } + agg.copy(aggregateExpressions = newAggExprs, child = newChild) + } + } + + // Fusing the hoisted aliases into an existing child `Project` (collapsing it onto its own child) + // is sound only if everything the fused Project carries still resolves against `grandChild`: + // - the retained projections (those whose output is still referenced), and + // - the hoisted alias expressions. + // If any of these references an attribute the Project introduces itself (not present in + // `grandChild.output`) -- e.g. a nested-variant extraction materialized as an intermediate alias + // -- collapsing would leave that reference unresolved, so we must not fuse. + private def canFuseIntoChildProject( + projectList: Seq[NamedExpression], + referenced: AttributeSet, + hoister: ExtractionHoister, + grandChild: LogicalPlan): Boolean = { + val grandChildOutput = grandChild.outputSet + val keptProjections = projectList.filter(e => referenced.contains(e.toAttribute)) + val carried = keptProjections ++ hoister.aliases + carried.forall(_.references.subsetOf(grandChildOutput)) + } + + private def rewriteSortUnderProject( + project: Project, projectList: Seq[NamedExpression], sort: Sort): LogicalPlan = { + val hoister = new ExtractionHoister + val newOrder = sort.order.map(hoister.hoist(_).asInstanceOf[SortOrder]) + if (hoister.isEmpty) { + project + } else { + // Columns still needed from the Sort's child: what the Project above consumes, plus what the + // rewritten order references directly (the alias attributes are supplied below, not from the + // child). A variant used only by the (now hoisted) order key is thereby dropped. + val needed = + AttributeSet(projectList.flatMap(_.references) ++ newOrder.flatMap(_.references)) + // `pushSideAliases` places the aliases in a Project directly above the child's non-join + // descendant, descending through any Join/Project chain between the Sort and the scan. For a + // flat child (scan / Filter above a scan) this is a single Project directly below the Sort, + // matching the previous behavior; when the Sort sits over a Join the aliases are pushed + // through it so the extraction reaches the scan instead of being read raw. + val newChild = pushSideAliases(sort.child, hoister.aliases, needed) + // Reproduce the original projectList so the alias columns do not leak into the output. + project.copy(child = sort.copy(order = newOrder, child = newChild)) + } + } + + // Handles a `Sort` that is NOT directly under a narrowing `Project` -- a bare/root Sort such as + // `SELECT * FROM t ORDER BY variant_get(v, '$.b')` (all order columns are already selected, so + // the analyzer adds no narrowing Project above the Sort), or a Sort under a + // GlobalLimit/LocalLimit. + // Unlike `rewriteSortUnderProject`, there is no Project above to tell us which columns remain + // live; a bare Sort's output IS its child's output, so ALL child columns must stay live. A + // variant present in the output (SELECT *) therefore stays raw AND gains an extra `_ve` slot, + // yielding a multi-slot shredded struct with a full-variant slot -- the same shape as a + // Sort-under-Project whose `v` is also selected. See the class doc. + private def rewriteBareSort(sort: Sort): LogicalPlan = { + val hoister = new ExtractionHoister + val newOrder = sort.order.map(hoister.hoist(_).asInstanceOf[SortOrder]) + if (hoister.isEmpty) { + sort + } else { + // All child output columns stay live: the bare Sort's output equals its child's output. The + // hoisted `_ve` aliases are supplied to `pushSideAliases` separately (not via `needed`). + val needed = sort.child.outputSet + // `pushSideAliases` places the aliases in a Project directly above the child's non-join + // descendant, descending through any Join/Project chain between the Sort and the scan. + val newChild = pushSideAliases(sort.child, hoister.aliases, needed) + // Wrap in a Project restoring the original output so the `_ve` alias columns do not leak. + // `sort.child.output` (original attributes, in order) is the correct projectList -- NOT + // `newSort.output`, which would include the `_ve` columns `pushSideAliases` added below. + // `sort.copy` preserves `sort.global` and `sort.hint`. + Project(sort.child.output, sort.copy(order = newOrder, child = newChild)) + } + } + + private def rewriteJoinUnderProject( + project: Project, projectList: Seq[NamedExpression], join: Join): LogicalPlan = { + val leftOutput = join.left.outputSet + val rightOutput = join.right.outputSet + val leftHoister = new ExtractionHoister + val rightHoister = new ExtractionHoister + + // A single variant extraction references exactly one attribute, hence one join side; route it + // to that side's hoister. Anything not cleanly on one side is left in place. We hoist from both + // the join condition (its extractions feed the equi-key comparison) and the Project above the + // join (its extractions -- e.g. aggregate arguments hoisted here by `rewriteAggregate`, or a + // user's `SELECT variant_get(...)`), so the pushdown sees them below the join. + val newCondition = join.condition.map(_.transformDown { + case ex if isHoistable(ex) => + if (ex.references.subsetOf(leftOutput)) { + leftHoister.aliasFor(ex) + } else if (ex.references.subsetOf(rightOutput)) { + rightHoister.aliasFor(ex) + } else { + ex + } + }) + val newProjectList = + hoistProjectListExtractions(projectList, leftOutput, rightOutput, leftHoister, rightHoister) + + if (leftHoister.isEmpty && rightHoister.isEmpty) { + project + } else { + // Columns still live above the Join: what the (rewritten) Project consumes and what the + // rewritten condition references directly (alias attributes are supplied by the side Projects + // below). A side's variant used only by a hoisted extraction is dropped rather than passed + // through, so no redundant full-variant slot is requested. + val needed = AttributeSet( + newProjectList.flatMap(_.references) ++ + newCondition.map(_.references).getOrElse(AttributeSet.empty)) + // `pushSideAliases` places the aliases in a Project directly above the side's non-join child + // (recursing through nested joins), so the extractions reach the scan even when the side is + // itself a chain of joins. For a flat side (scan / Filter above a scan) this is a single + // Project, matching the previous behavior. + val newJoin = join.copy( + left = pushSideAliases(join.left, leftHoister.aliases, needed), + right = pushSideAliases(join.right, rightHoister.aliases, needed), + condition = newCondition) + project.copy(projectList = newProjectList, child = newJoin) + } + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScanSuite.scala index 67bee55c22756..d6ed274327a66 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScanSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScanSuite.scala @@ -55,6 +55,63 @@ trait PushVariantIntoScanSuiteBase extends SharedSparkSession { } } + // A single parquet-backed table for `withVariantParquetTables`: the temp view name to expose, its + // column schema, and the row literals to insert. + protected case class VariantTable(view: String, schema: String, inserts: Seq[String]) + + // Like `withVariantParquetData` but sets up several parquet tables at once, each exposed as a + // temp view under the current read path (V1 or V2, per `useV2`). Used by the join tests, which + // need a variant fact table plus dimension tables. Each table is written via V1 parquet to an + // external location, then re-read as a view so the V2 suites exercise the DSv2 scan. + protected def withVariantParquetTables(tables: VariantTable*)(body: => Unit): Unit = { + withTempPath { dir => + val setupNames = tables.map(t => s"temp_setup_${t.view}") + withTable(setupNames: _*) { + tables.zip(setupNames).foreach { case (t, setupName) => + val path = s"${dir.getCanonicalPath}/${t.view}" + sql(s"create table $setupName (${t.schema}) using PARQUET location '$path'") + t.inserts.foreach(values => sql(s"insert into $setupName values $values")) + } + } + val sourceListConf: Seq[(String, String)] = + if (useV2) Seq(SQLConf.USE_V1_SOURCE_LIST.key -> "") else Nil + withSQLConf(sourceListConf: _*) { + tables.foreach { t => + spark.read.parquet(s"${dir.getCanonicalPath}/${t.view}").createOrReplaceTempView(t.view) + } + try body finally tables.foreach(t => spark.catalog.dropTempView(t.view)) + } + } + } + + // Find the scan relation (V1 or V2) whose output contains a column named `columnName`, and return + // that column's data type. Used by multi-scan join tests to inspect the fact table's variant + // column. Fails if zero or more than one matching scan is found. + protected def scanColumnType(plan: LogicalPlan, columnName: String): DataType = { + val matching = plan.collect { + case s: DataSourceV2ScanRelation if s.output.exists(_.name == columnName) => s.output + case l: LogicalRelation if l.output.exists(_.name == columnName) => l.output + } + assert(matching.length == 1, + s"Expected exactly one scan with a column named '$columnName' but found " + + s"${matching.length}:\n$plan") + matching.head.find(_.name == columnName).get.dataType + } + + // Assert that a variant column shredded to a struct and (optionally) that it has no full-variant + // slot -- i.e. the whole blob is not read. + protected def assertShreddedStruct( + dataType: DataType, expectFullVariantSlot: Boolean = false): StructType = { + val struct = dataType match { + case s: StructType => s + case other => fail(s"Expected the variant column shredded to a struct, but got $other") + } + val hasFullVariantSlot = struct.fields.exists(_.dataType.isInstanceOf[VariantType]) + assert(hasFullVariantSlot == expectFullVariantSlot, + s"Expected full-variant slot present=$expectFullVariantSlot, but got $struct") + struct + } + protected def localTimeZone = spark.sessionState.conf.sessionLocalTimeZone // Return a `StructField` with the expected `VariantMetadata`. @@ -73,6 +130,268 @@ trait PushVariantIntoScanSuiteBase extends SharedSparkSession { } } + // Locate the single scan relation in an optimized plan and return its output, regardless of the + // read path (V1 `LogicalRelation` or V2 `DataSourceV2ScanRelation`). Lets a shared test assert + // shredding for both the V1 and V2 pushdown rules. + protected def scanRelationOutput(plan: LogicalPlan): Seq[Attribute] = { + val outputs = plan.collect { + case s: DataSourceV2ScanRelation => s.output + case l: LogicalRelation => l.output + } + assert(outputs.length == 1, + s"Expected exactly one scan relation but found ${outputs.length}:\n$plan") + outputs.head + } + + test("aggregate function argument variant_get is hoisted below the aggregate and shredded") { + // `max(variant_get(v, '$.price'))` lives inside the aggregate function, above the aggregate + // barrier that PhysicalOperation-based pushdown cannot see through. PullOutVariantExtractions + // hoists it into a Project directly below the Aggregate so both the V1 and V2 pushdown rules + // shred `v` to just the `$.price` slot, with no full-variant slot (the raw column is never + // read). Runs under both the V1 and V2 pushdown paths. + withVariantParquetData( + "v variant, v2 variant, name string", + "(parse_json('{\"price\": 10}'), parse_json('{\"junk\": 1}'), 'a')", + "(parse_json('{\"price\": 30}'), parse_json('{\"junk\": 2}'), 'a')", + "(parse_json('{\"price\": 20}'), parse_json('{\"junk\": 3}'), 'b')") { + val query = "select name, max(variant_get(v, '$.price', 'int')) as mx from T group by name" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // a -> 30, b -> 20 + val output = scanRelationOutput(sql(query).queryExecution.optimizedPlan) + val v = output.find(_.name == "v").getOrElse(fail(s"v missing from ${output.map(_.name)}")) + val vStruct = v.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + } + } + + test("sort order variant_get is hoisted below the sort and shredded") { + // The sort key variant_get(v, '$.price') lives in Sort.order. PullOutVariantExtractions hoists + // it into a Project below the Sort; since only `name` is selected (v is not otherwise live + // above the Sort), v shreds to the $.price slot with no full-variant slot. Runs under V1/V2. + withVariantParquetData( + "v variant, name string", + "(parse_json('{\"price\": 3}'), 'x')", + "(parse_json('{\"price\": 1}'), 'z')", + "(parse_json('{\"price\": 2}'), 'y')") { + val query = "select name from T order by variant_get(v, '$.price', 'int')" + val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().map(_.getString(0)).toList + } + assert(expectedOrder == List("z", "y", "x"), s"baseline sanity: $expectedOrder") + assert(sql(query).collect().map(_.getString(0)).toList == expectedOrder, + "ORDER BY variant_get produced the wrong row order") + val output = scanRelationOutput(sql(query).queryExecution.optimizedPlan) + val vStruct = output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + } + } + + test("bare/root sort (SELECT *): order-key variant_get is hoisted and shredded") { + // SELECT * keeps v in the query output, so the sort key variant_get(v, '$.price') lives in a + // bare/root Sort with no narrowing Project above it (the analyzer adds no Project since every + // order column is already selected). PullOutVariantExtractions handles this via rewriteBareSort + // in its second pass. Because v is in the SELECT * output it stays live, so v shreds to a + // multi-slot struct WITH a full-variant slot (the raw blob is still materialized) plus the + // typed $.price slot the sort reads. Runs under V1/V2. + withVariantParquetData( + "v variant, name string", + "(parse_json('{\"price\": 3}'), 'x')", + "(parse_json('{\"price\": 1}'), 'z')", + "(parse_json('{\"price\": 2}'), 'y')") { + val query = "select * from T order by variant_get(v, '$.price', 'int')" + // Order-sensitive: compare the non-variant `name` column sequence. + val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().map(_.getString(1)).toList + } + assert(expectedOrder == List("z", "y", "x"), s"baseline sanity: $expectedOrder") + assert(sql(query).collect().map(_.getString(1)).toList == expectedOrder, + "SELECT * ORDER BY variant_get produced the wrong row order") + val output = scanRelationOutput(sql(query).queryExecution.optimizedPlan) + assertShreddedStruct( + output.find(_.name == "v").get.dataType, expectFullVariantSlot = true) + } + } + + test("bare/root sort (SELECT id, v): order-key variant_get is hoisted and shredded") { + // Explicitly selecting the variant column (SELECT id, v) produces the same bare/root Sort shape + // as SELECT *: no narrowing Project above the Sort. Exercises rewriteBareSort's second pass; + // v stays live (it is selected) so it shreds with a full-variant slot plus the $.b slot. + withVariantParquetData( + "id int, v variant", + "(1, parse_json('{\"b\": 3}'))", + "(2, parse_json('{\"b\": 1}'))", + "(3, parse_json('{\"b\": 2}'))") { + val query = "select id, v from T order by variant_get(v, '$.b', 'int')" + val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().map(_.getInt(0)).toList + } + assert(expectedOrder == List(2, 3, 1), s"baseline sanity: $expectedOrder") + assert(sql(query).collect().map(_.getInt(0)).toList == expectedOrder, + "SELECT id, v ORDER BY variant_get produced the wrong row order") + val output = scanRelationOutput(sql(query).queryExecution.optimizedPlan) + assertShreddedStruct( + output.find(_.name == "v").get.dataType, expectFullVariantSlot = true) + } + } + + test("bare sort over a join (SELECT *): order-key extraction pushed through the join") { + // A bare/root Sort whose child is a Join, with all join columns selected (no narrowing Project + // above the Sort). The hoisted order-key alias must be pushed DOWN through the join to the ss + // scan side. `data` is in the query output, so it shreds with a full-variant slot plus the + // $.qty slot. Runs under V1/V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')", "(2, 'y')"))) { + val query = + "select ss.k, ss.data, d.name from SS ss join DIM d on ss.k = d.k " + + "order by variant_get(ss.data, '$.qty', 'double')" + val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().map(_.getInt(0)).toList + } + assert(sql(query).collect().map(_.getInt(0)).toList == expectedOrder, + s"SELECT with join columns ORDER BY variant_get produced the wrong row order: " + + s"got ${sql(query).collect().map(_.getInt(0)).toList}, expected $expectedOrder") + assertShreddedStruct( + scanColumnType(sql(query).queryExecution.optimizedPlan, "data"), + expectFullVariantSlot = true) + } + } + + test("cast(v as variant) in aggregate function arg is not hoisted") { + // The aggregate argument is `cast(v as variant)`: a whole-variant read whose sole requested + // field targets VariantType. PullOutVariantExtractions must NOT hoist it (isHoistable excludes + // VariantType-target casts) -- hoisting would add a Project + `_ve` alias for no shredding + // benefit (the pushdown drops a lone VariantType slot anyway, see SPARK-57499). Assert the rule + // introduces no hoist alias, so the plan matches what the pull-out rule being off would give. + // Plan-shape only (no execution): a lone-VariantType shred is a separate, pre-existing concern. + // Runs under both the V1 and V2 pushdown paths. + withVariantParquetData( + "v variant, name string", + "(parse_json('{\"price\": 10}'), 'a')", + "(parse_json('{\"price\": 20}'), 'b')") { + val query = "select name, count(cast(v as variant)) as c from T group by name" + val plan = sql(query).queryExecution.optimizedPlan + val hoistAlias = plan.exists(_.expressions.exists(_.exists { + case a: Alias => a.name.startsWith("_ve") + case _ => false + })) + assert(!hoistAlias, + s"Expected no `_ve` hoist alias for a VariantType-target cast, but plan was:\n$plan") + } + } + + test("two-arg variant_get (returns VariantType) in aggregate function arg is not hoisted") { + // variant_get(v, '$.price') with no target type returns VariantType. isHoistable must NOT hoist + // it: the pushdown drops a lone VariantType requested field (SPARK-57499), so hoisting + // produces a `_ve` alias for no shredding benefit. It would also re-match on the second + // idempotence pass, breaking the Once-batch check. Assert no `_ve` alias is introduced. + withVariantParquetData( + "v variant, name string", + "(parse_json('{\"price\": 10}'), 'a')", + "(parse_json('{\"price\": 20}'), 'b')") { + val query = "select name, count(variant_get(v, '$.price')) as c from T group by name" + val plan = sql(query).queryExecution.optimizedPlan + val hoistAlias = plan.exists(_.expressions.exists(_.exists { + case a: Alias => a.name.startsWith("_ve") + case _ => false + })) + assert(!hoistAlias, + s"Expected no `_ve` hoist alias for a two-arg variant_get (VariantType result), " + + s"but plan was:\n$plan") + } + } + + test("listagg(DISTINCT v:a::string) WITHIN GROUP (ORDER BY v:a::string) does not break " + + "Once-batch idempotence") { + // Regression test for the idempotence bug: `v:a::string` desugars to + // `cast(variant_get(v, '$.a') as string)`, where the inner two-arg variant_get returns + // VariantType. Excluding that two-arg form from hoisting (isHoistable's VariantGet branch) + // leaves the whole `cast(variant_get(...))` untouched -- so nothing is hoisted here and there + // is no VariantType `_ve` alias for the Cast branch to re-hoist on the next pass. Without the + // exclusion the VariantGet would hoist to `_ve0: VariantType`, then `cast(_ve0 as string)` + // would re-match, producing a different plan and crashing with + // "Once strategy's idempotence is broken". + withVariantParquetData( + "v variant", + "(parse_json('{\"a\": \"x\"}'))", + "(parse_json('{\"a\": \"y\"}'))", + "(parse_json('{\"a\": \"x\"}'))") { + val query = + "select listagg(distinct variant_get(v, '$.a', 'string'), ',') " + + "within group (order by variant_get(v, '$.a', 'string')) from T" + // Must execute without an idempotence exception and return the expected deduplicated result. + checkAnswer(sql(query), Seq(org.apache.spark.sql.Row("x,y"))) + } + } + + test("sort key hoisted while a different projected extraction stays above the sort: correct") { + // The projected extraction variant_get(v, '$.a') sits in the select list (above the Sort, + // lifted by PhysicalOperation), while the sort key variant_get(v, '$.b') is hoisted below the + // Sort. Because the projectList still references v raw, the hoist must NOT drop v -- it stays + // live above the Sort so the projected extraction can be computed. This guards the liveness + // computation in rewriteSortUnderProject: results and row order must both be correct. Runs + // under both the V1 and V2 pushdown paths. + withVariantParquetData( + "v variant, name string", + "(parse_json('{\"a\": 100, \"b\": 3}'), 'x')", + "(parse_json('{\"a\": 200, \"b\": 1}'), 'z')", + "(parse_json('{\"a\": 300, \"b\": 2}'), 'y')") { + val query = + "select name, variant_get(v, '$.a', 'int') as a " + + "from T order by variant_get(v, '$.b', 'int')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().toSeq + } + // Baseline sanity: ordered by $.b ascending -> z(1), y(2), x(3) with their $.a values. + assert(expected.map(r => (r.getString(0), r.getInt(1))) == + Seq(("z", 200), ("y", 300), ("x", 100)), s"baseline sanity: $expected") + // Order-sensitive comparison: a mis-hoist that dropped v or mis-shredded would change the + // projected $.a value or the row order, which an order-insensitive checkAnswer would miss. + assert(sql(query).collect().toSeq.map(r => (r.getString(0), r.getInt(1))) == + expected.map(r => (r.getString(0), r.getInt(1))), + "projected extraction / order incorrect after hoisting the sort key") + } + } + + test("group by + aggregate hoist fuses into one Project, no stacked Project") { + // GROUP BY variant_get(v, '$.k') is relocated below the Aggregate by PullOutGroupingExpressions + // (creating a Project), and the aggregated variant_get(v, '$.price') is hoisted by + // PullOutVariantExtractions. The hoist must FUSE its aliases into that existing Project rather + // than stack a second one -- the earlyScanPushDownRules batch runs Once with no CollapseProject + // to flatten it afterward. Assert no two directly-stacked Projects survive in the optimized + // plan. Runs under both the V1 and V2 pushdown paths. + withVariantParquetData( + "v variant", + "(parse_json('{\"k\": \"a\", \"price\": 10}'))", + "(parse_json('{\"k\": \"a\", \"price\": 30}'))", + "(parse_json('{\"k\": \"b\", \"price\": 20}'))") { + val query = "select variant_get(v, '$.k', 'string') as k, " + + "max(variant_get(v, '$.price', 'int')) as mx " + + "from T group by variant_get(v, '$.k', 'string')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // a -> 30, b -> 20 + val plan = sql(query).queryExecution.optimizedPlan + val stacked = plan.exists { + case Project(_, _: Project) => true + case _ => false + } + assert(!stacked, + s"Expected no directly-stacked Project nodes after hoist fusion, but got:\n$plan") + } + } + // Returns true iff `t` or any of its causes is an INVALID_VARIANT_CAST error. The failure may // surface directly or be wrapped in a task failure. protected def hasCastCondition(t: Throwable): Boolean = t match { @@ -275,6 +594,317 @@ trait PushVariantIntoScanSuiteBase extends SharedSparkSession { } } } + + // A variant fact table `SS(k int, data variant)` used by the push-through-join tests below. + private def factTable(view: String = "SS"): VariantTable = VariantTable( + view, "k int, data variant", + Seq( + "(1, parse_json('{\"qty\": 10, \"id\": \"a\"}'))", + "(1, parse_json('{\"qty\": 30, \"id\": \"a\"}'))", + "(2, parse_json('{\"qty\": 20, \"id\": \"b\"}'))")) + + test("aggregate over a single inner join: variant hoisted through the join and shredded") { + // AVG(variant_get(ss.data, '$.qty')) sits in an Aggregate whose child is a Join. The extraction + // is hoisted, then pushed down through the join onto the ss side so it lands directly above the + // ss scan. `data` is used ONLY via extractions (aggregate arg + GROUP BY key), so it shreds + // with no full-variant slot -- the whole blob is never read. Runs under both V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')", "(2, 'y')"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "avg(variant_get(ss.data, '$.qty', 'double')) as avg_qty " + + "from SS ss join DIM d on ss.k = d.k " + + "group by variant_get(ss.data, '$.id', 'string')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // a -> 20.0, b -> 20.0 + assertShreddedStruct(scanColumnType(sql(query).queryExecution.optimizedPlan, "data")) + } + } + + test("aggregate over chained joins: variant pushed through all join levels and shredded") { + // The ss variant is joined to three dimensions. The aggregate-argument extractions must be + // pushed down through EVERY join level to reach the ss scan. This guards the recursion in + // pushSideAliases: a single non-recursive hoist would leave the extraction above the outermost + // join and `data` would still be read raw. Runs under both V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("D1", "k int, a string", Seq("(1, 'p')", "(2, 'q')")), + VariantTable("D2", "k int, b string", Seq("(1, 'r')", "(2, 's')")), + VariantTable("D3", "k int, c string", Seq("(1, 't')", "(2, 'u')"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "max(variant_get(ss.data, '$.qty', 'double')) as max_qty " + + "from SS ss " + + "join D1 on ss.k = D1.k join D2 on ss.k = D2.k join D3 on ss.k = D3.k " + + "group by variant_get(ss.data, '$.id', 'string')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // a -> 30.0, b -> 20.0 + assertShreddedStruct(scanColumnType(sql(query).queryExecution.optimizedPlan, "data")) + } + } + + test("variant used raw and extracted across a join: keeps a full-variant slot") { + // ss.data is both selected raw (needs the whole blob) and extracted. After the extraction is + // pushed to the ss scan side, `data` stays live (still referenced raw), so it shreds to the + // typed slot PLUS a full-variant slot -- the raw value is preserved. Runs under both V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')", "(2, 'y')"))) { + val query = + "select ss.data as raw, variant_get(ss.data, '$.qty', 'double') as qty " + + "from SS ss join DIM d on ss.k = d.k" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) + val struct = + assertShreddedStruct( + scanColumnType(sql(query).queryExecution.optimizedPlan, "data"), + expectFullVariantSlot = true) + // One typed slot ($.qty) plus the full-variant slot. + assert(struct.fields.length == 2, s"Expected two slots, but got $struct") + } + } + + test("aggregate over a left outer join: extraction on the preserved side is shredded") { + // ss is the preserved (left) side of a LEFT JOIN. Its variant extraction pushes onto the left + // side; unmatched rows (null-padded on the right) do not affect the ss-side extraction. Assert + // no full-variant slot and correct results including the unmatched row. Runs under V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "max(variant_get(ss.data, '$.qty', 'double')) as max_qty " + + "from SS ss left join DIM d on ss.k = d.k " + + "group by variant_get(ss.data, '$.id', 'string')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // a -> 30.0 (k=1 matched), b -> 20.0 (k=2 unmatched) + assertShreddedStruct(scanColumnType(sql(query).queryExecution.optimizedPlan, "data")) + } + } + + test("pull-out flag off reads the whole variant across a join") { + // With PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS disabled, the aggregate-argument extractions + // are not hoisted or pushed through the join. `ss.data` flows up through the join as a bare + // attribute, so the scan reads the whole blob -- either left raw as VariantType, or shredded to + // a single full-variant slot (struct<0:variant>). Either way there is no shredding benefit. + // Guards the config gate for the new join push-through path. Runs under both V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')", "(2, 'y')"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "avg(variant_get(ss.data, '$.qty', 'double')) as avg_qty " + + "from SS ss join DIM d on ss.k = d.k " + + "group by variant_get(ss.data, '$.id', 'string')" + withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS.key -> "false") { + val dataType = scanColumnType(sql(query).queryExecution.optimizedPlan, "data") + val readsWholeBlob = dataType match { + case _: VariantType => true + case s: StructType => s.fields.exists(_.dataType.isInstanceOf[VariantType]) + case _ => false + } + assert(readsWholeBlob, + s"Expected data read as a whole blob with the pull-out flag off, but got $dataType") + } + } + } + + test("aggregate over a left semi join: left-side extraction pushed and shredded") { + // A LEFT SEMI join's output is only the left (ss) columns. The ss-side aggregate extraction is + // pushed onto the left side and shreds with no full-variant slot; the semi join's existence + // semantics are unchanged. Runs under both V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int", Seq("(1)"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "max(variant_get(ss.data, '$.qty', 'double')) as max_qty " + + "from SS ss left semi join DIM d on ss.k = d.k " + + "group by variant_get(ss.data, '$.id', 'string')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // only k=1 rows survive -> a -> 30.0 + assertShreddedStruct(scanColumnType(sql(query).queryExecution.optimizedPlan, "data")) + } + } + + test("left outer join: extraction on the NULLABLE (right) side is pushed and value-preserving") { + // The extraction reads the DIM (right) variant, which is the NULLABLE side of a LEFT OUTER + // join: unmatched left rows are null-padded on the right. Pushing the extraction below the join + // computes variant_get BEFORE null-padding; the join then null-pads the `_ve` alias for + // unmatched rows. This must equal the baseline, where variant_get(NULL) is evaluated ABOVE the + // join for those rows -- i.e. null-padding commutes with the extraction (variant_get(NULL) = + // NULL). Asserting correctness here is the real point; also assert DIM.data shreds. The `ss` + // (left) side has its own unrelated extraction so both scans are exercised. Runs under V1/V2. + withVariantParquetTables( + factTable(), + VariantTable( + "DIM", "k int, data variant", + // Only k=1 matches SS's k in {1, 2}; SS rows with k=2 are unmatched -> right side null. + Seq("(1, parse_json('{\"label\": \"L1\"}'))"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "variant_get(d.data, '$.label', 'string') as label " + + "from SS ss left join DIM d on ss.k = d.k " + + "order by id, label" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + // Order-sensitive comparison so a wrong null for unmatched rows cannot be masked. + assert(sql(query).collect().toSeq.map(r => (r.get(0), r.get(1))) == + expected.toSeq.map(r => (r.get(0), r.get(1))), + "Nullable-side extraction over a left outer join produced wrong values") + // Both SS.data and DIM.data are extracted only via variant_get; each shreds with no + // full-variant slot on its own scan (both scans expose a `data` column, so assert per scan). + val plan = sql(query).queryExecution.optimizedPlan + val dataTypes = plan.collect { + case s: DataSourceV2ScanRelation => s.output + case l: LogicalRelation => l.output + }.flatMap(_.filter(_.name == "data")).map(_.dataType) + assert(dataTypes.length == 2, s"Expected two `data` scan columns, but got:\n$plan") + dataTypes.foreach(dt => assertShreddedStruct(dt)) + } + } + + test("full outer join: extractions on both (nullable) sides are pushed and value-preserving") { + // In a FULL OUTER join both sides are nullable: unmatched rows are null-padded on the opposite + // side. Extractions on each side push onto that side, computed before null-padding; the join + // null-pads the resulting `_ve` aliases for the unmatched rows. Result must match the baseline + // that evaluates variant_get above the join (variant_get(NULL) = NULL). Runs under V1 and V2. + withVariantParquetTables( + VariantTable( + "L", "k int, data variant", + Seq("(1, parse_json('{\"lv\": \"a\"}'))", "(2, parse_json('{\"lv\": \"b\"}'))")), + VariantTable( + "R", "k int, data variant", + Seq("(2, parse_json('{\"rv\": \"y\"}'))", "(3, parse_json('{\"rv\": \"z\"}'))"))) { + val query = + "select variant_get(l.data, '$.lv', 'string') as lv, " + + "variant_get(r.data, '$.rv', 'string') as rv " + + "from L l full outer join R r on l.k = r.k " + + "order by lv, rv" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + assert(sql(query).collect().toSeq.map(r => (r.get(0), r.get(1))) == + expected.toSeq.map(r => (r.get(0), r.get(1))), + "Both-nullable-side extractions over a full outer join produced wrong values") + // Both L.data and R.data are extracted only via variant_get, so each shreds with no + // full-variant slot on its own scan. Both scans expose a column named `data`, so assert on + // every `data` scan output rather than via the single-match scanColumnType helper. + val plan = sql(query).queryExecution.optimizedPlan + val dataTypes = plan.collect { + case s: DataSourceV2ScanRelation => s.output + case l: LogicalRelation => l.output + }.flatMap(_.filter(_.name == "data")).map(_.dataType) + assert(dataTypes.length == 2, s"Expected two `data` scan columns, but got:\n$plan") + dataTypes.foreach(dt => assertShreddedStruct(dt)) + } + } + + test("strict cast pushed through a join: cast-error deferral covers join-eliminated rows") { + // SS has a row whose data is a non-castable type; that row is eliminated by the join (its k + // has no match in DIM). The aggregate-argument extraction `cast(ss.data as int)` is hoisted + // onto the SS side and pushed through the join to the scan. + // - With deferral OFF: the strict cast is evaluated at the scan on the eliminated row and + // raises INVALID_VARIANT_CAST (the same trade-off as below-Filter pushdown). + // - With deferral ON: the cast-error companion slot carries the error instead; the join + // eliminates the row before its error is consumed; the query returns correct values. + // Data: SS has k=1 (string data, bad for cast) and k=2 (int data, good); DIM has k=2 only. + // Result: only the k=2 row survives the join and is included in the aggregate. + withVariantParquetTables( + VariantTable( + "SS", "k int, data variant", + Seq("(1, parse_json('\"hello\"'))", "(2, parse_json('42'))")), + VariantTable( + "DIM", "k int, name string", + Seq("(2, 'match')"))) { + val query = "select avg(cast(ss.data as int)) as avg_val from SS ss join DIM d on ss.k = d.k" + + // Without deferral, the strict cast pushed into the scan raises on the k=1 row (malformed) + // even though that row is eliminated by the join condition ss.k = d.k (no k=1 in DIM). + withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN_DEFER_CAST_ERROR.key -> "false") { + val ex = intercept[Exception](sql(query).collect()) + assert(hasCastCondition(ex), s"Expected INVALID_VARIANT_CAST, got $ex") + } + + // With deferral, the cast error is deferred and not raised because the failing row is + // eliminated before its error is consumed. The query returns the correct average. + withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN_DEFER_CAST_ERROR.key -> "true") { + val rows = sql(query).collect() + assert(rows.length == 1, s"Expected 1 row, got ${rows.length}") + val avgVal = rows(0).getDouble(0) + assert(avgVal == 42.0, s"Expected avg=42.0 (only k=2 row survives), got $avgVal") + } + } + } + + test("left semi join with a right-side variant_get in the condition: right side shredded") { + // The join condition extracts from the RIGHT (DIM) variant. For a LEFT SEMI join the right side + // is not in the join output, but the condition is still evaluated over both children, so the + // right-side extraction is hoisted onto the right side and its `_ve` alias feeds the condition. + // Exercises the right-side routing in rewriteJoinUnderProject for a LeftExistence join. Assert + // correct existence semantics and that DIM.data shreds with no full-variant slot. V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable( + "DIM", "dk int, data variant", + Seq("(1, parse_json('{\"jk\": 1}'))", "(2, parse_json('{\"jk\": 9}'))"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id " + + "from SS ss left semi join DIM d " + + "on ss.k = variant_get(d.data, '$.jk', 'int') " + + "order by id" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // ss.k in {1,2}; DIM jk in {1,9}; only k=1 rows survive + // DIM.data (right side) is used only via the condition extraction, so it shreds to the $.jk + // slot with no full-variant slot. Both SS and DIM expose a `data` column; assert both scans + // shred (the SS-side `$.id` extraction and the DIM-side `$.jk` condition extraction). + val plan = sql(query).queryExecution.optimizedPlan + val dataTypes = plan.collect { + case s: DataSourceV2ScanRelation => s.output + case l: LogicalRelation => l.output + }.flatMap(_.filter(_.name == "data")).map(_.dataType) + assert(dataTypes.length == 2, s"Expected two `data` scan columns, but got:\n$plan") + dataTypes.foreach(dt => assertShreddedStruct(dt)) + } + } + + test("sort over a join: order-key extraction is pushed through the join and shredded") { + // ORDER BY variant_get(ss.data, ...) sits in a Sort whose child is a Join (via the intermediate + // Project the optimizer leaves above the join). The hoisted order-key alias must be pushed DOWN + // through the join onto the ss side to reach the ss scan; parking it in a Project above the + // join would leave `data` read raw (PhysicalOperation stops at the join). Only ss.k is + // selected, so `data` is used only via the order key and shreds with no full-variant slot. + // Runs under V1/V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')", "(2, 'y')"))) { + val query = + "select ss.k from SS ss join DIM d on ss.k = d.k " + + "order by variant_get(ss.data, '$.qty', 'double')" + // Order-sensitive: compare the exact row sequence against the pushdown-disabled baseline. + val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().map(_.getInt(0)).toList + } + assert(sql(query).collect().map(_.getInt(0)).toList == expectedOrder, + s"ORDER BY variant_get over a join produced the wrong row order: got " + + s"${sql(query).collect().map(_.getInt(0)).toList}, expected $expectedOrder") + assertShreddedStruct(scanColumnType(sql(query).queryExecution.optimizedPlan, "data")) + } + } } // V1 DataSource tests with parameterized reader type @@ -651,6 +1281,72 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } } + test(s"V2 test - nested variant extraction in aggregate function arg is hoisted and shredded " + + s"($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (vs struct, name string) " + + s"using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(named_struct('v1', parse_json('{\"price\": 10}'), 'i', 1), 'a'), " + + "(named_struct('v1', parse_json('{\"price\": 30}'), 'i', 2), 'a'), " + + "(named_struct('v1', parse_json('{\"price\": 20}'), 'i', 3), 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // The extraction is on a variant nested in a struct (vs.v1). Its child is a GetStructField + // chain rooted at an attribute, which isVariantColumnPath accepts, so it is hoisted and the + // nested variant shreds to the $.price slot. + val query = + "select name, max(variant_get(vs.v1, '$.price', 'int')) as mx from T_V2 group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) // a -> 30, b -> 20 + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vsStruct = scanRelation.output.find(_.name == "vs").get.dataType match { + case s: StructType => s + case other => fail(s"Expected vs to stay a struct, but got $other") + } + val v1Struct = vsStruct.fields.find(_.name == "v1").get.dataType match { + case s: StructType => s + case other => fail(s"Expected nested v1 shredded to struct, but got $other") + } + assert(!v1Struct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot in nested v1, but got $v1Struct") + } + } + } + + test(s"V2 test - cast(variant) in aggregate function arg is hoisted and shredded ($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant, name string) using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('\"10\"'), 'a'), (parse_json('\"30\"'), 'a'), (parse_json('\"20\"'), 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // The aggregate argument is `cast(v as string)`, not variant_get. isHoistable's Cast branch + // (matching collectRequestedFields) makes it eligible, so v shreds to the cast slot. + val query = "select name, max(cast(v as string)) as mx from T_V2 group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vStruct = scanRelation.output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + } + } + } + test(s"V2 test - no pushdown when struct is used ($readerName)") { withTempPath { dir => val path = dir.getCanonicalPath @@ -1031,7 +1727,7 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } } - test(s"V2 test - order by variant_get: correct ordering, variant read raw, sibling pruned " + + test(s"V2 test - order by variant_get: hoisted and shredded, correct ordering, sibling pruned " + s"($readerName)") { withTempPath { dir => val path = dir.getCanonicalPath @@ -1045,11 +1741,11 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { spark.read.parquet(path).createOrReplaceTempView("T_V2") - // The sort key variant_get(v, '$.price') lives in Sort.order, above the scan window, so v - // is lifted in only as a bare reference -> a whole-variant request -> v stays raw and the - // sort evaluates on the real variant. Compare ORDER-SENSITIVELY: a shredded full-variant - // slot would collapse to a boolean placeholder and silently mis-order, which the - // order-insensitive checkAnswer would not catch. + // The sort key variant_get(v, '$.price') lives in Sort.order. PullOutVariantExtractions + // hoists it into a Project below the Sort, and since v is not otherwise referenced above + // the Sort (only `name` is selected), the raw v is dropped and v shreds to the $.price slot + // (no full-variant slot). Compare ORDER-SENSITIVELY: a placeholder slot would silently + // mis-order, which the order-insensitive checkAnswer would not catch. val query = "select name from T_V2 order by variant_get(v, '$.price', 'int')" val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { sql(query).collect().map(_.getString(0)).toList @@ -1061,15 +1757,18 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) assert(scanRelation.output.map(_.name).toSet == Set("v", "name"), s"Expected scan output {v, name} but got ${scanRelation.output.map(_.name)}") - val vAttr = scanRelation.output.find(_.name == "v").get - assert(vAttr.dataType.isInstanceOf[VariantType], - s"Expected v left as raw VariantType, but got ${vAttr.dataType}") + val vStruct = scanRelation.output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") } } } - test(s"V2 test - aggregate max(variant_get) with no local filter: no codegen crash, variant " + - s"read raw ($readerName)") { + test(s"V2 test - aggregate max(variant_get) with no local filter: hoisted and shredded " + + s"($readerName)") { withTempPath { dir => val path = dir.getCanonicalPath withTable("temp_v1") { @@ -1082,11 +1781,10 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { spark.read.parquet(path).createOrReplaceTempView("T_V2") - // variant_get(v, '$.price') is inside an aggregate function, above the aggregate barrier, - // with no local filter/projection on v, so v is lifted in only as a bare reference -> a - // whole-variant request. A whole-variant read is kept raw: shredding it to a lone - // full-variant slot would collapse to a boolean placeholder, and max(variant_get(, - // ...)) would fail to codegen. Keeping v raw yields correct aggregates with a valid plan. + // variant_get(v, '$.price') is inside an aggregate function, above the aggregate barrier. + // PullOutVariantExtractions hoists it into a Project directly below the Aggregate, so the + // pushdown shreds v to just the $.price slot: no full-variant slot (so no boolean + // placeholder to break codegen), and the raw column is never read. val query = "select name, max(variant_get(v, '$.price', 'int')) as mx from T_V2 group by name" val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { @@ -1095,15 +1793,152 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant checkAnswer(sql(query), expectedRows) // a -> 30, b -> 20 val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) val vAttr = scanRelation.output.find(_.name == "v").get - assert(vAttr.dataType.isInstanceOf[VariantType], - s"Expected v left as raw VariantType, but got ${vAttr.dataType}") + val vStruct = vAttr.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") assert(!scanRelation.output.exists(_.name == "v2"), s"Expected v2 pruned but got ${scanRelation.output.map(_.name)}") } } } - test(s"V2 test - join on variant_get key: no crash, variant read raw, sibling pruned " + + test(s"V2 test - aggregate over the whole variant is left raw, no hoist ($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant, name string) using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('{\"price\": 10}'), 'a'), (parse_json('{\"price\": 20}'), 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // count(v) references the whole variant, not a field extraction. PullOutVariantExtractions + // has nothing to hoist, and the pushdown keeps v raw (shredding a sole full-variant request + // saves no I/O). + val query = "select name, count(v) as c from T_V2 group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vAttr = scanRelation.output.find(_.name == "v").get + assert(vAttr.dataType.isInstanceOf[VariantType], + s"Expected v left raw, but got ${vAttr.dataType}") + } + } + } + + test(s"V2 test - GROUP BY key and aggregate function argument both shredded, dedup " + + s"($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant) using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('{\"k\": \"a\", \"price\": 10}')), " + + "(parse_json('{\"k\": \"a\", \"price\": 30}')), " + + "(parse_json('{\"k\": \"b\", \"price\": 20}'))") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // $.k is the GROUP BY key (relocated below the Aggregate by PullOutGroupingExpressions); + // $.price appears twice inside aggregate functions (relocated by the pull-out rule). + // The two $.price extractions must dedup to a single struct slot, and there must be no + // full-variant slot. + val query = "select variant_get(v, '$.k', 'string') as k, " + + "max(variant_get(v, '$.price', 'int')) as mx, " + + "min(variant_get(v, '$.price', 'int')) as mn " + + "from T_V2 group by variant_get(v, '$.k', 'string')" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) // a -> (30, 10), b -> (20, 20) + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vStruct = scanRelation.output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + // Exactly two data slots: one for $.k, one shared for the two $.price extractions. + assert(vStruct.fields.length == 2, + s"Expected 2 shredded fields ($$.k and a deduped $$.price), but got $vStruct") + } + } + } + + test(s"V2 test - aggregate function arg with a non-literal path is not hoisted ($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant, p string, name string) " + + s"using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('{\"price\": 10}'), '$.price', 'a'), " + + "(parse_json('{\"price\": 20}'), '$.price', 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // The extraction path is a column (non-foldable), which the pushdown cannot handle, so the + // rule must not hoist it and v is read raw. Guards the `path.foldable` predicate. + val query = "select name, max(variant_get(v, p, 'int')) as mx from T_V2 group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vAttr = scanRelation.output.find(_.name == "v").get + assert(vAttr.dataType.isInstanceOf[VariantType], + s"Expected v left raw for a non-literal path, but got ${vAttr.dataType}") + } + } + } + + test(s"V2 test - join selecting the whole variant keeps v raw ($readerName)") { + withTempPath { dir => + val itemsPath = dir.getCanonicalPath + "/items" + val countriesPath = dir.getCanonicalPath + "/countries" + withTable("temp_items", "temp_countries") { + sql(s"create table temp_items (v variant, name string) using PARQUET location '$itemsPath'") + sql("insert into temp_items values " + + "(parse_json('{\"country_code\": \"CN\"}'), 'widget'), " + + "(parse_json('{\"country_code\": \"JP\"}'), 'gadget')") + sql(s"create table temp_countries (code string, country_name string) " + + s"using PARQUET location '$countriesPath'") + sql("insert into temp_countries values ('CN', 'China'), ('JP', 'Japan')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(itemsPath).createOrReplaceTempView("ITEMS_V2") + spark.read.parquet(countriesPath).createOrReplaceTempView("COUNTRIES_V2") + // i.v is selected raw (needed above the Join) AND used in the join key. The hoist must NOT + // drop it: v shreds with a full-variant slot plus the $.country_code slot. Guards the join + // liveness computation. + val query = + "select i.v, c.country_name from ITEMS_V2 i join COUNTRIES_V2 c " + + "on variant_get(i.v, '$.country_code', 'string') = c.code" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scans = sql(query).queryExecution.optimizedPlan.collect { + case s: DataSourceV2ScanRelation => s + } + val itemsScan = scans.find(_.output.exists(_.name == "v")).getOrElse( + fail(s"Could not find the items scan in:\n${sql(query).queryExecution.optimizedPlan}")) + val vStruct = itemsScan.output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected a full-variant slot because i.v is selected raw, but got $vStruct") + } + } + } + + test(s"V2 test - join on variant_get key: hoisted and shredded, sibling pruned " + s"($readerName)") { withTempPath { dir => val itemsPath = dir.getCanonicalPath + "/items" @@ -1127,21 +1962,125 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { sql(query).collect() } - // A variant_get join key produces a valid plan (no attribute-binding failure) and - // correct results; the key is read raw and the extraction is evaluated above the scan. checkAnswer(sql(query), expectedRows) val scans = sql(query).queryExecution.optimizedPlan.collect { case s: DataSourceV2ScanRelation => s } val itemsScan = scans.find(_.output.exists(_.name == "v")).getOrElse( fail(s"Could not find the items scan in:\n${sql(query).queryExecution.optimizedPlan}")) - // v is the join key (referenced only via variant_get in the join condition): it must be - // left as a raw variant, not shredded. v2 is unreferenced and must be pruned. + // v is the join key (referenced only via variant_get in the join condition). + // PullOutVariantExtractions hoists the extraction onto the items side, dropping the raw v + // so it shreds to just the $.country_code slot (no full-variant slot). v2 is pruned. assert(itemsScan.output.map(_.name).toSet == Set("v", "name"), s"Expected items scan output {v, name} but got ${itemsScan.output.map(_.name)}") + val vStruct = itemsScan.output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + } + } + } + + test(s"V2 test - self join on variant_get key: both sides shredded ($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant, name string) using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('{\"id\": 1}'), 'a'), (parse_json('{\"id\": 2}'), 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // Both sides' join keys are variant_get(_, '$.id'); each is hoisted onto its own side and + // both variant columns shred to the $.id slot with no full-variant slot. + val query = "select a.name, b.name from T_V2 a join T_V2 b " + + "on variant_get(a.v, '$.id', 'int') = variant_get(b.v, '$.id', 'int')" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scans = sql(query).queryExecution.optimizedPlan.collect { + case s: DataSourceV2ScanRelation => s + } + assert(scans.length == 2, s"Expected two scans but got ${scans.length}") + scans.foreach { s => + val vStruct = s.output.find(_.name == "v").get.dataType match { + case st: StructType => st + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + } + } + } + } + + test(s"V2 test - join on variant_get key with no narrowing Project: left raw, correct " + + s"($readerName)") { + withTempPath { dir => + val itemsPath = dir.getCanonicalPath + "/items" + val countriesPath = dir.getCanonicalPath + "/countries" + withTable("temp_items", "temp_countries") { + sql(s"create table temp_items (v variant, name string) using PARQUET location '$itemsPath'") + sql("insert into temp_items values " + + "(parse_json('{\"country_code\": \"CN\"}'), 'widget'), " + + "(parse_json('{\"country_code\": \"JP\"}'), 'gadget')") + sql(s"create table temp_countries (code string, country_name string) " + + s"using PARQUET location '$countriesPath'") + sql("insert into temp_countries values ('CN', 'China'), ('JP', 'Japan')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(itemsPath).createOrReplaceTempView("ITEMS_V2") + spark.read.parquet(countriesPath).createOrReplaceTempView("COUNTRIES_V2") + // `select *` leaves the Join without a narrowing Project directly above it, so + // PullOutVariantExtractions does not hoist the join-key extraction (see class doc: hoisting + // requires a Project above the barrier to know the live-above set). The plan must still be + // correct with v read raw -- this locks in the documented fallback behavior. + val query = + "select * from ITEMS_V2 i join COUNTRIES_V2 c " + + "on variant_get(i.v, '$.country_code', 'string') = c.code" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scans = sql(query).queryExecution.optimizedPlan.collect { + case s: DataSourceV2ScanRelation => s + } + val itemsScan = scans.find(_.output.exists(_.name == "v")).getOrElse( + fail(s"Could not find the items scan in:\n${sql(query).queryExecution.optimizedPlan}")) val vAttr = itemsScan.output.find(_.name == "v").get assert(vAttr.dataType.isInstanceOf[VariantType], - s"Expected v left as raw VariantType, but got ${vAttr.dataType}") + s"Expected v left raw with no narrowing Project above the join, " + + s"but got ${vAttr.dataType}") + } + } + } + + test(s"V2 test - pull-out flag off leaves aggregate function arg raw ($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant, name string) using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('{\"price\": 10}'), 'a'), (parse_json('{\"price\": 20}'), 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "", + SQLConf.PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS.key -> "false") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // With the pull-out rule disabled, the aggregate function argument is not hoisted and v is + // read raw (pushdown still enabled). Confirms the flag gates the rule. + val query = + "select name, max(variant_get(v, '$.price', 'int')) as mx from T_V2 group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vAttr = scanRelation.output.find(_.name == "v").get + assert(vAttr.dataType.isInstanceOf[VariantType], + s"Expected v left raw with pull-out disabled, but got ${vAttr.dataType}") } } } @@ -1177,8 +2116,8 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } } - test(s"V2 test - join on variant_get key through an aliasing projection: no crash, no " + - s"mis-shred ($readerName)") { + test(s"V2 test - join on variant_get key through an aliasing projection: hoisted and shredded " + + s"($readerName)") { withTempPath { dir => val itemsPath = dir.getCanonicalPath + "/items" val countriesPath = dir.getCanonicalPath + "/countries" @@ -1196,9 +2135,9 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant spark.read.parquet(itemsPath).createOrReplaceTempView("ITEMS_V2") spark.read.parquet(countriesPath).createOrReplaceTempView("COUNTRIES_V2") // The variant column is aliased (v AS vw) in a subquery before the join condition reads - // it via variant_get. Whether the optimizer inlines the alias or keeps it, the query must - // optimize without crashing, return correct results, and not over-shred (v2 pruned, the - // variant join key not turned into a struct that yields wrong data). + // it via variant_get. The optimizer inlines the alias, PullOutVariantExtractions hoists the + // extraction onto the items side, and the join key shreds to the $.country_code slot with + // correct results (checkAnswer) and no wrong data. v2 is pruned. val query = "select c.country_name from " + "(select v as vw, name as nm from ITEMS_V2) i join COUNTRIES_V2 c " + @@ -1217,10 +2156,14 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant // v2 is unreferenced and must be pruned. assert(!itemsScan.output.exists(_.name == "v2"), s"Expected v2 pruned but items scan was ${itemsScan.output.map(_.name)}") - // The variant join key must not be shredded: it is read as a raw variant. - val vAttr = itemsScan.output.find(a => a.name == "v" || a.name == "vw").get - assert(vAttr.dataType.isInstanceOf[VariantType], - s"Expected the variant join key left raw, but got ${vAttr.dataType}") + // The variant join key is hoisted and shredded to the $.country_code slot, no full-variant. + val keyAttr = itemsScan.output.find(a => a.name == "v" || a.name == "vw").get + val vStruct = keyAttr.dataType match { + case s: StructType => s + case other => fail(s"Expected the variant join key shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") } } } @@ -1692,6 +2635,36 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } } + test(s"Spark native managed table: aggregate hoist shreds without full-variant slot " + + s"($readerName)") { + // A Spark native (managed catalog) Parquet table created with CREATE TABLE ... USING PARQUET + // resolves to a V1 `LogicalRelation` regardless of USE_V1_SOURCE_LIST -- that config only + // affects the `spark.read`/DataSource resolution of external file reads, not managed catalog + // tables. PullOutVariantExtractions must still hoist the aggregate-function extraction just + // like it does for external-file reads, and the pushdown must shred v to the $.price slot + // with no full-variant slot. `scanRelationOutput` handles whichever scan node the plan uses. + withTable("T_native") { + sql("create table T_native (v variant, name string) using parquet") + sql("insert into T_native values " + + "(parse_json('{\"price\": 10}'), 'a'), " + + "(parse_json('{\"price\": 30}'), 'a'), " + + "(parse_json('{\"price\": 20}'), 'b')") + val query = + "select name, max(variant_get(v, '$.price', 'int')) as mx from T_native group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) // a -> 30, b -> 20 + val output = scanRelationOutput(sql(query).queryExecution.optimizedPlan) + val vStruct = output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot for a native managed table, but got $vStruct") + } + } + test(s"V2 No push down for JSON ($readerName)") { withTempPath { dir => val path = dir.getCanonicalPath From 18ad81147cbd86373ece320878b8469acd7f2e5f Mon Sep 17 00:00:00 2001 From: Andreas Neumann Date: Mon, 27 Jul 2026 09:05:47 -0700 Subject: [PATCH 09/11] [SPARK-57251][SDP] Validate SCD2 reserved framework columns at AutoCDC flow construction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What changes were proposed in this pull request? What changes were proposed in this pull request? This PR replaces https://github.com/apache/spark/pull/57488, which had to be abandoned due to severe semantic merge conflicts. It is an exact cherry-pick of the previous commits, plus a fix for the test cases that failed due to the merge conflicts. What were the merge conflicts? Two PRs disagreeing on the semantics when a source contains a reserved column but does not include that column in the column selection. This PR rejects that because it validates pre-column selection. The other PR (SPARK-58313) allowed that because it was doing the validation port column selection. I decided to fall back to the original behavior that already existed for SCD type 1 validate pre column selection. And we have SPARK-58325 to reconsider whether we want to do validation post column selection. AutoCdcMergeFlow validates at construction time that a flow's source change-data feed does not carry columns that collide with AutoCDC's internal columns. Until now, that validation (requireReservedPrefixAbsentInSourceColumns) only rejected column names starting with the reserved prefix __spark_autocdc_. SCD2, however, persists two framework columns to the target that do not carry that prefix — __START_AT and __END_AT (see Scd2BatchProcessor.reservedFrameworkColNames). A source column named __START_AT or __END_AT therefore slipped past the guard and would be silently overwritten during microbatch preprocessing. This PR closes that gap (the TODO(SPARK-57251) in Scd2BatchProcessor): - Adds requireReservedFrameworkColumnsAbsentInSourceColumns() to the AutoCdcMergeFlow constructor. For SCD2 flows it rejects any source column whose name collides (by exact name, resolver-aware so it respects spark.sql.caseSensitive) with a non-prefixed reserved framework column. SCD1 targets carry no such columns, so the check is a no-op for SCD1. - Runs the check before the flow's schema val is forced, so the actionable reserved-name error surfaces ahead of the temporary AUTOCDC_SCD2_NOT_SUPPORTED gate, and the check remains correct once SCD2 support lands. - Adds a new error condition AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT (SQLSTATE 42710), distinct from the existing prefix-based AUTOCDC_RESERVED_COLUMN_NAME_PREFIX_CONFLICT, since this collision is by exact name rather than by prefix. - Widens the visibility of Scd2BatchProcessor.reservedFrameworkColNames to private[pipelines] so the flow layer can validate against the single source of truth. ### Why are the changes needed? Without this check, a user whose CDC source happens to contain a __START_AT or __END_AT column would have that data silently overwritten by AutoCDC's SCD2 framework columns, with no error and no diagnostic — a data-correctness footgun. Failing fast at flow construction with a user-actionable error ("rename or remove the column") is the intended UX, consistent with the existing reserved-prefix guard. ### Does this PR introduce _any_ user-facing change? Yes. An AutoCDC SCD2 flow whose source change-data feed contains a column named __START_AT or __END_AT (subject to case-sensitivity settings) now fails at flow construction with AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT instead of silently overwriting the column. There is no change for SCD1 flows, and no change for SCD2 flows whose sources do not use these names. (Note: SCD2 AutoCDC flows are not yet generally supported on master — they are still gated by AUTOCDC_SCD2_NOT_SUPPORTED — so no released behavior changes.) ### How was this patch tested? New unit tests in AutoCdcFlowSuite covering: - an SCD2 flow with a __START_AT/__END_AT source column is rejected with AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT; - the reserved-name check fires before the AUTOCDC_SCD2_NOT_SUPPORTED gate; - an SCD1 flow with the same column name is allowed and the column survives into the flow schema; - case-sensitivity behavior (spark.sql.caseSensitive true/false); - a guard test asserting the set of non-prefixed reserved names is exactly {__START_AT, __END_AT}, so a future rename can't silently un-cover the validation. ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 4.8) Closes #57527 from anew/spark-57251-reserved-columns-v2. Authored-by: Andreas Neumann Signed-off-by: Jose Torres --- .../resources/error/error-conditions.json | 6 + .../autocdc/Scd2BatchProcessor.scala | 15 +- .../spark/sql/pipelines/graph/Flow.scala | 40 ++++ .../pipelines/autocdc/AutoCdcFlowSuite.scala | 205 +++++++++++++----- 4 files changed, 204 insertions(+), 62 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index e4e91cb28760e..ed111426fe366 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -331,6 +331,12 @@ ], "sqlState" : "22023" }, + "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT" : { + "message" : [ + "The column `` in the schema collides with a reserved AutoCDC column name (using column name comparison). The following column names are reserved by AutoCDC and cannot appear in the source: . Rename or remove the column." + ], + "sqlState" : "42710" + }, "AUTOCDC_RESERVED_COLUMN_NAME_PREFIX_CONFLICT" : { "message" : [ "The column `` in the schema collides with the reserved AutoCDC column name prefix `` (using column name comparison). Rename or remove the column." diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala index fbf9bc15127ec..e41f223322bcb 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala @@ -1381,15 +1381,16 @@ object Scd2BatchProcessor { private[pipelines] val endAtColName: String = "__END_AT" /** - * Column names reserved by AutoCDC that will be projected onto the microbatch and - * eventually persisted in the target table. If the user's source dataframe contains any of - * these columns, SCD2 reconciliation will fail. + * Column names reserved by AutoCDC that are projected onto the microbatch and persisted in the + * target table. A source dataframe must not contain any of them. * - * TODO(SPARK-57251): validate at [[AutoCdcMergeFlow]] construction time that the source - * schema and column selection do not collide with these reserved names, so we fail fast - * with a user-actionable error instead of silently overwriting them at preprocess time. + * [[startAtColName]] and [[endAtColName]] do NOT carry the reserved + * [[AutoCdcReservedNames.prefix]], so a source-column collision with them is not caught by the + * prefix-based guard; [[org.apache.spark.sql.pipelines.graph.AutoCdcMergeFlow]] validates the + * source schema against the non-prefixed names in this set at construction time, failing fast + * instead of silently overwriting them at preprocess time. */ - private val reservedFrameworkColNames: Set[String] = Set( + private[pipelines] val reservedFrameworkColNames: Set[String] = Set( startAtColName, endAtColName, AutoCdcReservedNames.cdcMetadataColName diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala index 410b0486c787b..bb996d527cdce 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala @@ -253,6 +253,7 @@ class AutoCdcMergeFlow( val funcResult: FlowFunctionResult ) extends ResolvedFlow { requireReservedPrefixAbsentInSourceColumns() + requireReservedFrameworkColumnsAbsentInSourceColumns() def changeArgs: ChangeArgs = flow.changeArgs @@ -400,6 +401,45 @@ class AutoCdcMergeFlow( } } + /** + * Reject a source column that collides with an SCD2 reserved framework column not covered by + * [[requireReservedPrefixAbsentInSourceColumns]]: the prefix guard only rejects + * [[AutoCdcReservedNames.prefix]] names, but SCD2 also persists the non-prefixed + * [[Scd2BatchProcessor.startAtColName]] and [[Scd2BatchProcessor.endAtColName]]. Runs before + * [[schema]] is forced so the collision fails fast rather than being silently overwritten + * during preprocessing. No-op for SCD1, which has no such columns. + */ + private def requireReservedFrameworkColumnsAbsentInSourceColumns(): Unit = { + val resolver = spark.sessionState.conf.resolver + val reservedPrefix = AutoCdcReservedNames.prefix + + // Only the non-prefixed reserved names need checking here; prefixed ones are already rejected + // by requireReservedPrefixAbsentInSourceColumns. + val reservedNames: Set[String] = changeArgs.storedAsScdType match { + case ScdType.Type2 => + Scd2BatchProcessor.reservedFrameworkColNames.filterNot(_.startsWith(reservedPrefix)) + case ScdType.Type1 => + Set.empty + } + + df.schema.fieldNames + .find(name => reservedNames.exists(resolver(_, name))) + .foreach { conflictingColumnName => + throw new AnalysisException( + errorClass = "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT", + messageParameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.of( + spark.sessionState.conf.caseSensitiveAnalysis + ), + "columnName" -> conflictingColumnName, + "schemaName" -> "changeDataFeed", + "scdType" -> changeArgs.storedAsScdType.label, + "reservedColumnNames" -> reservedNames.toSeq.sorted.mkString(", ") + ) + ) + } + } + /** * Validate all keys specified in changeArgs are actually present in the user-selected schema. */ diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala index 5769b264993bf..d31b9383354cb 100644 --- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala +++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala @@ -370,63 +370,42 @@ class AutoCdcFlowSuite extends QueryTest with SharedSparkSession { } } - test("AutoCdcMergeFlow.schema lets a selection exclude a source column that collides with " + - "a non-prefixed framework column, re-adding it from the framework") { - // Two of the three SCD2 framework columns -- __START_AT and __END_AT -- do not carry the - // reserved AutoCDC prefix, so a source change feed may legitimately contain columns with - // those names. The user excludes them via the column selection; they are dropped from the - // user-data portion and the engine's own framework columns are appended in their place, so - // each still appears exactly once in the output with the framework's type (the sequencing - // type), not the source column's type. - // - // The third framework column, _cdc_metadata, DOES carry the reserved prefix, so a source - // that contains it is rejected outright at flow construction -- it can never reach column - // selection and so is deliberately out of scope here. That rejection is covered separately by - // "AutoCdcMergeFlow rejects a source df column whose name equals the reserved CDC metadata - // column". - val session = spark - import session.implicits._ - // Source carries String-typed __START_AT / __END_AT columns alongside the data columns. - val sourceDf = MemoryStream[(Int, String, Option[Long], String, String)] - .toDS() - .toDF( - "id", - "name", - "seq", - Scd2BatchProcessor.startAtColName, - Scd2BatchProcessor.endAtColName) + test("AutoCdcMergeFlow rejects a source column named after a non-prefixed framework column " + + "even when a selection would exclude it") { + // Behavior change introduced by this PR (SPARK-57251): __START_AT / __END_AT do not carry the + // reserved AutoCDC prefix, so before this change a source could legitimately contain columns + // with those names and exclude them via the column selection. The new + // requireReservedFrameworkColumnsAbsentInSourceColumns guard runs against the RAW source + // schema (before column selection is applied), so such a source is now rejected outright at + // flow construction -- an ExcludeColumns selection cannot rescue it. Allowing an explicit + // opt-out (validating post-selection instead) is tracked separately by SPARK-58325. + val sourceDf = sourceDfWithExtraColumns( + Scd2BatchProcessor.startAtColName -> StringType, + Scd2BatchProcessor.endAtColName -> StringType) - val resolvedFlow = newAutoCdcMergeFlow( - sourceDf = sourceDf, - storedAsScdType = ScdType.Type2, - columnSelection = Some( - ColumnSelection.ExcludeColumns( - Seq( - UnqualifiedColumnName(Scd2BatchProcessor.startAtColName), - UnqualifiedColumnName(Scd2BatchProcessor.endAtColName)) + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow( + sourceDf = sourceDf, + storedAsScdType = ScdType.Type2, + columnSelection = Some( + ColumnSelection.ExcludeColumns( + Seq( + UnqualifiedColumnName(Scd2BatchProcessor.startAtColName), + UnqualifiedColumnName(Scd2BatchProcessor.endAtColName)) + ) + ) ) - ) - ) - - // Each framework column appears exactly once (the source copy was excluded, the framework - // copy appended) ... - assert( - resolvedFlow.schema.fieldNames.count(_ == Scd2BatchProcessor.startAtColName) == 1) - assert( - resolvedFlow.schema.fieldNames.count(_ == Scd2BatchProcessor.endAtColName) == 1) - // ... and carries the framework (sequencing) type, not the source String type. - assert(resolvedFlow.schema(Scd2BatchProcessor.startAtColName).dataType == LongType) - assert(resolvedFlow.schema(Scd2BatchProcessor.endAtColName).dataType == LongType) - // Full expected shape: the retained data columns followed by the framework columns. - assert( - resolvedFlow.schema.fieldNames.toSeq == - Seq( - "id", - "name", - "seq", - Scd2BatchProcessor.startAtColName, - Scd2BatchProcessor.endAtColName, - AutoCdcReservedNames.cdcMetadataColName + }, + condition = "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT", + sqlState = "42710", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "columnName" -> Scd2BatchProcessor.startAtColName, + "schemaName" -> "changeDataFeed", + "scdType" -> ScdType.Type2.label, + "reservedColumnNames" -> + Seq(Scd2BatchProcessor.endAtColName, Scd2BatchProcessor.startAtColName).mkString(", ") ) ) } @@ -703,6 +682,122 @@ class AutoCdcFlowSuite extends QueryTest with SharedSparkSession { } } + // =========================================================================================== + // AutoCdcMergeFlow reserved framework-column (non-prefixed) validation tests + // + // SCD2 persists framework columns __START_AT / __END_AT that do NOT carry the reserved + // AutoCDC prefix, so they are not caught by the prefix guard above. These tests lock in that a + // source column colliding with such a name is rejected at construction for SCD2, is allowed + // for SCD1 (which reserves no non-prefixed names), and that the check respects case-sensitivity. + // =========================================================================================== + + /** The SCD2 reserved framework column names that are not covered by the reserved prefix. */ + private val nonPrefixedScd2ReservedNames: Seq[String] = + Scd2BatchProcessor.reservedFrameworkColNames + .filterNot(_.startsWith(AutoCdcReservedNames.prefix)) + .toSeq + .sorted + + test("non-prefixed reserved names exist and are covered by this suite") { + // Guards against a future refactor renaming/removing __START_AT / __END_AT without updating + // the flow-construction validation: if this set ever empties, the tests below silently + // stop exercising anything. + assert( + nonPrefixedScd2ReservedNames == Seq("__END_AT", "__START_AT"), + s"Unexpected non-prefixed SCD2 reserved names: $nonPrefixedScd2ReservedNames" + ) + } + + test( + "an SCD2 flow with a source column colliding with a reserved framework column is rejected " + + "at construction" + ) { + nonPrefixedScd2ReservedNames.foreach { reservedName => + val sourceDf = sourceDfWithExtraColumns(reservedName -> StringType) + + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow(sourceDf, storedAsScdType = ScdType.Type2) + }, + condition = "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT", + sqlState = "42710", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "columnName" -> reservedName, + "schemaName" -> "changeDataFeed", + "scdType" -> ScdType.Type2.label, + "reservedColumnNames" -> nonPrefixedScd2ReservedNames.mkString(", ") + ) + ) + } + } + + test( + "the reserved framework-column check runs before the SCD2-not-supported gate" + ) { + // The reserved-name error is more actionable than AUTOCDC_SCD2_NOT_SUPPORTED, so it must win + // for an SCD2 flow that both is unsupported and carries a colliding source column. This also + // keeps the check meaningful today (before SCD2 is supported) and correct once it lands. + val sourceDf = sourceDfWithExtraColumns(Scd2BatchProcessor.startAtColName -> StringType) + val ex = intercept[AnalysisException] { + newAutoCdcMergeFlow(sourceDf, storedAsScdType = ScdType.Type2) + } + assert(ex.getCondition == "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT") + } + + test( + "an SCD1 flow with a source column matching an SCD2-only reserved name is allowed" + ) { + // SCD1 targets carry no non-prefixed framework columns, so __START_AT / __END_AT are ordinary + // user columns there. Construction succeeds and the column survives into the flow schema. + nonPrefixedScd2ReservedNames.foreach { reservedName => + val sourceDf = sourceDfWithExtraColumns(reservedName -> StringType) + val resolvedFlow = newAutoCdcMergeFlow(sourceDf, storedAsScdType = ScdType.Type1) + assert(resolvedFlow.schema.fieldNames.contains(reservedName)) + } + } + + test( + "an uppercase reserved framework-column name is rejected for SCD2 when caseSensitive=false" + ) { + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { + val conflictingName = Scd2BatchProcessor.startAtColName.toLowerCase(Locale.ROOT) + val sourceDf = sourceDfWithExtraColumns(conflictingName -> StringType) + + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow(sourceDf, storedAsScdType = ScdType.Type2) + }, + condition = "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT", + sqlState = "42710", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "columnName" -> conflictingName, + "schemaName" -> "changeDataFeed", + "scdType" -> ScdType.Type2.label, + "reservedColumnNames" -> nonPrefixedScd2ReservedNames.mkString(", ") + ) + ) + } + } + + test( + "a differently-cased reserved framework-column name does not trip the reserved check for " + + "SCD2 when caseSensitive=true" + ) { + // Under case-sensitive analysis, a lowercase variant is a distinct identifier and does not + // collide with the reserved (uppercase) framework name, consistent with the prefix guard. + // The reserved-name check therefore does NOT fire and the flow constructs successfully, + // keeping the lowercase column as an ordinary user column in the schema. + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { + val nonConflictingName = Scd2BatchProcessor.startAtColName.toLowerCase(Locale.ROOT) + val sourceDf = sourceDfWithExtraColumns(nonConflictingName -> StringType) + + val resolvedFlow = newAutoCdcMergeFlow(sourceDf, storedAsScdType = ScdType.Type2) + assert(resolvedFlow.schema.fieldNames.contains(nonConflictingName)) + } + } + // =========================================================================================== // AutoCdcMergeFlow keys-presence validation tests (requireKeysPresentInSelectedSchema) // =========================================================================================== From e8d5e4a962b434821ede3cf9cc3a046a8eead8a8 Mon Sep 17 00:00:00 2001 From: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:30:50 -0700 Subject: [PATCH 10/11] [SPARK-58354][PYTHON] Remove unused ArrowStreamPandasSerializer base class ### What changes were proposed in this pull request? Remove the unused `ArrowStreamPandasSerializer` base class. After all Pandas UDF serializer subclasses were removed (`ArrowStreamPandasUDFSerializer`, `GroupPandasUDFSerializer`, `ArrowStreamAggPandasUDFSerializer`, `ApplyInPandasWithStateSerializer`, `CogroupPandasUDFSerializer`, `TransformWithStateInPandasSerializer`), the base class became an orphan with no subclasses, no instantiations, and no other references anywhere in the Python tree. This also drops the now-orphaned `_normalize_packed` helper (only used by that class) and the imports that are no longer needed. ### Why are the changes needed? Dead code cleanup. The class and its sole helper are unreachable. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing tests. No behavior change. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #57544 from Yicong-Huang/SPARK-58354. Authored-by: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> Signed-off-by: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> --- python/pyspark/sql/pandas/serializers.py | 126 +---------------------- 1 file changed, 1 insertion(+), 125 deletions(-) diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py index 1ac0baad15e59..3b2bb187ee4dc 100644 --- a/python/pyspark/sql/pandas/serializers.py +++ b/python/pyspark/sql/pandas/serializers.py @@ -19,7 +19,7 @@ Serializers for PyArrow and pandas conversions. See `pyspark.serializers` for more details. """ -from typing import IO, TYPE_CHECKING, Iterable, Iterator, List, Optional, Tuple +from typing import IO, TYPE_CHECKING, Iterable, Iterator, List, Tuple from pyspark.errors import PySparkRuntimeError, PySparkValueError from pyspark.serializers import ( @@ -28,34 +28,11 @@ write_int, UTF8Deserializer, ) -from pyspark.sql.conversion import ( - ArrowBatchTransformer, - PandasToArrowConversion, -) -from pyspark.sql.types import ( - DataType, - StructType, - StructField, -) if TYPE_CHECKING: - import pandas as pd import pyarrow as pa -def _normalize_packed(packed): - """ - Normalize UDF output to a uniform tuple-of-tuples form. - - Iterator UDFs yield a single (series, spark_type) tuple directly, - while batched UDFs return a tuple of tuples ((s1, t1), (s2, t2), ...). - This function normalizes both forms to a tuple of tuples. - """ - if len(packed) == 2 and isinstance(packed[1], DataType): - return (packed,) - return tuple(packed) - - class SpecialLengths: END_OF_DATA_SECTION = -1 PYTHON_EXCEPTION_THROWN = -2 @@ -212,104 +189,3 @@ def load_stream( errorClass="INVALID_NUMBER_OF_DATAFRAMES_IN_GROUP", messageParameters={"dataframes_in_group": str(dataframes_in_group)}, ) - - -class ArrowStreamPandasSerializer(ArrowStreamSerializer): - """ - Serializes pandas.Series as Arrow data with Arrow streaming format. - - Parameters - ---------- - timezone : str - A timezone to respect when handling timestamp values - safecheck : bool - If True, conversion from Arrow to Pandas checks for overflow/truncation - int_to_decimal_coercion_enabled : bool - If True, applies additional coercions in Python before converting to Arrow. - This has performance penalties. - prefers_large_types : bool - If True, prefer large Arrow types (e.g., large_string instead of string). - struct_in_pandas : str, optional - How to represent struct in pandas ("dict", "row", etc.). Default is "dict". - ndarray_as_list : bool, optional - Whether to convert ndarray as list. Default is False. - prefer_int_ext_dtype : bool, optional - Whether to convert integers to Pandas ExtensionDType. Default is False. - df_for_struct : bool, optional - If True, convert struct columns to DataFrame instead of Series. Default is False. - """ - - def __init__( - self, - *, - timezone, - safecheck, - int_to_decimal_coercion_enabled: bool = False, - prefers_large_types: bool = False, - struct_in_pandas: str = "dict", - ndarray_as_list: bool = False, - prefer_int_ext_dtype: bool = False, - df_for_struct: bool = False, - input_type: Optional["StructType"] = None, - arrow_cast: bool = False, - ): - super().__init__() - self._timezone = timezone - self._safecheck = safecheck - self._int_to_decimal_coercion_enabled = int_to_decimal_coercion_enabled - self._prefers_large_types = prefers_large_types - self._struct_in_pandas = struct_in_pandas - self._ndarray_as_list = ndarray_as_list - self._prefer_int_ext_dtype = prefer_int_ext_dtype - self._df_for_struct = df_for_struct - if input_type is not None: - assert isinstance(input_type, StructType) - self._input_type = input_type - self._arrow_cast = arrow_cast - - def dump_stream(self, iterator, stream): - """ - Make ArrowRecordBatches from Pandas Series and serialize. - Each element in iterator is: - - For batched UDFs: tuple of (series, spark_type) tuples: ((s1, t1), (s2, t2), ...) - - For iterator UDFs: single (series, spark_type) tuple directly - """ - - def create_batch( - series_tuples: Tuple[Tuple["pd.Series", DataType], ...], - ) -> "pa.RecordBatch": - series_data = [s for s, _ in series_tuples] - types = [t for _, t in series_tuples] - schema = StructType([StructField(f"_{i}", t) for i, t in enumerate(types)]) - return PandasToArrowConversion.convert( - series_data, - schema, - timezone=self._timezone, - safecheck=self._safecheck, - prefers_large_types=self._prefers_large_types, - int_to_decimal_coercion_enabled=self._int_to_decimal_coercion_enabled, - ) - - super().dump_stream( - (create_batch(_normalize_packed(packed)) for packed in iterator), stream - ) - - def load_stream(self, stream): - """ - Deserialize ArrowRecordBatches to an Arrow table and return as a list of pandas.Series. - """ - yield from map( - lambda batch: ArrowBatchTransformer.to_pandas( - batch, - timezone=self._timezone, - schema=self._input_type, - struct_in_pandas=self._struct_in_pandas, - ndarray_as_list=self._ndarray_as_list, - prefer_int_ext_dtype=self._prefer_int_ext_dtype, - df_for_struct=self._df_for_struct, - ), - super().load_stream(stream), - ) - - def __repr__(self): - return "ArrowStreamPandasSerializer" From d369c070f9979b6a4629a34bc208a7fe8e3d1302 Mon Sep 17 00:00:00 2001 From: Ruslan Dautkhanov Date: Mon, 20 Jul 2026 13:11:16 -0600 Subject: [PATCH 11/11] [DRAFT][SPARK-51705][CONNECT][PYTHON] Support SparkSession.broadcast() over Spark Connect Candidate-A pickle lane, Python-only v1: client cloudpickles the value and uploads it via the existing cache/ artifact channel; new CreateBroadcastCommand/UnpersistBroadcastCommand on ExecutePlan materialize a server-side Broadcast[PythonBroadcast] on the live driver SparkContext in SessionHolder; a new PythonUDF.broadcast_ids field (6) is resolved by SparkConnectPlanner into SimplePythonFunction.broadcastVars. Executor/worker path reused unchanged. broadcast_id == driver-side Broadcast.id so the worker's _broadcastRegistry/_from_id resolves it. DRAFT / DO-NOT-MERGE: proto _pb2.py stubs still need regen via dev/connect-gen-protos.sh (buf); BROADCAST_VALUE_TOO_LARGE quota not yet enforced; Scala/ScalarScalaUDF deferred (see companion spike branch). Co-authored-by: Isaac --- .../resources/error/error-conditions.json | 5 + dev/sparktestsupport/modules.py | 1 + .../reference/pyspark.sql/spark_session.rst | 1 + python/pyspark/errors/error-conditions.json | 10 + python/pyspark/sql/connect/broadcast.py | 114 ++++++++ python/pyspark/sql/connect/client/core.py | 23 ++ python/pyspark/sql/connect/expressions.py | 9 + python/pyspark/sql/connect/proto/base_pb2.py | 262 +++++++++--------- python/pyspark/sql/connect/proto/base_pb2.pyi | 32 +++ .../pyspark/sql/connect/proto/commands_pb2.py | 214 +++++++------- .../sql/connect/proto/commands_pb2.pyi | 78 ++++++ .../sql/connect/proto/expressions_pb2.py | 48 ++-- .../sql/connect/proto/expressions_pb2.pyi | 13 + python/pyspark/sql/connect/session.py | 19 ++ python/pyspark/sql/session.py | 35 +++ .../tests/connect/test_connect_broadcast.py | 106 +++++++ .../sql/tests/connect/test_parity_udf.py | 6 +- .../main/protobuf/spark/connect/base.proto | 11 + .../protobuf/spark/connect/commands.proto | 23 ++ .../protobuf/spark/connect/expressions.proto | 5 + .../connect/planner/InvalidInputErrors.scala | 4 + .../connect/planner/SparkConnectPlanner.scala | 109 +++++++- .../sql/connect/service/SessionHolder.scala | 41 +++ .../SparkConnectSessionHolderSuite.scala | 45 ++- 24 files changed, 951 insertions(+), 263 deletions(-) create mode 100644 python/pyspark/sql/connect/broadcast.py create mode 100644 python/pyspark/sql/tests/connect/test_connect_broadcast.py diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index ed111426fe366..7dd73d6efef62 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -1298,6 +1298,11 @@ "" ] }, + "BROADCAST_NOT_FOUND" : { + "message" : [ + "Cannot find a broadcast variable with id: . It may never have been created on this session, may belong to another session, or may have been unpersisted or destroyed." + ] + }, "CANNOT_FIND_CACHED_LOCAL_RELATION" : { "message" : [ "Cannot find a cached local relation for hash: " diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 77aa43eb8d947..8805f42d5308b 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -1167,6 +1167,7 @@ def __hash__(self): # sql unittests "pyspark.sql.tests.connect.test_connect_plan", "pyspark.sql.tests.connect.test_connect_basic", + "pyspark.sql.tests.connect.test_connect_broadcast", "pyspark.sql.tests.connect.test_connect_dataframe_property", "pyspark.sql.tests.connect.test_connect_channel", "pyspark.sql.tests.connect.test_connect_clone_session", diff --git a/python/docs/source/reference/pyspark.sql/spark_session.rst b/python/docs/source/reference/pyspark.sql/spark_session.rst index 9b8dd0ae6564d..4057009096ec0 100644 --- a/python/docs/source/reference/pyspark.sql/spark_session.rst +++ b/python/docs/source/reference/pyspark.sql/spark_session.rst @@ -47,6 +47,7 @@ See also :class:`SparkSession`. SparkSession.addArtifact SparkSession.addArtifacts SparkSession.addTag + SparkSession.broadcast SparkSession.catalog SparkSession.clearTags SparkSession.conf diff --git a/python/pyspark/errors/error-conditions.json b/python/pyspark/errors/error-conditions.json index 07f47a0c91009..fdda2e9e71a48 100644 --- a/python/pyspark/errors/error-conditions.json +++ b/python/pyspark/errors/error-conditions.json @@ -30,6 +30,16 @@ "Length mismatch: Expected axis has elements, new values have elements." ] }, + "BROADCAST_NOT_FOUND": { + "message": [ + "Cannot find a broadcast variable with id: . It may never have been created on this session, may belong to another session, or may have been unpersisted or destroyed." + ] + }, + "BROADCAST_VALUE_TOO_LARGE": { + "message": [ + "The broadcast value of bytes exceeds the maximum allowed size of bytes." + ] + }, "BROADCAST_VARIABLE_NOT_LOADED": { "message": [ "Broadcast variable `` not loaded." diff --git a/python/pyspark/sql/connect/broadcast.py b/python/pyspark/sql/connect/broadcast.py new file mode 100644 index 0000000000000..4bd90d2d43652 --- /dev/null +++ b/python/pyspark/sql/connect/broadcast.py @@ -0,0 +1,114 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +(SPARK-51705) Client-side broadcast proxy for Spark Connect. + +``ConnectBroadcast`` mirrors the classic :class:`pyspark.core.broadcast.Broadcast` pickling +contract so that a UDF closure can reference a broadcast variable transparently. The value was +already uploaded to the server (cloudpickle -> cache artifact -> CreateBroadcastCommand) and a +driver-side broadcast id was returned; this proxy keeps a local copy of the value for driver-side +``.value`` reads, and on pickling emits ``(_from_id, (broadcast_id,))`` -- exactly what the classic +:class:`Broadcast` emits -- so the executor/worker resolves it from its ``_broadcastRegistry`` +keyed by that same id. No JVM is available on the Connect client, so all lifecycle operations are +routed to the server as commands. +""" + +import threading +from typing import Any, Generic, Tuple, TYPE_CHECKING, TypeVar + +# NOTE: This reuses the classic ``pyspark.core.broadcast`` machinery so the pickled UDF closure is +# byte-identical to classic PySpark (``_from_id`` + the driver-side broadcast id). ``pyspark.core`` +# is not available in a Spark Connect-only install (``pyspark-client``); this module is therefore +# imported lazily (from ``SparkSession.broadcast`` and ``PythonUDF.to_plan``), never at package +# import time, so the base Connect package still imports without the classic core present. +from pyspark.core.broadcast import Broadcast, _from_id + +if TYPE_CHECKING: + from pyspark.sql.connect.session import SparkSession + +T = TypeVar("T") + + +class _BroadcastCaptureRegistry(threading.local): + """Thread-local registry of broadcast ids captured while pickling a UDF closure. + + Thread-local is correct because ``CloudPickleSerializer().dumps(...)`` in + ``PythonUDF.to_plan`` and the subsequent drain into ``PythonUDF.broadcast_ids`` happen + synchronously on the same plan-building thread. This mirrors the classic + :class:`pyspark.core.broadcast.BroadcastPickleRegistry`. + """ + + def __init__(self) -> None: + self.__dict__.setdefault("_registry", set()) + + def add(self, bid: int) -> None: + self._registry.add(bid) + + def drain(self) -> "list[int]": + """Return the captured ids and clear the registry (dumps-then-drain-then-clear).""" + captured = list(self._registry) + self._registry.clear() + return captured + + +# Module-level thread-local registry, drained by ``PythonUDF.to_plan``. +_broadcast_capture_registry = _BroadcastCaptureRegistry() + + +class ConnectBroadcast(Broadcast, Generic[T]): + """A broadcast variable created with :meth:`SparkSession.broadcast` over Spark Connect. + + See Also + -------- + pyspark.core.broadcast.Broadcast + """ + + def __init__(self, session: "SparkSession", broadcast_id: int, value: T) -> None: + # Intentionally does NOT call Broadcast.__init__ -- there is no live SparkContext on the + # Connect client. We populate only the attributes needed for driver reads and pickling. + self._session = session + self._broadcast_id = broadcast_id + self._value = value + # Mirror the classic executor-side shape: no JVM broadcast / SparkContext handle. + self._jbroadcast = None + self._sc = None + self._python_broadcast = None + + @property + def value(self) -> T: + """Return the broadcasted value (kept locally on the client for driver-side reads).""" + return self._value + + def unpersist(self, blocking: bool = False) -> None: + """Delete cached copies of this broadcast on the executors via the server.""" + self._session._client._unpersist_broadcast( + self._broadcast_id, blocking=blocking, destroy=False + ) + + def destroy(self, blocking: bool = False) -> None: + """Destroy all data and metadata related to this broadcast via the server.""" + self._session._client._unpersist_broadcast( + self._broadcast_id, blocking=blocking, destroy=True + ) + + def __reduce__(self) -> Tuple[Any, Tuple[int]]: + # Verbatim classic contract: reference the shared classic ``_from_id`` and the driver-side + # broadcast id, so the worker resolves it from ``pyspark.core.broadcast._broadcastRegistry`` + # keyed by the same id. Also side-register the id so ``PythonUDF.to_plan`` can attach it to + # the proto ``broadcast_ids`` field. + _broadcast_capture_registry.add(self._broadcast_id) + return _from_id, (self._broadcast_id,) diff --git a/python/pyspark/sql/connect/client/core.py b/python/pyspark/sql/connect/client/core.py index 19618f2d79681..64445bb56c5a3 100644 --- a/python/pyspark/sql/connect/client/core.py +++ b/python/pyspark/sql/connect/client/core.py @@ -1883,6 +1883,8 @@ def handle_response( if b.HasField("create_resource_profile_command_result"): profile_id = b.create_resource_profile_command_result.profile_id yield {"create_resource_profile_command_result": profile_id} + if b.HasField("create_broadcast_result"): + yield {"create_broadcast_result": b.create_broadcast_result.broadcast_id} if b.HasField("checkpoint_command_result"): yield { "checkpoint_command_result": proto_to_remote_cached_dataframe( @@ -2512,6 +2514,27 @@ def _create_profile(self, profile: pb2.ResourceProfile) -> int: profile_id = properties["create_resource_profile_command_result"] return profile_id + def _create_broadcast(self, artifact_hash: str, size_bytes: int) -> int: + """(SPARK-51705) Create a broadcast variable from an already-uploaded cache artifact and + return the server-assigned (driver-side) broadcast id.""" + logger.debug("Creating a broadcast variable") + cmd = pb2.Command() + cmd.create_broadcast_command.artifact_hash = artifact_hash + cmd.create_broadcast_command.size_bytes = size_bytes + _, properties, _ = self.execute_command(cmd) + return properties["create_broadcast_result"] + + def _unpersist_broadcast( + self, broadcast_id: int, blocking: bool = False, destroy: bool = False + ) -> None: + """(SPARK-51705) Unpersist (or destroy) a broadcast variable created over Connect.""" + logger.debug("Unpersisting a broadcast variable") + cmd = pb2.Command() + cmd.unpersist_broadcast_command.broadcast_id = broadcast_id + cmd.unpersist_broadcast_command.blocking = blocking + cmd.unpersist_broadcast_command.destroy = destroy + self.execute_command(cmd) + def _delete_ml_cache(self, cache_ids: List[str], evict_only: bool = False) -> List[str]: # try best to delete the cache try: diff --git a/python/pyspark/sql/connect/expressions.py b/python/pyspark/sql/connect/expressions.py index 57270398118f7..ae304fac4e8c4 100644 --- a/python/pyspark/sql/connect/expressions.py +++ b/python/pyspark/sql/connect/expressions.py @@ -763,6 +763,15 @@ def to_plan(self, session: "SparkConnectClient") -> proto.PythonUDF: expr.eval_type = self._eval_type expr.command = CloudPickleSerializer().dumps((self._func, output_type)) expr.python_ver = self._python_ver + # (SPARK-51705) Drain the thread-local broadcast capture registry populated while pickling + # the command above (ConnectBroadcast.__reduce__ side-registers its id). This mirrors the + # classic dumps-then-drain-then-clear idiom in rdd._prepare_for_python_RDD, and covers both + # inline UDFs and spark.udf.register since both funnel through this to_plan. + from pyspark.sql.connect.broadcast import _broadcast_capture_registry + + broadcast_ids = _broadcast_capture_registry.drain() + if broadcast_ids: + expr.broadcast_ids.extend(broadcast_ids) return expr def __repr__(self) -> str: diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py index a77c61ca6d2b4..0ec973ce5c56b 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.py +++ b/python/pyspark/sql/connect/proto/base_pb2.py @@ -46,7 +46,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xe3\x03\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommand\x12\\\n\x14\x63ompressed_operation\x18\x03 \x01(\x0b\x32\'.spark.connect.Plan.CompressedOperationH\x00R\x13\x63ompressedOperation\x1a\x8e\x02\n\x13\x43ompressedOperation\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12G\n\x07op_type\x18\x02 \x01(\x0e\x32..spark.connect.Plan.CompressedOperation.OpTypeR\x06opType\x12L\n\x11\x63ompression_codec\x18\x03 \x01(\x0e\x32\x1f.spark.connect.CompressionCodecR\x10\x63ompressionCodec"L\n\x06OpType\x12\x17\n\x13OP_TYPE_UNSPECIFIED\x10\x00\x12\x14\n\x10OP_TYPE_RELATION\x10\x01\x12\x13\n\x0fOP_TYPE_COMMAND\x10\x02\x42\t\n\x07op_type"z\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xf5\x14\n\x12\x41nalyzePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x11 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12\x42\n\x06schema\x18\x04 \x01(\x0b\x32(.spark.connect.AnalyzePlanRequest.SchemaH\x00R\x06schema\x12\x45\n\x07\x65xplain\x18\x05 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.ExplainH\x00R\x07\x65xplain\x12O\n\x0btree_string\x18\x06 \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.TreeStringH\x00R\ntreeString\x12\x46\n\x08is_local\x18\x07 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.IsLocalH\x00R\x07isLocal\x12R\n\x0cis_streaming\x18\x08 \x01(\x0b\x32-.spark.connect.AnalyzePlanRequest.IsStreamingH\x00R\x0bisStreaming\x12O\n\x0binput_files\x18\t \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.InputFilesH\x00R\ninputFiles\x12U\n\rspark_version\x18\n \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SparkVersionH\x00R\x0csparkVersion\x12I\n\tddl_parse\x18\x0b \x01(\x0b\x32*.spark.connect.AnalyzePlanRequest.DDLParseH\x00R\x08\x64\x64lParse\x12X\n\x0esame_semantics\x18\x0c \x01(\x0b\x32/.spark.connect.AnalyzePlanRequest.SameSemanticsH\x00R\rsameSemantics\x12U\n\rsemantic_hash\x18\r \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SemanticHashH\x00R\x0csemanticHash\x12\x45\n\x07persist\x18\x0e \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.PersistH\x00R\x07persist\x12K\n\tunpersist\x18\x0f \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.UnpersistH\x00R\tunpersist\x12_\n\x11get_storage_level\x18\x10 \x01(\x0b\x32\x31.spark.connect.AnalyzePlanRequest.GetStorageLevelH\x00R\x0fgetStorageLevel\x12M\n\x0bjson_to_ddl\x18\x12 \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.JsonToDDLH\x00R\tjsonToDdl\x1a\x31\n\x06Schema\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\xbb\x02\n\x07\x45xplain\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12X\n\x0c\x65xplain_mode\x18\x02 \x01(\x0e\x32\x35.spark.connect.AnalyzePlanRequest.Explain.ExplainModeR\x0b\x65xplainMode"\xac\x01\n\x0b\x45xplainMode\x12\x1c\n\x18\x45XPLAIN_MODE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XPLAIN_MODE_SIMPLE\x10\x01\x12\x19\n\x15\x45XPLAIN_MODE_EXTENDED\x10\x02\x12\x18\n\x14\x45XPLAIN_MODE_CODEGEN\x10\x03\x12\x15\n\x11\x45XPLAIN_MODE_COST\x10\x04\x12\x1a\n\x16\x45XPLAIN_MODE_FORMATTED\x10\x05\x1aZ\n\nTreeString\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12\x19\n\x05level\x18\x02 \x01(\x05H\x00R\x05level\x88\x01\x01\x42\x08\n\x06_level\x1a\x32\n\x07IsLocal\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x36\n\x0bIsStreaming\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x35\n\nInputFiles\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x0e\n\x0cSparkVersion\x1a)\n\x08\x44\x44LParse\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlString\x1ay\n\rSameSemantics\x12\x34\n\x0btarget_plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\ntargetPlan\x12\x32\n\nother_plan\x18\x02 \x01(\x0b\x32\x13.spark.connect.PlanR\totherPlan\x1a\x37\n\x0cSemanticHash\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x97\x01\n\x07Persist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x45\n\rstorage_level\x18\x02 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level\x1an\n\tUnpersist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x1f\n\x08\x62locking\x18\x02 \x01(\x08H\x00R\x08\x62locking\x88\x01\x01\x42\x0b\n\t_blocking\x1a\x46\n\x0fGetStorageLevel\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a,\n\tJsonToDDL\x12\x1f\n\x0bjson_string\x18\x01 \x01(\tR\njsonStringB\t\n\x07\x61nalyzeB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xca\x0e\n\x13\x41nalyzePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12\x43\n\x06schema\x18\x02 \x01(\x0b\x32).spark.connect.AnalyzePlanResponse.SchemaH\x00R\x06schema\x12\x46\n\x07\x65xplain\x18\x03 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.ExplainH\x00R\x07\x65xplain\x12P\n\x0btree_string\x18\x04 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.TreeStringH\x00R\ntreeString\x12G\n\x08is_local\x18\x05 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.IsLocalH\x00R\x07isLocal\x12S\n\x0cis_streaming\x18\x06 \x01(\x0b\x32..spark.connect.AnalyzePlanResponse.IsStreamingH\x00R\x0bisStreaming\x12P\n\x0binput_files\x18\x07 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.InputFilesH\x00R\ninputFiles\x12V\n\rspark_version\x18\x08 \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SparkVersionH\x00R\x0csparkVersion\x12J\n\tddl_parse\x18\t \x01(\x0b\x32+.spark.connect.AnalyzePlanResponse.DDLParseH\x00R\x08\x64\x64lParse\x12Y\n\x0esame_semantics\x18\n \x01(\x0b\x32\x30.spark.connect.AnalyzePlanResponse.SameSemanticsH\x00R\rsameSemantics\x12V\n\rsemantic_hash\x18\x0b \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SemanticHashH\x00R\x0csemanticHash\x12\x46\n\x07persist\x18\x0c \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.PersistH\x00R\x07persist\x12L\n\tunpersist\x18\r \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.UnpersistH\x00R\tunpersist\x12`\n\x11get_storage_level\x18\x0e \x01(\x0b\x32\x32.spark.connect.AnalyzePlanResponse.GetStorageLevelH\x00R\x0fgetStorageLevel\x12N\n\x0bjson_to_ddl\x18\x10 \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.JsonToDDLH\x00R\tjsonToDdl\x1a\x39\n\x06Schema\x12/\n\x06schema\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1a\x30\n\x07\x45xplain\x12%\n\x0e\x65xplain_string\x18\x01 \x01(\tR\rexplainString\x1a-\n\nTreeString\x12\x1f\n\x0btree_string\x18\x01 \x01(\tR\ntreeString\x1a$\n\x07IsLocal\x12\x19\n\x08is_local\x18\x01 \x01(\x08R\x07isLocal\x1a\x30\n\x0bIsStreaming\x12!\n\x0cis_streaming\x18\x01 \x01(\x08R\x0bisStreaming\x1a"\n\nInputFiles\x12\x14\n\x05\x66iles\x18\x01 \x03(\tR\x05\x66iles\x1a(\n\x0cSparkVersion\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\x1a;\n\x08\x44\x44LParse\x12/\n\x06parsed\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06parsed\x1a\'\n\rSameSemantics\x12\x16\n\x06result\x18\x01 \x01(\x08R\x06result\x1a&\n\x0cSemanticHash\x12\x16\n\x06result\x18\x01 \x01(\x05R\x06result\x1a\t\n\x07Persist\x1a\x0b\n\tUnpersist\x1aS\n\x0fGetStorageLevel\x12@\n\rstorage_level\x18\x01 \x01(\x0b\x32\x1b.spark.connect.StorageLevelR\x0cstorageLevel\x1a*\n\tJsonToDDL\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlStringB\x08\n\x06result"\x83\x06\n\x12\x45xecutePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12&\n\x0coperation_id\x18\x06 \x01(\tH\x01R\x0boperationId\x88\x01\x01\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12X\n\x0frequest_options\x18\x05 \x03(\x0b\x32/.spark.connect.ExecutePlanRequest.RequestOptionR\x0erequestOptions\x12\x12\n\x04tags\x18\x07 \x03(\tR\x04tags\x1a\x85\x02\n\rRequestOption\x12K\n\x10reattach_options\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ReattachOptionsH\x00R\x0freattachOptions\x12^\n\x17result_chunking_options\x18\x02 \x01(\x0b\x32$.spark.connect.ResultChunkingOptionsH\x00R\x15resultChunkingOptions\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x10\n\x0erequest_optionB)\n\'_client_observed_server_side_session_idB\x0f\n\r_operation_idB\x0e\n\x0c_client_type"\x87\x1c\n\x13\x45xecutePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12!\n\x0coperation_id\x18\x0c \x01(\tR\x0boperationId\x12\x1f\n\x0bresponse_id\x18\r \x01(\tR\nresponseId\x12P\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32-.spark.connect.ExecutePlanResponse.ArrowBatchH\x00R\narrowBatch\x12\x63\n\x12sql_command_result\x18\x05 \x01(\x0b\x32\x33.spark.connect.ExecutePlanResponse.SqlCommandResultH\x00R\x10sqlCommandResult\x12~\n#write_stream_operation_start_result\x18\x08 \x01(\x0b\x32..spark.connect.WriteStreamOperationStartResultH\x00R\x1fwriteStreamOperationStartResult\x12q\n\x1estreaming_query_command_result\x18\t \x01(\x0b\x32*.spark.connect.StreamingQueryCommandResultH\x00R\x1bstreamingQueryCommandResult\x12k\n\x1cget_resources_command_result\x18\n \x01(\x0b\x32(.spark.connect.GetResourcesCommandResultH\x00R\x19getResourcesCommandResult\x12\x87\x01\n&streaming_query_manager_command_result\x18\x0b \x01(\x0b\x32\x31.spark.connect.StreamingQueryManagerCommandResultH\x00R"streamingQueryManagerCommandResult\x12\x87\x01\n&streaming_query_listener_events_result\x18\x10 \x01(\x0b\x32\x31.spark.connect.StreamingQueryListenerEventsResultH\x00R"streamingQueryListenerEventsResult\x12\\\n\x0fresult_complete\x18\x0e \x01(\x0b\x32\x31.spark.connect.ExecutePlanResponse.ResultCompleteH\x00R\x0eresultComplete\x12\x87\x01\n&create_resource_profile_command_result\x18\x11 \x01(\x0b\x32\x31.spark.connect.CreateResourceProfileCommandResultH\x00R"createResourceProfileCommandResult\x12\x65\n\x12\x65xecution_progress\x18\x12 \x01(\x0b\x32\x34.spark.connect.ExecutePlanResponse.ExecutionProgressH\x00R\x11\x65xecutionProgress\x12\x64\n\x19\x63heckpoint_command_result\x18\x13 \x01(\x0b\x32&.spark.connect.CheckpointCommandResultH\x00R\x17\x63heckpointCommandResult\x12L\n\x11ml_command_result\x18\x14 \x01(\x0b\x32\x1e.spark.connect.MlCommandResultH\x00R\x0fmlCommandResult\x12X\n\x15pipeline_event_result\x18\x15 \x01(\x0b\x32".spark.connect.PipelineEventResultH\x00R\x13pipelineEventResult\x12^\n\x17pipeline_command_result\x18\x16 \x01(\x0b\x32$.spark.connect.PipelineCommandResultH\x00R\x15pipelineCommandResult\x12\x8d\x01\n(pipeline_query_function_execution_signal\x18\x17 \x01(\x0b\x32\x33.spark.connect.PipelineQueryFunctionExecutionSignalH\x00R$pipelineQueryFunctionExecutionSignal\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x44\n\x07metrics\x18\x04 \x01(\x0b\x32*.spark.connect.ExecutePlanResponse.MetricsR\x07metrics\x12]\n\x10observed_metrics\x18\x06 \x03(\x0b\x32\x32.spark.connect.ExecutePlanResponse.ObservedMetricsR\x0fobservedMetrics\x12/\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1aG\n\x10SqlCommandResult\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a\xf8\x01\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x12&\n\x0cstart_offset\x18\x03 \x01(\x03H\x00R\x0bstartOffset\x88\x01\x01\x12$\n\x0b\x63hunk_index\x18\x04 \x01(\x03H\x01R\nchunkIndex\x88\x01\x01\x12\x32\n\x13num_chunks_in_batch\x18\x05 \x01(\x03H\x02R\x10numChunksInBatch\x88\x01\x01\x42\x0f\n\r_start_offsetB\x0e\n\x0c_chunk_indexB\x16\n\x14_num_chunks_in_batch\x1a\x85\x04\n\x07Metrics\x12Q\n\x07metrics\x18\x01 \x03(\x0b\x32\x37.spark.connect.ExecutePlanResponse.Metrics.MetricObjectR\x07metrics\x1a\xcc\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12z\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32M.spark.connect.ExecutePlanResponse.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1a{\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ExecutePlanResponse.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricType\x1a\x93\x02\n\x0fObservedMetrics\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x12\x12\n\x04keys\x18\x03 \x03(\tR\x04keys\x12\x17\n\x07plan_id\x18\x04 \x01(\x03R\x06planId\x12)\n\x0eroot_error_idx\x18\x05 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x06 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrorsB\x11\n\x0f_root_error_idx\x1a\x10\n\x0eResultComplete\x1a\xcd\x02\n\x11\x45xecutionProgress\x12V\n\x06stages\x18\x01 \x03(\x0b\x32>.spark.connect.ExecutePlanResponse.ExecutionProgress.StageInfoR\x06stages\x12,\n\x12num_inflight_tasks\x18\x02 \x01(\x03R\x10numInflightTasks\x1a\xb1\x01\n\tStageInfo\x12\x19\n\x08stage_id\x18\x01 \x01(\x03R\x07stageId\x12\x1b\n\tnum_tasks\x18\x02 \x01(\x03R\x08numTasks\x12.\n\x13num_completed_tasks\x18\x03 \x01(\x03R\x11numCompletedTasks\x12(\n\x10input_bytes_read\x18\x04 \x01(\x03R\x0einputBytesRead\x12\x12\n\x04\x64one\x18\x05 \x01(\x08R\x04\x64oneB\x0f\n\rresponse_type"A\n\x08KeyValue\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x19\n\x05value\x18\x02 \x01(\tH\x00R\x05value\x88\x01\x01\x42\x08\n\x06_value"\xaf\t\n\rConfigRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x44\n\toperation\x18\x03 \x01(\x0b\x32&.spark.connect.ConfigRequest.OperationR\toperation\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x1a\xf2\x03\n\tOperation\x12\x34\n\x03set\x18\x01 \x01(\x0b\x32 .spark.connect.ConfigRequest.SetH\x00R\x03set\x12\x34\n\x03get\x18\x02 \x01(\x0b\x32 .spark.connect.ConfigRequest.GetH\x00R\x03get\x12W\n\x10get_with_default\x18\x03 \x01(\x0b\x32+.spark.connect.ConfigRequest.GetWithDefaultH\x00R\x0egetWithDefault\x12G\n\nget_option\x18\x04 \x01(\x0b\x32&.spark.connect.ConfigRequest.GetOptionH\x00R\tgetOption\x12>\n\x07get_all\x18\x05 \x01(\x0b\x32#.spark.connect.ConfigRequest.GetAllH\x00R\x06getAll\x12:\n\x05unset\x18\x06 \x01(\x0b\x32".spark.connect.ConfigRequest.UnsetH\x00R\x05unset\x12P\n\ris_modifiable\x18\x07 \x01(\x0b\x32).spark.connect.ConfigRequest.IsModifiableH\x00R\x0cisModifiableB\t\n\x07op_type\x1a\\\n\x03Set\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1b\n\x06silent\x18\x02 \x01(\x08H\x00R\x06silent\x88\x01\x01\x42\t\n\x07_silent\x1a\x19\n\x03Get\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a?\n\x0eGetWithDefault\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x1f\n\tGetOption\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a\x30\n\x06GetAll\x12\x1b\n\x06prefix\x18\x01 \x01(\tH\x00R\x06prefix\x88\x01\x01\x42\t\n\x07_prefix\x1a\x1b\n\x05Unset\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a"\n\x0cIsModifiable\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keysB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xaf\x01\n\x0e\x43onfigResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x04 \x01(\tR\x13serverSideSessionId\x12-\n\x05pairs\x18\x02 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1a\n\x08warnings\x18\x03 \x03(\tR\x08warnings"\xea\x07\n\x13\x41\x64\x64\x41rtifactsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12$\n\x0b\x63lient_type\x18\x06 \x01(\tH\x02R\nclientType\x88\x01\x01\x12@\n\x05\x62\x61tch\x18\x03 \x01(\x0b\x32(.spark.connect.AddArtifactsRequest.BatchH\x00R\x05\x62\x61tch\x12Z\n\x0b\x62\x65gin_chunk\x18\x04 \x01(\x0b\x32\x37.spark.connect.AddArtifactsRequest.BeginChunkedArtifactH\x00R\nbeginChunk\x12H\n\x05\x63hunk\x18\x05 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkH\x00R\x05\x63hunk\x1a\x35\n\rArtifactChunk\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12\x10\n\x03\x63rc\x18\x02 \x01(\x03R\x03\x63rc\x1ao\n\x13SingleChunkArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x04\x64\x61ta\x1a]\n\x05\x42\x61tch\x12T\n\tartifacts\x18\x01 \x03(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactR\tartifacts\x1a\xc1\x01\n\x14\x42\x65ginChunkedArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0btotal_bytes\x18\x02 \x01(\x03R\ntotalBytes\x12\x1d\n\nnum_chunks\x18\x03 \x01(\x03R\tnumChunks\x12U\n\rinitial_chunk\x18\x04 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x0cinitialChunkB\t\n\x07payloadB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x02\n\x14\x41\x64\x64\x41rtifactsResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\tartifacts\x18\x01 \x03(\x0b\x32\x33.spark.connect.AddArtifactsResponse.ArtifactSummaryR\tartifacts\x1aQ\n\x0f\x41rtifactSummary\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x11is_crc_successful\x18\x02 \x01(\x08R\x0fisCrcSuccessful"\xc6\x02\n\x17\x41rtifactStatusesRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12\x14\n\x05names\x18\x04 \x03(\tR\x05namesB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xe0\x02\n\x18\x41rtifactStatusesResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\x08statuses\x18\x01 \x03(\x0b\x32\x35.spark.connect.ArtifactStatusesResponse.StatusesEntryR\x08statuses\x1as\n\rStatusesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ArtifactStatusesResponse.ArtifactStatusR\x05value:\x02\x38\x01\x1a(\n\x0e\x41rtifactStatus\x12\x16\n\x06\x65xists\x18\x01 \x01(\x08R\x06\x65xists"\xdb\x04\n\x10InterruptRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12T\n\x0einterrupt_type\x18\x04 \x01(\x0e\x32-.spark.connect.InterruptRequest.InterruptTypeR\rinterruptType\x12%\n\roperation_tag\x18\x05 \x01(\tH\x00R\x0coperationTag\x12#\n\x0coperation_id\x18\x06 \x01(\tH\x00R\x0boperationId"\x80\x01\n\rInterruptType\x12\x1e\n\x1aINTERRUPT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12INTERRUPT_TYPE_ALL\x10\x01\x12\x16\n\x12INTERRUPT_TYPE_TAG\x10\x02\x12\x1f\n\x1bINTERRUPT_TYPE_OPERATION_ID\x10\x03\x42\x0b\n\tinterruptB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x01\n\x11InterruptResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\'\n\x0finterrupted_ids\x18\x02 \x03(\tR\x0einterruptedIds"5\n\x0fReattachOptions\x12"\n\x0creattachable\x18\x01 \x01(\x08R\x0creattachable"\xb5\x01\n\x15ResultChunkingOptions\x12;\n\x1a\x61llow_arrow_batch_chunking\x18\x01 \x01(\x08R\x17\x61llowArrowBatchChunking\x12@\n\x1apreferred_arrow_chunk_size\x18\x02 \x01(\x03H\x00R\x17preferredArrowChunkSize\x88\x01\x01\x42\x1d\n\x1b_preferred_arrow_chunk_size"\x96\x03\n\x16ReattachExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x06 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x12-\n\x10last_response_id\x18\x05 \x01(\tH\x02R\x0elastResponseId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x13\n\x11_last_response_id"\xc9\x04\n\x15ReleaseExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12R\n\x0brelease_all\x18\x05 \x01(\x0b\x32/.spark.connect.ReleaseExecuteRequest.ReleaseAllH\x00R\nreleaseAll\x12X\n\rrelease_until\x18\x06 \x01(\x0b\x32\x31.spark.connect.ReleaseExecuteRequest.ReleaseUntilH\x00R\x0creleaseUntil\x1a\x0c\n\nReleaseAll\x1a/\n\x0cReleaseUntil\x12\x1f\n\x0bresponse_id\x18\x01 \x01(\tR\nresponseIdB\t\n\x07releaseB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x01\n\x16ReleaseExecuteResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12&\n\x0coperation_id\x18\x02 \x01(\tH\x00R\x0boperationId\x88\x01\x01\x42\x0f\n\r_operation_id"\xd4\x01\n\x15ReleaseSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12\'\n\x0f\x61llow_reconnect\x18\x04 \x01(\x08R\x0e\x61llowReconnectB\x0e\n\x0c_client_type"l\n\x16ReleaseSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId"\xcc\x02\n\x18\x46\x65tchErrorDetailsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x19\n\x08\x65rror_id\x18\x03 \x01(\tR\x07\x65rrorId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xd9\x0f\n\x19\x46\x65tchErrorDetailsResponse\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\x1d\n\nsession_id\x18\x04 \x01(\tR\tsessionId\x12)\n\x0eroot_error_idx\x18\x01 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x02 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrors\x1a\xae\x01\n\x11StackTraceElement\x12\'\n\x0f\x64\x65\x63laring_class\x18\x01 \x01(\tR\x0e\x64\x65\x63laringClass\x12\x1f\n\x0bmethod_name\x18\x02 \x01(\tR\nmethodName\x12 \n\tfile_name\x18\x03 \x01(\tH\x00R\x08\x66ileName\x88\x01\x01\x12\x1f\n\x0bline_number\x18\x04 \x01(\x05R\nlineNumberB\x0c\n\n_file_name\x1a\xf0\x02\n\x0cQueryContext\x12\x64\n\x0c\x63ontext_type\x18\n \x01(\x0e\x32\x41.spark.connect.FetchErrorDetailsResponse.QueryContext.ContextTypeR\x0b\x63ontextType\x12\x1f\n\x0bobject_type\x18\x01 \x01(\tR\nobjectType\x12\x1f\n\x0bobject_name\x18\x02 \x01(\tR\nobjectName\x12\x1f\n\x0bstart_index\x18\x03 \x01(\x05R\nstartIndex\x12\x1d\n\nstop_index\x18\x04 \x01(\x05R\tstopIndex\x12\x1a\n\x08\x66ragment\x18\x05 \x01(\tR\x08\x66ragment\x12\x1b\n\tcall_site\x18\x06 \x01(\tR\x08\x63\x61llSite\x12\x18\n\x07summary\x18\x07 \x01(\tR\x07summary"%\n\x0b\x43ontextType\x12\x07\n\x03SQL\x10\x00\x12\r\n\tDATAFRAME\x10\x01\x1a\xa6\x04\n\x0eSparkThrowable\x12$\n\x0b\x65rror_class\x18\x01 \x01(\tH\x00R\nerrorClass\x88\x01\x01\x12}\n\x12message_parameters\x18\x02 \x03(\x0b\x32N.spark.connect.FetchErrorDetailsResponse.SparkThrowable.MessageParametersEntryR\x11messageParameters\x12\\\n\x0equery_contexts\x18\x03 \x03(\x0b\x32\x35.spark.connect.FetchErrorDetailsResponse.QueryContextR\rqueryContexts\x12 \n\tsql_state\x18\x04 \x01(\tH\x01R\x08sqlState\x88\x01\x01\x12r\n\x14\x62reaking_change_info\x18\x05 \x01(\x0b\x32;.spark.connect.FetchErrorDetailsResponse.BreakingChangeInfoH\x02R\x12\x62reakingChangeInfo\x88\x01\x01\x1a\x44\n\x16MessageParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_error_classB\x0c\n\n_sql_stateB\x17\n\x15_breaking_change_info\x1a\xfa\x01\n\x12\x42reakingChangeInfo\x12+\n\x11migration_message\x18\x01 \x03(\tR\x10migrationMessage\x12k\n\x11mitigation_config\x18\x02 \x01(\x0b\x32\x39.spark.connect.FetchErrorDetailsResponse.MitigationConfigH\x00R\x10mitigationConfig\x88\x01\x01\x12$\n\x0bneeds_audit\x18\x03 \x01(\x08H\x01R\nneedsAudit\x88\x01\x01\x42\x14\n\x12_mitigation_configB\x0e\n\x0c_needs_audit\x1a:\n\x10MitigationConfig\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\x1a\xdb\x02\n\x05\x45rror\x12\x30\n\x14\x65rror_type_hierarchy\x18\x01 \x03(\tR\x12\x65rrorTypeHierarchy\x12\x18\n\x07message\x18\x02 \x01(\tR\x07message\x12[\n\x0bstack_trace\x18\x03 \x03(\x0b\x32:.spark.connect.FetchErrorDetailsResponse.StackTraceElementR\nstackTrace\x12 \n\tcause_idx\x18\x04 \x01(\x05H\x00R\x08\x63\x61useIdx\x88\x01\x01\x12\x65\n\x0fspark_throwable\x18\x05 \x01(\x0b\x32\x37.spark.connect.FetchErrorDetailsResponse.SparkThrowableH\x01R\x0esparkThrowable\x88\x01\x01\x42\x0c\n\n_cause_idxB\x12\n\x10_spark_throwableB\x11\n\x0f_root_error_idx"Z\n\x17\x43heckpointCommandResult\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation"\xea\x02\n\x13\x43loneSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12)\n\x0enew_session_id\x18\x04 \x01(\tH\x02R\x0cnewSessionId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x11\n\x0f_new_session_id"\xcc\x01\n\x14\x43loneSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12$\n\x0enew_session_id\x18\x03 \x01(\tR\x0cnewSessionId\x12:\n\x1anew_server_side_session_id\x18\x04 \x01(\tR\x16newServerSideSessionId"\xd3\x04\n\x10GetStatusRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12V\n&client_observed_server_side_session_id\x18\x04 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12\x66\n\x10operation_status\x18\x05 \x01(\x0b\x32\x36.spark.connect.GetStatusRequest.OperationStatusRequestH\x02R\x0foperationStatus\x88\x01\x01\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1at\n\x16OperationStatusRequest\x12#\n\roperation_ids\x18\x01 \x03(\tR\x0coperationIds\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensionsB\x0e\n\x0c_client_typeB)\n\'_client_observed_server_side_session_idB\x13\n\x11_operation_status"\xad\x05\n\x11GetStatusResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12_\n\x12operation_statuses\x18\x03 \x03(\x0b\x32\x30.spark.connect.GetStatusResponse.OperationStatusR\x11operationStatuses\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1a\xab\x03\n\x0fOperationStatus\x12!\n\x0coperation_id\x18\x01 \x01(\tR\x0boperationId\x12U\n\x05state\x18\x02 \x01(\x0e\x32?.spark.connect.GetStatusResponse.OperationStatus.OperationStateR\x05state\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xe6\x01\n\x0eOperationState\x12\x1f\n\x1bOPERATION_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17OPERATION_STATE_UNKNOWN\x10\x01\x12\x1b\n\x17OPERATION_STATE_RUNNING\x10\x02\x12\x1f\n\x1bOPERATION_STATE_TERMINATING\x10\x03\x12\x1d\n\x19OPERATION_STATE_SUCCEEDED\x10\x04\x12\x1a\n\x16OPERATION_STATE_FAILED\x10\x05\x12\x1d\n\x19OPERATION_STATE_CANCELLED\x10\x06*Q\n\x10\x43ompressionCodec\x12!\n\x1d\x43OMPRESSION_CODEC_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x43OMPRESSION_CODEC_ZSTD\x10\x01\x32\xdf\x08\n\x13SparkConnectService\x12X\n\x0b\x45xecutePlan\x12!.spark.connect.ExecutePlanRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12V\n\x0b\x41nalyzePlan\x12!.spark.connect.AnalyzePlanRequest\x1a".spark.connect.AnalyzePlanResponse"\x00\x12G\n\x06\x43onfig\x12\x1c.spark.connect.ConfigRequest\x1a\x1d.spark.connect.ConfigResponse"\x00\x12[\n\x0c\x41\x64\x64\x41rtifacts\x12".spark.connect.AddArtifactsRequest\x1a#.spark.connect.AddArtifactsResponse"\x00(\x01\x12\x63\n\x0e\x41rtifactStatus\x12&.spark.connect.ArtifactStatusesRequest\x1a\'.spark.connect.ArtifactStatusesResponse"\x00\x12P\n\tInterrupt\x12\x1f.spark.connect.InterruptRequest\x1a .spark.connect.InterruptResponse"\x00\x12`\n\x0fReattachExecute\x12%.spark.connect.ReattachExecuteRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12_\n\x0eReleaseExecute\x12$.spark.connect.ReleaseExecuteRequest\x1a%.spark.connect.ReleaseExecuteResponse"\x00\x12_\n\x0eReleaseSession\x12$.spark.connect.ReleaseSessionRequest\x1a%.spark.connect.ReleaseSessionResponse"\x00\x12h\n\x11\x46\x65tchErrorDetails\x12\'.spark.connect.FetchErrorDetailsRequest\x1a(.spark.connect.FetchErrorDetailsResponse"\x00\x12Y\n\x0c\x43loneSession\x12".spark.connect.CloneSessionRequest\x1a#.spark.connect.CloneSessionResponse"\x00\x12P\n\tGetStatus\x12\x1f.spark.connect.GetStatusRequest\x1a .spark.connect.GetStatusResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xe3\x03\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommand\x12\\\n\x14\x63ompressed_operation\x18\x03 \x01(\x0b\x32\'.spark.connect.Plan.CompressedOperationH\x00R\x13\x63ompressedOperation\x1a\x8e\x02\n\x13\x43ompressedOperation\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12G\n\x07op_type\x18\x02 \x01(\x0e\x32..spark.connect.Plan.CompressedOperation.OpTypeR\x06opType\x12L\n\x11\x63ompression_codec\x18\x03 \x01(\x0e\x32\x1f.spark.connect.CompressionCodecR\x10\x63ompressionCodec"L\n\x06OpType\x12\x17\n\x13OP_TYPE_UNSPECIFIED\x10\x00\x12\x14\n\x10OP_TYPE_RELATION\x10\x01\x12\x13\n\x0fOP_TYPE_COMMAND\x10\x02\x42\t\n\x07op_type"z\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xf5\x14\n\x12\x41nalyzePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x11 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12\x42\n\x06schema\x18\x04 \x01(\x0b\x32(.spark.connect.AnalyzePlanRequest.SchemaH\x00R\x06schema\x12\x45\n\x07\x65xplain\x18\x05 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.ExplainH\x00R\x07\x65xplain\x12O\n\x0btree_string\x18\x06 \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.TreeStringH\x00R\ntreeString\x12\x46\n\x08is_local\x18\x07 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.IsLocalH\x00R\x07isLocal\x12R\n\x0cis_streaming\x18\x08 \x01(\x0b\x32-.spark.connect.AnalyzePlanRequest.IsStreamingH\x00R\x0bisStreaming\x12O\n\x0binput_files\x18\t \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.InputFilesH\x00R\ninputFiles\x12U\n\rspark_version\x18\n \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SparkVersionH\x00R\x0csparkVersion\x12I\n\tddl_parse\x18\x0b \x01(\x0b\x32*.spark.connect.AnalyzePlanRequest.DDLParseH\x00R\x08\x64\x64lParse\x12X\n\x0esame_semantics\x18\x0c \x01(\x0b\x32/.spark.connect.AnalyzePlanRequest.SameSemanticsH\x00R\rsameSemantics\x12U\n\rsemantic_hash\x18\r \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SemanticHashH\x00R\x0csemanticHash\x12\x45\n\x07persist\x18\x0e \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.PersistH\x00R\x07persist\x12K\n\tunpersist\x18\x0f \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.UnpersistH\x00R\tunpersist\x12_\n\x11get_storage_level\x18\x10 \x01(\x0b\x32\x31.spark.connect.AnalyzePlanRequest.GetStorageLevelH\x00R\x0fgetStorageLevel\x12M\n\x0bjson_to_ddl\x18\x12 \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.JsonToDDLH\x00R\tjsonToDdl\x1a\x31\n\x06Schema\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\xbb\x02\n\x07\x45xplain\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12X\n\x0c\x65xplain_mode\x18\x02 \x01(\x0e\x32\x35.spark.connect.AnalyzePlanRequest.Explain.ExplainModeR\x0b\x65xplainMode"\xac\x01\n\x0b\x45xplainMode\x12\x1c\n\x18\x45XPLAIN_MODE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XPLAIN_MODE_SIMPLE\x10\x01\x12\x19\n\x15\x45XPLAIN_MODE_EXTENDED\x10\x02\x12\x18\n\x14\x45XPLAIN_MODE_CODEGEN\x10\x03\x12\x15\n\x11\x45XPLAIN_MODE_COST\x10\x04\x12\x1a\n\x16\x45XPLAIN_MODE_FORMATTED\x10\x05\x1aZ\n\nTreeString\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12\x19\n\x05level\x18\x02 \x01(\x05H\x00R\x05level\x88\x01\x01\x42\x08\n\x06_level\x1a\x32\n\x07IsLocal\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x36\n\x0bIsStreaming\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x35\n\nInputFiles\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x0e\n\x0cSparkVersion\x1a)\n\x08\x44\x44LParse\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlString\x1ay\n\rSameSemantics\x12\x34\n\x0btarget_plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\ntargetPlan\x12\x32\n\nother_plan\x18\x02 \x01(\x0b\x32\x13.spark.connect.PlanR\totherPlan\x1a\x37\n\x0cSemanticHash\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x97\x01\n\x07Persist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x45\n\rstorage_level\x18\x02 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level\x1an\n\tUnpersist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x1f\n\x08\x62locking\x18\x02 \x01(\x08H\x00R\x08\x62locking\x88\x01\x01\x42\x0b\n\t_blocking\x1a\x46\n\x0fGetStorageLevel\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a,\n\tJsonToDDL\x12\x1f\n\x0bjson_string\x18\x01 \x01(\tR\njsonStringB\t\n\x07\x61nalyzeB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xca\x0e\n\x13\x41nalyzePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12\x43\n\x06schema\x18\x02 \x01(\x0b\x32).spark.connect.AnalyzePlanResponse.SchemaH\x00R\x06schema\x12\x46\n\x07\x65xplain\x18\x03 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.ExplainH\x00R\x07\x65xplain\x12P\n\x0btree_string\x18\x04 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.TreeStringH\x00R\ntreeString\x12G\n\x08is_local\x18\x05 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.IsLocalH\x00R\x07isLocal\x12S\n\x0cis_streaming\x18\x06 \x01(\x0b\x32..spark.connect.AnalyzePlanResponse.IsStreamingH\x00R\x0bisStreaming\x12P\n\x0binput_files\x18\x07 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.InputFilesH\x00R\ninputFiles\x12V\n\rspark_version\x18\x08 \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SparkVersionH\x00R\x0csparkVersion\x12J\n\tddl_parse\x18\t \x01(\x0b\x32+.spark.connect.AnalyzePlanResponse.DDLParseH\x00R\x08\x64\x64lParse\x12Y\n\x0esame_semantics\x18\n \x01(\x0b\x32\x30.spark.connect.AnalyzePlanResponse.SameSemanticsH\x00R\rsameSemantics\x12V\n\rsemantic_hash\x18\x0b \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SemanticHashH\x00R\x0csemanticHash\x12\x46\n\x07persist\x18\x0c \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.PersistH\x00R\x07persist\x12L\n\tunpersist\x18\r \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.UnpersistH\x00R\tunpersist\x12`\n\x11get_storage_level\x18\x0e \x01(\x0b\x32\x32.spark.connect.AnalyzePlanResponse.GetStorageLevelH\x00R\x0fgetStorageLevel\x12N\n\x0bjson_to_ddl\x18\x10 \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.JsonToDDLH\x00R\tjsonToDdl\x1a\x39\n\x06Schema\x12/\n\x06schema\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1a\x30\n\x07\x45xplain\x12%\n\x0e\x65xplain_string\x18\x01 \x01(\tR\rexplainString\x1a-\n\nTreeString\x12\x1f\n\x0btree_string\x18\x01 \x01(\tR\ntreeString\x1a$\n\x07IsLocal\x12\x19\n\x08is_local\x18\x01 \x01(\x08R\x07isLocal\x1a\x30\n\x0bIsStreaming\x12!\n\x0cis_streaming\x18\x01 \x01(\x08R\x0bisStreaming\x1a"\n\nInputFiles\x12\x14\n\x05\x66iles\x18\x01 \x03(\tR\x05\x66iles\x1a(\n\x0cSparkVersion\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\x1a;\n\x08\x44\x44LParse\x12/\n\x06parsed\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06parsed\x1a\'\n\rSameSemantics\x12\x16\n\x06result\x18\x01 \x01(\x08R\x06result\x1a&\n\x0cSemanticHash\x12\x16\n\x06result\x18\x01 \x01(\x05R\x06result\x1a\t\n\x07Persist\x1a\x0b\n\tUnpersist\x1aS\n\x0fGetStorageLevel\x12@\n\rstorage_level\x18\x01 \x01(\x0b\x32\x1b.spark.connect.StorageLevelR\x0cstorageLevel\x1a*\n\tJsonToDDL\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlStringB\x08\n\x06result"\x83\x06\n\x12\x45xecutePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12&\n\x0coperation_id\x18\x06 \x01(\tH\x01R\x0boperationId\x88\x01\x01\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12X\n\x0frequest_options\x18\x05 \x03(\x0b\x32/.spark.connect.ExecutePlanRequest.RequestOptionR\x0erequestOptions\x12\x12\n\x04tags\x18\x07 \x03(\tR\x04tags\x1a\x85\x02\n\rRequestOption\x12K\n\x10reattach_options\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ReattachOptionsH\x00R\x0freattachOptions\x12^\n\x17result_chunking_options\x18\x02 \x01(\x0b\x32$.spark.connect.ResultChunkingOptionsH\x00R\x15resultChunkingOptions\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x10\n\x0erequest_optionB)\n\'_client_observed_server_side_session_idB\x0f\n\r_operation_idB\x0e\n\x0c_client_type"\xe7\x1c\n\x13\x45xecutePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12!\n\x0coperation_id\x18\x0c \x01(\tR\x0boperationId\x12\x1f\n\x0bresponse_id\x18\r \x01(\tR\nresponseId\x12P\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32-.spark.connect.ExecutePlanResponse.ArrowBatchH\x00R\narrowBatch\x12\x63\n\x12sql_command_result\x18\x05 \x01(\x0b\x32\x33.spark.connect.ExecutePlanResponse.SqlCommandResultH\x00R\x10sqlCommandResult\x12~\n#write_stream_operation_start_result\x18\x08 \x01(\x0b\x32..spark.connect.WriteStreamOperationStartResultH\x00R\x1fwriteStreamOperationStartResult\x12q\n\x1estreaming_query_command_result\x18\t \x01(\x0b\x32*.spark.connect.StreamingQueryCommandResultH\x00R\x1bstreamingQueryCommandResult\x12k\n\x1cget_resources_command_result\x18\n \x01(\x0b\x32(.spark.connect.GetResourcesCommandResultH\x00R\x19getResourcesCommandResult\x12\x87\x01\n&streaming_query_manager_command_result\x18\x0b \x01(\x0b\x32\x31.spark.connect.StreamingQueryManagerCommandResultH\x00R"streamingQueryManagerCommandResult\x12\x87\x01\n&streaming_query_listener_events_result\x18\x10 \x01(\x0b\x32\x31.spark.connect.StreamingQueryListenerEventsResultH\x00R"streamingQueryListenerEventsResult\x12\\\n\x0fresult_complete\x18\x0e \x01(\x0b\x32\x31.spark.connect.ExecutePlanResponse.ResultCompleteH\x00R\x0eresultComplete\x12\x87\x01\n&create_resource_profile_command_result\x18\x11 \x01(\x0b\x32\x31.spark.connect.CreateResourceProfileCommandResultH\x00R"createResourceProfileCommandResult\x12\x65\n\x12\x65xecution_progress\x18\x12 \x01(\x0b\x32\x34.spark.connect.ExecutePlanResponse.ExecutionProgressH\x00R\x11\x65xecutionProgress\x12\x64\n\x19\x63heckpoint_command_result\x18\x13 \x01(\x0b\x32&.spark.connect.CheckpointCommandResultH\x00R\x17\x63heckpointCommandResult\x12L\n\x11ml_command_result\x18\x14 \x01(\x0b\x32\x1e.spark.connect.MlCommandResultH\x00R\x0fmlCommandResult\x12X\n\x15pipeline_event_result\x18\x15 \x01(\x0b\x32".spark.connect.PipelineEventResultH\x00R\x13pipelineEventResult\x12^\n\x17pipeline_command_result\x18\x16 \x01(\x0b\x32$.spark.connect.PipelineCommandResultH\x00R\x15pipelineCommandResult\x12\x8d\x01\n(pipeline_query_function_execution_signal\x18\x17 \x01(\x0b\x32\x33.spark.connect.PipelineQueryFunctionExecutionSignalH\x00R$pipelineQueryFunctionExecutionSignal\x12^\n\x17\x63reate_broadcast_result\x18\x18 \x01(\x0b\x32$.spark.connect.CreateBroadcastResultH\x00R\x15\x63reateBroadcastResult\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x44\n\x07metrics\x18\x04 \x01(\x0b\x32*.spark.connect.ExecutePlanResponse.MetricsR\x07metrics\x12]\n\x10observed_metrics\x18\x06 \x03(\x0b\x32\x32.spark.connect.ExecutePlanResponse.ObservedMetricsR\x0fobservedMetrics\x12/\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1aG\n\x10SqlCommandResult\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a\xf8\x01\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x12&\n\x0cstart_offset\x18\x03 \x01(\x03H\x00R\x0bstartOffset\x88\x01\x01\x12$\n\x0b\x63hunk_index\x18\x04 \x01(\x03H\x01R\nchunkIndex\x88\x01\x01\x12\x32\n\x13num_chunks_in_batch\x18\x05 \x01(\x03H\x02R\x10numChunksInBatch\x88\x01\x01\x42\x0f\n\r_start_offsetB\x0e\n\x0c_chunk_indexB\x16\n\x14_num_chunks_in_batch\x1a\x85\x04\n\x07Metrics\x12Q\n\x07metrics\x18\x01 \x03(\x0b\x32\x37.spark.connect.ExecutePlanResponse.Metrics.MetricObjectR\x07metrics\x1a\xcc\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12z\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32M.spark.connect.ExecutePlanResponse.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1a{\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ExecutePlanResponse.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricType\x1a\x93\x02\n\x0fObservedMetrics\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x12\x12\n\x04keys\x18\x03 \x03(\tR\x04keys\x12\x17\n\x07plan_id\x18\x04 \x01(\x03R\x06planId\x12)\n\x0eroot_error_idx\x18\x05 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x06 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrorsB\x11\n\x0f_root_error_idx\x1a\x10\n\x0eResultComplete\x1a\xcd\x02\n\x11\x45xecutionProgress\x12V\n\x06stages\x18\x01 \x03(\x0b\x32>.spark.connect.ExecutePlanResponse.ExecutionProgress.StageInfoR\x06stages\x12,\n\x12num_inflight_tasks\x18\x02 \x01(\x03R\x10numInflightTasks\x1a\xb1\x01\n\tStageInfo\x12\x19\n\x08stage_id\x18\x01 \x01(\x03R\x07stageId\x12\x1b\n\tnum_tasks\x18\x02 \x01(\x03R\x08numTasks\x12.\n\x13num_completed_tasks\x18\x03 \x01(\x03R\x11numCompletedTasks\x12(\n\x10input_bytes_read\x18\x04 \x01(\x03R\x0einputBytesRead\x12\x12\n\x04\x64one\x18\x05 \x01(\x08R\x04\x64oneB\x0f\n\rresponse_type"A\n\x08KeyValue\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x19\n\x05value\x18\x02 \x01(\tH\x00R\x05value\x88\x01\x01\x42\x08\n\x06_value"\xaf\t\n\rConfigRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x44\n\toperation\x18\x03 \x01(\x0b\x32&.spark.connect.ConfigRequest.OperationR\toperation\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x1a\xf2\x03\n\tOperation\x12\x34\n\x03set\x18\x01 \x01(\x0b\x32 .spark.connect.ConfigRequest.SetH\x00R\x03set\x12\x34\n\x03get\x18\x02 \x01(\x0b\x32 .spark.connect.ConfigRequest.GetH\x00R\x03get\x12W\n\x10get_with_default\x18\x03 \x01(\x0b\x32+.spark.connect.ConfigRequest.GetWithDefaultH\x00R\x0egetWithDefault\x12G\n\nget_option\x18\x04 \x01(\x0b\x32&.spark.connect.ConfigRequest.GetOptionH\x00R\tgetOption\x12>\n\x07get_all\x18\x05 \x01(\x0b\x32#.spark.connect.ConfigRequest.GetAllH\x00R\x06getAll\x12:\n\x05unset\x18\x06 \x01(\x0b\x32".spark.connect.ConfigRequest.UnsetH\x00R\x05unset\x12P\n\ris_modifiable\x18\x07 \x01(\x0b\x32).spark.connect.ConfigRequest.IsModifiableH\x00R\x0cisModifiableB\t\n\x07op_type\x1a\\\n\x03Set\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1b\n\x06silent\x18\x02 \x01(\x08H\x00R\x06silent\x88\x01\x01\x42\t\n\x07_silent\x1a\x19\n\x03Get\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a?\n\x0eGetWithDefault\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x1f\n\tGetOption\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a\x30\n\x06GetAll\x12\x1b\n\x06prefix\x18\x01 \x01(\tH\x00R\x06prefix\x88\x01\x01\x42\t\n\x07_prefix\x1a\x1b\n\x05Unset\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a"\n\x0cIsModifiable\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keysB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xaf\x01\n\x0e\x43onfigResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x04 \x01(\tR\x13serverSideSessionId\x12-\n\x05pairs\x18\x02 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1a\n\x08warnings\x18\x03 \x03(\tR\x08warnings"\xea\x07\n\x13\x41\x64\x64\x41rtifactsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12$\n\x0b\x63lient_type\x18\x06 \x01(\tH\x02R\nclientType\x88\x01\x01\x12@\n\x05\x62\x61tch\x18\x03 \x01(\x0b\x32(.spark.connect.AddArtifactsRequest.BatchH\x00R\x05\x62\x61tch\x12Z\n\x0b\x62\x65gin_chunk\x18\x04 \x01(\x0b\x32\x37.spark.connect.AddArtifactsRequest.BeginChunkedArtifactH\x00R\nbeginChunk\x12H\n\x05\x63hunk\x18\x05 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkH\x00R\x05\x63hunk\x1a\x35\n\rArtifactChunk\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12\x10\n\x03\x63rc\x18\x02 \x01(\x03R\x03\x63rc\x1ao\n\x13SingleChunkArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x04\x64\x61ta\x1a]\n\x05\x42\x61tch\x12T\n\tartifacts\x18\x01 \x03(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactR\tartifacts\x1a\xc1\x01\n\x14\x42\x65ginChunkedArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0btotal_bytes\x18\x02 \x01(\x03R\ntotalBytes\x12\x1d\n\nnum_chunks\x18\x03 \x01(\x03R\tnumChunks\x12U\n\rinitial_chunk\x18\x04 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x0cinitialChunkB\t\n\x07payloadB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x02\n\x14\x41\x64\x64\x41rtifactsResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\tartifacts\x18\x01 \x03(\x0b\x32\x33.spark.connect.AddArtifactsResponse.ArtifactSummaryR\tartifacts\x1aQ\n\x0f\x41rtifactSummary\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x11is_crc_successful\x18\x02 \x01(\x08R\x0fisCrcSuccessful"\xc6\x02\n\x17\x41rtifactStatusesRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12\x14\n\x05names\x18\x04 \x03(\tR\x05namesB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xe0\x02\n\x18\x41rtifactStatusesResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\x08statuses\x18\x01 \x03(\x0b\x32\x35.spark.connect.ArtifactStatusesResponse.StatusesEntryR\x08statuses\x1as\n\rStatusesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ArtifactStatusesResponse.ArtifactStatusR\x05value:\x02\x38\x01\x1a(\n\x0e\x41rtifactStatus\x12\x16\n\x06\x65xists\x18\x01 \x01(\x08R\x06\x65xists"\xdb\x04\n\x10InterruptRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12T\n\x0einterrupt_type\x18\x04 \x01(\x0e\x32-.spark.connect.InterruptRequest.InterruptTypeR\rinterruptType\x12%\n\roperation_tag\x18\x05 \x01(\tH\x00R\x0coperationTag\x12#\n\x0coperation_id\x18\x06 \x01(\tH\x00R\x0boperationId"\x80\x01\n\rInterruptType\x12\x1e\n\x1aINTERRUPT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12INTERRUPT_TYPE_ALL\x10\x01\x12\x16\n\x12INTERRUPT_TYPE_TAG\x10\x02\x12\x1f\n\x1bINTERRUPT_TYPE_OPERATION_ID\x10\x03\x42\x0b\n\tinterruptB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x01\n\x11InterruptResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\'\n\x0finterrupted_ids\x18\x02 \x03(\tR\x0einterruptedIds"5\n\x0fReattachOptions\x12"\n\x0creattachable\x18\x01 \x01(\x08R\x0creattachable"\xb5\x01\n\x15ResultChunkingOptions\x12;\n\x1a\x61llow_arrow_batch_chunking\x18\x01 \x01(\x08R\x17\x61llowArrowBatchChunking\x12@\n\x1apreferred_arrow_chunk_size\x18\x02 \x01(\x03H\x00R\x17preferredArrowChunkSize\x88\x01\x01\x42\x1d\n\x1b_preferred_arrow_chunk_size"\x96\x03\n\x16ReattachExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x06 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x12-\n\x10last_response_id\x18\x05 \x01(\tH\x02R\x0elastResponseId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x13\n\x11_last_response_id"\xc9\x04\n\x15ReleaseExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12R\n\x0brelease_all\x18\x05 \x01(\x0b\x32/.spark.connect.ReleaseExecuteRequest.ReleaseAllH\x00R\nreleaseAll\x12X\n\rrelease_until\x18\x06 \x01(\x0b\x32\x31.spark.connect.ReleaseExecuteRequest.ReleaseUntilH\x00R\x0creleaseUntil\x1a\x0c\n\nReleaseAll\x1a/\n\x0cReleaseUntil\x12\x1f\n\x0bresponse_id\x18\x01 \x01(\tR\nresponseIdB\t\n\x07releaseB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x01\n\x16ReleaseExecuteResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12&\n\x0coperation_id\x18\x02 \x01(\tH\x00R\x0boperationId\x88\x01\x01\x42\x0f\n\r_operation_id"\xd4\x01\n\x15ReleaseSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12\'\n\x0f\x61llow_reconnect\x18\x04 \x01(\x08R\x0e\x61llowReconnectB\x0e\n\x0c_client_type"l\n\x16ReleaseSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId"\xcc\x02\n\x18\x46\x65tchErrorDetailsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x19\n\x08\x65rror_id\x18\x03 \x01(\tR\x07\x65rrorId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xd9\x0f\n\x19\x46\x65tchErrorDetailsResponse\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\x1d\n\nsession_id\x18\x04 \x01(\tR\tsessionId\x12)\n\x0eroot_error_idx\x18\x01 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x02 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrors\x1a\xae\x01\n\x11StackTraceElement\x12\'\n\x0f\x64\x65\x63laring_class\x18\x01 \x01(\tR\x0e\x64\x65\x63laringClass\x12\x1f\n\x0bmethod_name\x18\x02 \x01(\tR\nmethodName\x12 \n\tfile_name\x18\x03 \x01(\tH\x00R\x08\x66ileName\x88\x01\x01\x12\x1f\n\x0bline_number\x18\x04 \x01(\x05R\nlineNumberB\x0c\n\n_file_name\x1a\xf0\x02\n\x0cQueryContext\x12\x64\n\x0c\x63ontext_type\x18\n \x01(\x0e\x32\x41.spark.connect.FetchErrorDetailsResponse.QueryContext.ContextTypeR\x0b\x63ontextType\x12\x1f\n\x0bobject_type\x18\x01 \x01(\tR\nobjectType\x12\x1f\n\x0bobject_name\x18\x02 \x01(\tR\nobjectName\x12\x1f\n\x0bstart_index\x18\x03 \x01(\x05R\nstartIndex\x12\x1d\n\nstop_index\x18\x04 \x01(\x05R\tstopIndex\x12\x1a\n\x08\x66ragment\x18\x05 \x01(\tR\x08\x66ragment\x12\x1b\n\tcall_site\x18\x06 \x01(\tR\x08\x63\x61llSite\x12\x18\n\x07summary\x18\x07 \x01(\tR\x07summary"%\n\x0b\x43ontextType\x12\x07\n\x03SQL\x10\x00\x12\r\n\tDATAFRAME\x10\x01\x1a\xa6\x04\n\x0eSparkThrowable\x12$\n\x0b\x65rror_class\x18\x01 \x01(\tH\x00R\nerrorClass\x88\x01\x01\x12}\n\x12message_parameters\x18\x02 \x03(\x0b\x32N.spark.connect.FetchErrorDetailsResponse.SparkThrowable.MessageParametersEntryR\x11messageParameters\x12\\\n\x0equery_contexts\x18\x03 \x03(\x0b\x32\x35.spark.connect.FetchErrorDetailsResponse.QueryContextR\rqueryContexts\x12 \n\tsql_state\x18\x04 \x01(\tH\x01R\x08sqlState\x88\x01\x01\x12r\n\x14\x62reaking_change_info\x18\x05 \x01(\x0b\x32;.spark.connect.FetchErrorDetailsResponse.BreakingChangeInfoH\x02R\x12\x62reakingChangeInfo\x88\x01\x01\x1a\x44\n\x16MessageParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_error_classB\x0c\n\n_sql_stateB\x17\n\x15_breaking_change_info\x1a\xfa\x01\n\x12\x42reakingChangeInfo\x12+\n\x11migration_message\x18\x01 \x03(\tR\x10migrationMessage\x12k\n\x11mitigation_config\x18\x02 \x01(\x0b\x32\x39.spark.connect.FetchErrorDetailsResponse.MitigationConfigH\x00R\x10mitigationConfig\x88\x01\x01\x12$\n\x0bneeds_audit\x18\x03 \x01(\x08H\x01R\nneedsAudit\x88\x01\x01\x42\x14\n\x12_mitigation_configB\x0e\n\x0c_needs_audit\x1a:\n\x10MitigationConfig\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\x1a\xdb\x02\n\x05\x45rror\x12\x30\n\x14\x65rror_type_hierarchy\x18\x01 \x03(\tR\x12\x65rrorTypeHierarchy\x12\x18\n\x07message\x18\x02 \x01(\tR\x07message\x12[\n\x0bstack_trace\x18\x03 \x03(\x0b\x32:.spark.connect.FetchErrorDetailsResponse.StackTraceElementR\nstackTrace\x12 \n\tcause_idx\x18\x04 \x01(\x05H\x00R\x08\x63\x61useIdx\x88\x01\x01\x12\x65\n\x0fspark_throwable\x18\x05 \x01(\x0b\x32\x37.spark.connect.FetchErrorDetailsResponse.SparkThrowableH\x01R\x0esparkThrowable\x88\x01\x01\x42\x0c\n\n_cause_idxB\x12\n\x10_spark_throwableB\x11\n\x0f_root_error_idx"Z\n\x17\x43heckpointCommandResult\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation":\n\x15\x43reateBroadcastResult\x12!\n\x0c\x62roadcast_id\x18\x01 \x01(\x03R\x0b\x62roadcastId"\xea\x02\n\x13\x43loneSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12)\n\x0enew_session_id\x18\x04 \x01(\tH\x02R\x0cnewSessionId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x11\n\x0f_new_session_id"\xcc\x01\n\x14\x43loneSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12$\n\x0enew_session_id\x18\x03 \x01(\tR\x0cnewSessionId\x12:\n\x1anew_server_side_session_id\x18\x04 \x01(\tR\x16newServerSideSessionId"\xd3\x04\n\x10GetStatusRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12V\n&client_observed_server_side_session_id\x18\x04 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12\x66\n\x10operation_status\x18\x05 \x01(\x0b\x32\x36.spark.connect.GetStatusRequest.OperationStatusRequestH\x02R\x0foperationStatus\x88\x01\x01\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1at\n\x16OperationStatusRequest\x12#\n\roperation_ids\x18\x01 \x03(\tR\x0coperationIds\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensionsB\x0e\n\x0c_client_typeB)\n\'_client_observed_server_side_session_idB\x13\n\x11_operation_status"\xad\x05\n\x11GetStatusResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12_\n\x12operation_statuses\x18\x03 \x03(\x0b\x32\x30.spark.connect.GetStatusResponse.OperationStatusR\x11operationStatuses\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1a\xab\x03\n\x0fOperationStatus\x12!\n\x0coperation_id\x18\x01 \x01(\tR\x0boperationId\x12U\n\x05state\x18\x02 \x01(\x0e\x32?.spark.connect.GetStatusResponse.OperationStatus.OperationStateR\x05state\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xe6\x01\n\x0eOperationState\x12\x1f\n\x1bOPERATION_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17OPERATION_STATE_UNKNOWN\x10\x01\x12\x1b\n\x17OPERATION_STATE_RUNNING\x10\x02\x12\x1f\n\x1bOPERATION_STATE_TERMINATING\x10\x03\x12\x1d\n\x19OPERATION_STATE_SUCCEEDED\x10\x04\x12\x1a\n\x16OPERATION_STATE_FAILED\x10\x05\x12\x1d\n\x19OPERATION_STATE_CANCELLED\x10\x06*Q\n\x10\x43ompressionCodec\x12!\n\x1d\x43OMPRESSION_CODEC_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x43OMPRESSION_CODEC_ZSTD\x10\x01\x32\xdf\x08\n\x13SparkConnectService\x12X\n\x0b\x45xecutePlan\x12!.spark.connect.ExecutePlanRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12V\n\x0b\x41nalyzePlan\x12!.spark.connect.AnalyzePlanRequest\x1a".spark.connect.AnalyzePlanResponse"\x00\x12G\n\x06\x43onfig\x12\x1c.spark.connect.ConfigRequest\x1a\x1d.spark.connect.ConfigResponse"\x00\x12[\n\x0c\x41\x64\x64\x41rtifacts\x12".spark.connect.AddArtifactsRequest\x1a#.spark.connect.AddArtifactsResponse"\x00(\x01\x12\x63\n\x0e\x41rtifactStatus\x12&.spark.connect.ArtifactStatusesRequest\x1a\'.spark.connect.ArtifactStatusesResponse"\x00\x12P\n\tInterrupt\x12\x1f.spark.connect.InterruptRequest\x1a .spark.connect.InterruptResponse"\x00\x12`\n\x0fReattachExecute\x12%.spark.connect.ReattachExecuteRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12_\n\x0eReleaseExecute\x12$.spark.connect.ReleaseExecuteRequest\x1a%.spark.connect.ReleaseExecuteResponse"\x00\x12_\n\x0eReleaseSession\x12$.spark.connect.ReleaseSessionRequest\x1a%.spark.connect.ReleaseSessionResponse"\x00\x12h\n\x11\x46\x65tchErrorDetails\x12\'.spark.connect.FetchErrorDetailsRequest\x1a(.spark.connect.FetchErrorDetailsResponse"\x00\x12Y\n\x0c\x43loneSession\x12".spark.connect.CloneSessionRequest\x1a#.spark.connect.CloneSessionResponse"\x00\x12P\n\tGetStatus\x12\x1f.spark.connect.GetStatusRequest\x1a .spark.connect.GetStatusResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -71,8 +71,8 @@ _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" ]._serialized_options = b"8\001" - _globals["_COMPRESSIONCODEC"]._serialized_start = 19991 - _globals["_COMPRESSIONCODEC"]._serialized_end = 20072 + _globals["_COMPRESSIONCODEC"]._serialized_start = 20147 + _globals["_COMPRESSIONCODEC"]._serialized_end = 20228 _globals["_PLAN"]._serialized_start = 275 _globals["_PLAN"]._serialized_end = 758 _globals["_PLAN_COMPRESSEDOPERATION"]._serialized_start = 477 @@ -148,139 +148,141 @@ _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_start = 5868 _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_end = 6129 _globals["_EXECUTEPLANRESPONSE"]._serialized_start = 6208 - _globals["_EXECUTEPLANRESPONSE"]._serialized_end = 9799 - _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_start = 8308 - _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_end = 8379 - _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_start = 8382 - _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_end = 8630 - _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_start = 8633 - _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_end = 9150 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_start = 8728 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_end = 9060 + _globals["_EXECUTEPLANRESPONSE"]._serialized_end = 9895 + _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_start = 8404 + _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_end = 8475 + _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_start = 8478 + _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_end = 8726 + _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_start = 8729 + _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_end = 9246 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_start = 8824 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_end = 9156 _globals[ "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY" - ]._serialized_start = 8937 + ]._serialized_start = 9033 _globals[ "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY" - ]._serialized_end = 9060 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 9062 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 9150 - _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 9153 - _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_end = 9428 - _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_start = 9430 - _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_end = 9446 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_start = 9449 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_end = 9782 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_start = 9605 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_end = 9782 - _globals["_KEYVALUE"]._serialized_start = 9801 - _globals["_KEYVALUE"]._serialized_end = 9866 - _globals["_CONFIGREQUEST"]._serialized_start = 9869 - _globals["_CONFIGREQUEST"]._serialized_end = 11068 - _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 10177 - _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 10675 - _globals["_CONFIGREQUEST_SET"]._serialized_start = 10677 - _globals["_CONFIGREQUEST_SET"]._serialized_end = 10769 - _globals["_CONFIGREQUEST_GET"]._serialized_start = 10771 - _globals["_CONFIGREQUEST_GET"]._serialized_end = 10796 - _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 10798 - _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 10861 - _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 10863 - _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 10894 - _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 10896 - _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 10944 - _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 10946 - _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 10973 - _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 10975 - _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 11009 - _globals["_CONFIGRESPONSE"]._serialized_start = 11071 - _globals["_CONFIGRESPONSE"]._serialized_end = 11246 - _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 11249 - _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 12251 - _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 11724 - _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 11777 - _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 11779 - _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 11890 - _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 11892 - _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 11985 - _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 11988 - _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 12181 - _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 12254 - _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 12526 - _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 12445 - _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 12526 - _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 12529 - _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 12855 - _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 12858 - _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 13210 - _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 13053 - _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 13168 - _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 13170 - _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 13210 - _globals["_INTERRUPTREQUEST"]._serialized_start = 13213 - _globals["_INTERRUPTREQUEST"]._serialized_end = 13816 - _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 13616 - _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 13744 - _globals["_INTERRUPTRESPONSE"]._serialized_start = 13819 - _globals["_INTERRUPTRESPONSE"]._serialized_end = 13963 - _globals["_REATTACHOPTIONS"]._serialized_start = 13965 - _globals["_REATTACHOPTIONS"]._serialized_end = 14018 - _globals["_RESULTCHUNKINGOPTIONS"]._serialized_start = 14021 - _globals["_RESULTCHUNKINGOPTIONS"]._serialized_end = 14202 - _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 14205 - _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 14611 - _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 14614 - _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 15199 - _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 15068 - _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 15080 - _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 15082 - _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 15129 - _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 15202 - _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 15367 - _globals["_RELEASESESSIONREQUEST"]._serialized_start = 15370 - _globals["_RELEASESESSIONREQUEST"]._serialized_end = 15582 - _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 15584 - _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 15692 - _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 15695 - _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 16027 - _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 16030 - _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 18039 - _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 16259 - _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 16433 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 16436 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 16804 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 16767 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 16804 - _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 16807 - _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 17357 + ]._serialized_end = 9156 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 9158 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 9246 + _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 9249 + _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_end = 9524 + _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_start = 9526 + _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_end = 9542 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_start = 9545 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_end = 9878 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_start = 9701 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_end = 9878 + _globals["_KEYVALUE"]._serialized_start = 9897 + _globals["_KEYVALUE"]._serialized_end = 9962 + _globals["_CONFIGREQUEST"]._serialized_start = 9965 + _globals["_CONFIGREQUEST"]._serialized_end = 11164 + _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 10273 + _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 10771 + _globals["_CONFIGREQUEST_SET"]._serialized_start = 10773 + _globals["_CONFIGREQUEST_SET"]._serialized_end = 10865 + _globals["_CONFIGREQUEST_GET"]._serialized_start = 10867 + _globals["_CONFIGREQUEST_GET"]._serialized_end = 10892 + _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 10894 + _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 10957 + _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 10959 + _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 10990 + _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 10992 + _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 11040 + _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 11042 + _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 11069 + _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 11071 + _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 11105 + _globals["_CONFIGRESPONSE"]._serialized_start = 11167 + _globals["_CONFIGRESPONSE"]._serialized_end = 11342 + _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 11345 + _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 12347 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 11820 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 11873 + _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 11875 + _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 11986 + _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 11988 + _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 12081 + _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 12084 + _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 12277 + _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 12350 + _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 12622 + _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 12541 + _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 12622 + _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 12625 + _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 12951 + _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 12954 + _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 13306 + _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 13149 + _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 13264 + _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 13266 + _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 13306 + _globals["_INTERRUPTREQUEST"]._serialized_start = 13309 + _globals["_INTERRUPTREQUEST"]._serialized_end = 13912 + _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 13712 + _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 13840 + _globals["_INTERRUPTRESPONSE"]._serialized_start = 13915 + _globals["_INTERRUPTRESPONSE"]._serialized_end = 14059 + _globals["_REATTACHOPTIONS"]._serialized_start = 14061 + _globals["_REATTACHOPTIONS"]._serialized_end = 14114 + _globals["_RESULTCHUNKINGOPTIONS"]._serialized_start = 14117 + _globals["_RESULTCHUNKINGOPTIONS"]._serialized_end = 14298 + _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 14301 + _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 14707 + _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 14710 + _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 15295 + _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 15164 + _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 15176 + _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 15178 + _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 15225 + _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 15298 + _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 15463 + _globals["_RELEASESESSIONREQUEST"]._serialized_start = 15466 + _globals["_RELEASESESSIONREQUEST"]._serialized_end = 15678 + _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 15680 + _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 15788 + _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 15791 + _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 16123 + _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 16126 + _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 18135 + _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 16355 + _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 16529 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 16532 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 16900 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 16863 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 16900 + _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 16903 + _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 17453 _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" - ]._serialized_start = 17234 + ]._serialized_start = 17330 _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" - ]._serialized_end = 17302 - _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_start = 17360 - _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_end = 17610 - _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_start = 17612 - _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_end = 17670 - _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 17673 - _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 18020 - _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 18041 - _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 18131 - _globals["_CLONESESSIONREQUEST"]._serialized_start = 18134 - _globals["_CLONESESSIONREQUEST"]._serialized_end = 18496 - _globals["_CLONESESSIONRESPONSE"]._serialized_start = 18499 - _globals["_CLONESESSIONRESPONSE"]._serialized_end = 18703 - _globals["_GETSTATUSREQUEST"]._serialized_start = 18706 - _globals["_GETSTATUSREQUEST"]._serialized_end = 19301 - _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_start = 19105 - _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_end = 19221 - _globals["_GETSTATUSRESPONSE"]._serialized_start = 19304 - _globals["_GETSTATUSRESPONSE"]._serialized_end = 19989 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_start = 19562 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_end = 19989 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_start = 19759 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_end = 19989 - _globals["_SPARKCONNECTSERVICE"]._serialized_start = 20075 - _globals["_SPARKCONNECTSERVICE"]._serialized_end = 21194 + ]._serialized_end = 17398 + _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_start = 17456 + _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_end = 17706 + _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_start = 17708 + _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_end = 17766 + _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 17769 + _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 18116 + _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 18137 + _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 18227 + _globals["_CREATEBROADCASTRESULT"]._serialized_start = 18229 + _globals["_CREATEBROADCASTRESULT"]._serialized_end = 18287 + _globals["_CLONESESSIONREQUEST"]._serialized_start = 18290 + _globals["_CLONESESSIONREQUEST"]._serialized_end = 18652 + _globals["_CLONESESSIONRESPONSE"]._serialized_start = 18655 + _globals["_CLONESESSIONRESPONSE"]._serialized_end = 18859 + _globals["_GETSTATUSREQUEST"]._serialized_start = 18862 + _globals["_GETSTATUSREQUEST"]._serialized_end = 19457 + _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_start = 19261 + _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_end = 19377 + _globals["_GETSTATUSRESPONSE"]._serialized_start = 19460 + _globals["_GETSTATUSRESPONSE"]._serialized_end = 20145 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_start = 19718 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_end = 20145 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_start = 19915 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_end = 20145 + _globals["_SPARKCONNECTSERVICE"]._serialized_start = 20231 + _globals["_SPARKCONNECTSERVICE"]._serialized_end = 21350 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi index 2db3132cd0c01..1fa31946f00ab 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.pyi +++ b/python/pyspark/sql/connect/proto/base_pb2.pyi @@ -1759,6 +1759,7 @@ class ExecutePlanResponse(google.protobuf.message.Message): PIPELINE_EVENT_RESULT_FIELD_NUMBER: builtins.int PIPELINE_COMMAND_RESULT_FIELD_NUMBER: builtins.int PIPELINE_QUERY_FUNCTION_EXECUTION_SIGNAL_FIELD_NUMBER: builtins.int + CREATE_BROADCAST_RESULT_FIELD_NUMBER: builtins.int EXTENSION_FIELD_NUMBER: builtins.int METRICS_FIELD_NUMBER: builtins.int OBSERVED_METRICS_FIELD_NUMBER: builtins.int @@ -1841,6 +1842,9 @@ class ExecutePlanResponse(google.protobuf.message.Message): register its result with the server. """ @property + def create_broadcast_result(self) -> global___CreateBroadcastResult: + """(SPARK-51705) Server-assigned handle for a CreateBroadcastCommand.""" + @property def extension(self) -> google.protobuf.any_pb2.Any: """Support arbitrary result objects.""" @property @@ -1889,6 +1893,7 @@ class ExecutePlanResponse(google.protobuf.message.Message): | None = ..., pipeline_query_function_execution_signal: pyspark.sql.connect.proto.pipelines_pb2.PipelineQueryFunctionExecutionSignal | None = ..., + create_broadcast_result: global___CreateBroadcastResult | None = ..., extension: google.protobuf.any_pb2.Any | None = ..., metrics: global___ExecutePlanResponse.Metrics | None = ..., observed_metrics: collections.abc.Iterable[global___ExecutePlanResponse.ObservedMetrics] @@ -1902,6 +1907,8 @@ class ExecutePlanResponse(google.protobuf.message.Message): b"arrow_batch", "checkpoint_command_result", b"checkpoint_command_result", + "create_broadcast_result", + b"create_broadcast_result", "create_resource_profile_command_result", b"create_resource_profile_command_result", "execution_progress", @@ -1945,6 +1952,8 @@ class ExecutePlanResponse(google.protobuf.message.Message): b"arrow_batch", "checkpoint_command_result", b"checkpoint_command_result", + "create_broadcast_result", + b"create_broadcast_result", "create_resource_profile_command_result", b"create_resource_profile_command_result", "execution_progress", @@ -2010,6 +2019,7 @@ class ExecutePlanResponse(google.protobuf.message.Message): "pipeline_event_result", "pipeline_command_result", "pipeline_query_function_execution_signal", + "create_broadcast_result", "extension", ] | None @@ -4203,6 +4213,28 @@ class CheckpointCommandResult(google.protobuf.message.Message): global___CheckpointCommandResult = CheckpointCommandResult +class CreateBroadcastResult(google.protobuf.message.Message): + """(SPARK-51705) Server-assigned broadcast handle (mirrors CheckpointCommandResult).""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + BROADCAST_ID_FIELD_NUMBER: builtins.int + broadcast_id: builtins.int + """(Required) The driver-side broadcast id. The client embeds this id in the pickled + Broadcast closure (Broadcast._from_id), which the executor/worker resolves against its + _broadcastRegistry keyed by the same id. + """ + def __init__( + self, + *, + broadcast_id: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["broadcast_id", b"broadcast_id"] + ) -> None: ... + +global___CreateBroadcastResult = CreateBroadcastResult + class CloneSessionRequest(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py index 78af0ddbfe318..f5155853ca0c3 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.py +++ b/python/pyspark/sql/connect/proto/commands_pb2.py @@ -44,7 +44,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xfb\x0e\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeView\x12O\n\x12write_operation_v2\x18\x04 \x01(\x0b\x32\x1f.spark.connect.WriteOperationV2H\x00R\x10writeOperationV2\x12<\n\x0bsql_command\x18\x05 \x01(\x0b\x32\x19.spark.connect.SqlCommandH\x00R\nsqlCommand\x12k\n\x1cwrite_stream_operation_start\x18\x06 \x01(\x0b\x32(.spark.connect.WriteStreamOperationStartH\x00R\x19writeStreamOperationStart\x12^\n\x17streaming_query_command\x18\x07 \x01(\x0b\x32$.spark.connect.StreamingQueryCommandH\x00R\x15streamingQueryCommand\x12X\n\x15get_resources_command\x18\x08 \x01(\x0b\x32".spark.connect.GetResourcesCommandH\x00R\x13getResourcesCommand\x12t\n\x1fstreaming_query_manager_command\x18\t \x01(\x0b\x32+.spark.connect.StreamingQueryManagerCommandH\x00R\x1cstreamingQueryManagerCommand\x12m\n\x17register_table_function\x18\n \x01(\x0b\x32\x33.spark.connect.CommonInlineUserDefinedTableFunctionH\x00R\x15registerTableFunction\x12\x81\x01\n$streaming_query_listener_bus_command\x18\x0b \x01(\x0b\x32/.spark.connect.StreamingQueryListenerBusCommandH\x00R streamingQueryListenerBusCommand\x12\x64\n\x14register_data_source\x18\x0c \x01(\x0b\x32\x30.spark.connect.CommonInlineUserDefinedDataSourceH\x00R\x12registerDataSource\x12t\n\x1f\x63reate_resource_profile_command\x18\r \x01(\x0b\x32+.spark.connect.CreateResourceProfileCommandH\x00R\x1c\x63reateResourceProfileCommand\x12Q\n\x12\x63heckpoint_command\x18\x0e \x01(\x0b\x32 .spark.connect.CheckpointCommandH\x00R\x11\x63heckpointCommand\x12\x84\x01\n%remove_cached_remote_relation_command\x18\x0f \x01(\x0b\x32\x30.spark.connect.RemoveCachedRemoteRelationCommandH\x00R!removeCachedRemoteRelationCommand\x12_\n\x18merge_into_table_command\x18\x10 \x01(\x0b\x32$.spark.connect.MergeIntoTableCommandH\x00R\x15mergeIntoTableCommand\x12\x39\n\nml_command\x18\x11 \x01(\x0b\x32\x18.spark.connect.MlCommandH\x00R\tmlCommand\x12\x61\n\x18\x65xecute_external_command\x18\x12 \x01(\x0b\x32%.spark.connect.ExecuteExternalCommandH\x00R\x16\x65xecuteExternalCommand\x12K\n\x10pipeline_command\x18\x13 \x01(\x0b\x32\x1e.spark.connect.PipelineCommandH\x00R\x0fpipelineCommand\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x0e\n\x0c\x63ommand_type"\xaa\x04\n\nSqlCommand\x12\x14\n\x03sql\x18\x01 \x01(\tB\x02\x18\x01R\x03sql\x12;\n\x04\x61rgs\x18\x02 \x03(\x0b\x32#.spark.connect.SqlCommand.ArgsEntryB\x02\x18\x01R\x04\x61rgs\x12@\n\x08pos_args\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralB\x02\x18\x01R\x07posArgs\x12Z\n\x0fnamed_arguments\x18\x04 \x03(\x0b\x32-.spark.connect.SqlCommand.NamedArgumentsEntryB\x02\x18\x01R\x0enamedArguments\x12\x42\n\rpos_arguments\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionB\x02\x18\x01R\x0cposArguments\x12-\n\x05input\x18\x06 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x1aZ\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01\x1a\\\n\x13NamedArgumentsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value:\x02\x38\x01"\x96\x01\n\x1a\x43reateDataFrameViewCommand\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n\tis_global\x18\x03 \x01(\x08R\x08isGlobal\x12\x18\n\x07replace\x18\x04 \x01(\x08R\x07replace"\xfe\x08\n\x0eWriteOperation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1b\n\x06source\x18\x02 \x01(\tH\x01R\x06source\x88\x01\x01\x12\x14\n\x04path\x18\x03 \x01(\tH\x00R\x04path\x12?\n\x05table\x18\x04 \x01(\x0b\x32\'.spark.connect.WriteOperation.SaveTableH\x00R\x05table\x12:\n\x04mode\x18\x05 \x01(\x0e\x32&.spark.connect.WriteOperation.SaveModeR\x04mode\x12*\n\x11sort_column_names\x18\x06 \x03(\tR\x0fsortColumnNames\x12\x31\n\x14partitioning_columns\x18\x07 \x03(\tR\x13partitioningColumns\x12\x43\n\tbucket_by\x18\x08 \x01(\x0b\x32&.spark.connect.WriteOperation.BucketByR\x08\x62ucketBy\x12\x44\n\x07options\x18\t \x03(\x0b\x32*.spark.connect.WriteOperation.OptionsEntryR\x07options\x12-\n\x12\x63lustering_columns\x18\n \x03(\tR\x11\x63lusteringColumns\x12\x32\n\x15with_schema_evolution\x18\x0b \x01(\x08R\x13withSchemaEvolution\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x82\x02\n\tSaveTable\x12\x1d\n\ntable_name\x18\x01 \x01(\tR\ttableName\x12X\n\x0bsave_method\x18\x02 \x01(\x0e\x32\x37.spark.connect.WriteOperation.SaveTable.TableSaveMethodR\nsaveMethod"|\n\x0fTableSaveMethod\x12!\n\x1dTABLE_SAVE_METHOD_UNSPECIFIED\x10\x00\x12#\n\x1fTABLE_SAVE_METHOD_SAVE_AS_TABLE\x10\x01\x12!\n\x1dTABLE_SAVE_METHOD_INSERT_INTO\x10\x02\x1a[\n\x08\x42ucketBy\x12.\n\x13\x62ucket_column_names\x18\x01 \x03(\tR\x11\x62ucketColumnNames\x12\x1f\n\x0bnum_buckets\x18\x02 \x01(\x05R\nnumBuckets"\x89\x01\n\x08SaveMode\x12\x19\n\x15SAVE_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10SAVE_MODE_APPEND\x10\x01\x12\x17\n\x13SAVE_MODE_OVERWRITE\x10\x02\x12\x1d\n\x19SAVE_MODE_ERROR_IF_EXISTS\x10\x03\x12\x14\n\x10SAVE_MODE_IGNORE\x10\x04\x42\x0b\n\tsave_typeB\t\n\x07_source"\x90\x07\n\x10WriteOperationV2\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1d\n\ntable_name\x18\x02 \x01(\tR\ttableName\x12\x1f\n\x08provider\x18\x03 \x01(\tH\x00R\x08provider\x88\x01\x01\x12L\n\x14partitioning_columns\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13partitioningColumns\x12\x46\n\x07options\x18\x05 \x03(\x0b\x32,.spark.connect.WriteOperationV2.OptionsEntryR\x07options\x12_\n\x10table_properties\x18\x06 \x03(\x0b\x32\x34.spark.connect.WriteOperationV2.TablePropertiesEntryR\x0ftableProperties\x12\x38\n\x04mode\x18\x07 \x01(\x0e\x32$.spark.connect.WriteOperationV2.ModeR\x04mode\x12J\n\x13overwrite_condition\x18\x08 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x12overwriteCondition\x12-\n\x12\x63lustering_columns\x18\t \x03(\tR\x11\x63lusteringColumns\x12\x32\n\x15with_schema_evolution\x18\n \x01(\x08R\x13withSchemaEvolution\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x42\n\x14TablePropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"\x9f\x01\n\x04Mode\x12\x14\n\x10MODE_UNSPECIFIED\x10\x00\x12\x0f\n\x0bMODE_CREATE\x10\x01\x12\x12\n\x0eMODE_OVERWRITE\x10\x02\x12\x1d\n\x19MODE_OVERWRITE_PARTITIONS\x10\x03\x12\x0f\n\x0bMODE_APPEND\x10\x04\x12\x10\n\x0cMODE_REPLACE\x10\x05\x12\x1a\n\x16MODE_CREATE_OR_REPLACE\x10\x06\x42\x0b\n\t_provider"\x93\x07\n\x19WriteStreamOperationStart\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06\x66ormat\x18\x02 \x01(\tR\x06\x66ormat\x12O\n\x07options\x18\x03 \x03(\x0b\x32\x35.spark.connect.WriteStreamOperationStart.OptionsEntryR\x07options\x12:\n\x19partitioning_column_names\x18\x04 \x03(\tR\x17partitioningColumnNames\x12:\n\x18processing_time_interval\x18\x05 \x01(\tH\x00R\x16processingTimeInterval\x12%\n\ravailable_now\x18\x06 \x01(\x08H\x00R\x0c\x61vailableNow\x12\x14\n\x04once\x18\x07 \x01(\x08H\x00R\x04once\x12\x46\n\x1e\x63ontinuous_checkpoint_interval\x18\x08 \x01(\tH\x00R\x1c\x63ontinuousCheckpointInterval\x12\x39\n\x18real_time_batch_duration\x18\x64 \x01(\tH\x00R\x15realTimeBatchDuration\x12\x1f\n\x0boutput_mode\x18\t \x01(\tR\noutputMode\x12\x1d\n\nquery_name\x18\n \x01(\tR\tqueryName\x12\x14\n\x04path\x18\x0b \x01(\tH\x01R\x04path\x12\x1f\n\ntable_name\x18\x0c \x01(\tH\x01R\ttableName\x12N\n\x0e\x66oreach_writer\x18\r \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\rforeachWriter\x12L\n\rforeach_batch\x18\x0e \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\x0c\x66oreachBatch\x12\x36\n\x17\x63lustering_column_names\x18\x0f \x03(\tR\x15\x63lusteringColumnNames\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07triggerB\x12\n\x10sink_destination"\xb3\x01\n\x18StreamingForeachFunction\x12\x43\n\x0fpython_function\x18\x01 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x0epythonFunction\x12\x46\n\x0escala_function\x18\x02 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\rscalaFunctionB\n\n\x08\x66unction"\xd4\x01\n\x1fWriteStreamOperationStartResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12<\n\x18query_started_event_json\x18\x03 \x01(\tH\x00R\x15queryStartedEventJson\x88\x01\x01\x42\x1b\n\x19_query_started_event_json"A\n\x18StreamingQueryInstanceId\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x15\n\x06run_id\x18\x02 \x01(\tR\x05runId"\xf8\x04\n\x15StreamingQueryCommand\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x18\n\x06status\x18\x02 \x01(\x08H\x00R\x06status\x12%\n\rlast_progress\x18\x03 \x01(\x08H\x00R\x0clastProgress\x12)\n\x0frecent_progress\x18\x04 \x01(\x08H\x00R\x0erecentProgress\x12\x14\n\x04stop\x18\x05 \x01(\x08H\x00R\x04stop\x12\x34\n\x15process_all_available\x18\x06 \x01(\x08H\x00R\x13processAllAvailable\x12O\n\x07\x65xplain\x18\x07 \x01(\x0b\x32\x33.spark.connect.StreamingQueryCommand.ExplainCommandH\x00R\x07\x65xplain\x12\x1e\n\texception\x18\x08 \x01(\x08H\x00R\texception\x12k\n\x11\x61wait_termination\x18\t \x01(\x0b\x32<.spark.connect.StreamingQueryCommand.AwaitTerminationCommandH\x00R\x10\x61waitTermination\x1a,\n\x0e\x45xplainCommand\x12\x1a\n\x08\x65xtended\x18\x01 \x01(\x08R\x08\x65xtended\x1aL\n\x17\x41waitTerminationCommand\x12"\n\ntimeout_ms\x18\x02 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_msB\t\n\x07\x63ommand"\xf5\x08\n\x1bStreamingQueryCommandResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12Q\n\x06status\x18\x02 \x01(\x0b\x32\x37.spark.connect.StreamingQueryCommandResult.StatusResultH\x00R\x06status\x12j\n\x0frecent_progress\x18\x03 \x01(\x0b\x32?.spark.connect.StreamingQueryCommandResult.RecentProgressResultH\x00R\x0erecentProgress\x12T\n\x07\x65xplain\x18\x04 \x01(\x0b\x32\x38.spark.connect.StreamingQueryCommandResult.ExplainResultH\x00R\x07\x65xplain\x12Z\n\texception\x18\x05 \x01(\x0b\x32:.spark.connect.StreamingQueryCommandResult.ExceptionResultH\x00R\texception\x12p\n\x11\x61wait_termination\x18\x06 \x01(\x0b\x32\x41.spark.connect.StreamingQueryCommandResult.AwaitTerminationResultH\x00R\x10\x61waitTermination\x1a\xaa\x01\n\x0cStatusResult\x12%\n\x0estatus_message\x18\x01 \x01(\tR\rstatusMessage\x12*\n\x11is_data_available\x18\x02 \x01(\x08R\x0fisDataAvailable\x12*\n\x11is_trigger_active\x18\x03 \x01(\x08R\x0fisTriggerActive\x12\x1b\n\tis_active\x18\x04 \x01(\x08R\x08isActive\x1aH\n\x14RecentProgressResult\x12\x30\n\x14recent_progress_json\x18\x05 \x03(\tR\x12recentProgressJson\x1a\'\n\rExplainResult\x12\x16\n\x06result\x18\x01 \x01(\tR\x06result\x1a\xc5\x01\n\x0f\x45xceptionResult\x12\x30\n\x11\x65xception_message\x18\x01 \x01(\tH\x00R\x10\x65xceptionMessage\x88\x01\x01\x12$\n\x0b\x65rror_class\x18\x02 \x01(\tH\x01R\nerrorClass\x88\x01\x01\x12$\n\x0bstack_trace\x18\x03 \x01(\tH\x02R\nstackTrace\x88\x01\x01\x42\x14\n\x12_exception_messageB\x0e\n\x0c_error_classB\x0e\n\x0c_stack_trace\x1a\x38\n\x16\x41waitTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminatedB\r\n\x0bresult_type"\xbd\x06\n\x1cStreamingQueryManagerCommand\x12\x18\n\x06\x61\x63tive\x18\x01 \x01(\x08H\x00R\x06\x61\x63tive\x12\x1d\n\tget_query\x18\x02 \x01(\tH\x00R\x08getQuery\x12|\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32\x46.spark.connect.StreamingQueryManagerCommand.AwaitAnyTerminationCommandH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12n\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0b\x61\x64\x64Listener\x12t\n\x0fremove_listener\x18\x06 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0eremoveListener\x12\'\n\x0elist_listeners\x18\x07 \x01(\x08H\x00R\rlistListeners\x1aO\n\x1a\x41waitAnyTerminationCommand\x12"\n\ntimeout_ms\x18\x01 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_ms\x1a\xcd\x01\n\x1dStreamingQueryListenerCommand\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x12U\n\x17python_listener_payload\x18\x02 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x15pythonListenerPayload\x88\x01\x01\x12\x0e\n\x02id\x18\x03 \x01(\tR\x02idB\x1a\n\x18_python_listener_payloadB\t\n\x07\x63ommand"\xb4\x08\n"StreamingQueryManagerCommandResult\x12X\n\x06\x61\x63tive\x18\x01 \x01(\x0b\x32>.spark.connect.StreamingQueryManagerCommandResult.ActiveResultH\x00R\x06\x61\x63tive\x12`\n\x05query\x18\x02 \x01(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceH\x00R\x05query\x12\x81\x01\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32K.spark.connect.StreamingQueryManagerCommandResult.AwaitAnyTerminationResultH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12#\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x08H\x00R\x0b\x61\x64\x64Listener\x12)\n\x0fremove_listener\x18\x06 \x01(\x08H\x00R\x0eremoveListener\x12{\n\x0elist_listeners\x18\x07 \x01(\x0b\x32R.spark.connect.StreamingQueryManagerCommandResult.ListStreamingQueryListenerResultH\x00R\rlistListeners\x1a\x7f\n\x0c\x41\x63tiveResult\x12o\n\x0e\x61\x63tive_queries\x18\x01 \x03(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceR\ractiveQueries\x1as\n\x16StreamingQueryInstance\x12\x37\n\x02id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x02id\x12\x17\n\x04name\x18\x02 \x01(\tH\x00R\x04name\x88\x01\x01\x42\x07\n\x05_name\x1a;\n\x19\x41waitAnyTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminated\x1aK\n\x1eStreamingQueryListenerInstance\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x1a\x45\n ListStreamingQueryListenerResult\x12!\n\x0clistener_ids\x18\x01 \x03(\tR\x0blistenerIdsB\r\n\x0bresult_type"\xad\x01\n StreamingQueryListenerBusCommand\x12;\n\x19\x61\x64\x64_listener_bus_listener\x18\x01 \x01(\x08H\x00R\x16\x61\x64\x64ListenerBusListener\x12\x41\n\x1cremove_listener_bus_listener\x18\x02 \x01(\x08H\x00R\x19removeListenerBusListenerB\t\n\x07\x63ommand"\x83\x01\n\x1bStreamingQueryListenerEvent\x12\x1d\n\nevent_json\x18\x01 \x01(\tR\teventJson\x12\x45\n\nevent_type\x18\x02 \x01(\x0e\x32&.spark.connect.StreamingQueryEventTypeR\teventType"\xcc\x01\n"StreamingQueryListenerEventsResult\x12\x42\n\x06\x65vents\x18\x01 \x03(\x0b\x32*.spark.connect.StreamingQueryListenerEventR\x06\x65vents\x12\x42\n\x1blistener_bus_listener_added\x18\x02 \x01(\x08H\x00R\x18listenerBusListenerAdded\x88\x01\x01\x42\x1e\n\x1c_listener_bus_listener_added"\x15\n\x13GetResourcesCommand"\xd4\x01\n\x19GetResourcesCommandResult\x12U\n\tresources\x18\x01 \x03(\x0b\x32\x37.spark.connect.GetResourcesCommandResult.ResourcesEntryR\tresources\x1a`\n\x0eResourcesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32".spark.connect.ResourceInformationR\x05value:\x02\x38\x01"X\n\x1c\x43reateResourceProfileCommand\x12\x38\n\x07profile\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ResourceProfileR\x07profile"C\n"CreateResourceProfileCommandResult\x12\x1d\n\nprofile_id\x18\x01 \x01(\x05R\tprofileId"d\n!RemoveCachedRemoteRelationCommand\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation"\xcd\x01\n\x11\x43heckpointCommand\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x14\n\x05local\x18\x02 \x01(\x08R\x05local\x12\x14\n\x05\x65\x61ger\x18\x03 \x01(\x08R\x05\x65\x61ger\x12\x45\n\rstorage_level\x18\x04 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xe8\x03\n\x15MergeIntoTableCommand\x12*\n\x11target_table_name\x18\x01 \x01(\tR\x0ftargetTableName\x12\x43\n\x11source_table_plan\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x0fsourceTablePlan\x12\x42\n\x0fmerge_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0emergeCondition\x12>\n\rmatch_actions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0cmatchActions\x12I\n\x13not_matched_actions\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x11notMatchedActions\x12[\n\x1dnot_matched_by_source_actions\x18\x06 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x19notMatchedBySourceActions\x12\x32\n\x15with_schema_evolution\x18\x07 \x01(\x08R\x13withSchemaEvolution"\xd4\x01\n\x16\x45xecuteExternalCommand\x12\x16\n\x06runner\x18\x01 \x01(\tR\x06runner\x12\x18\n\x07\x63ommand\x18\x02 \x01(\tR\x07\x63ommand\x12L\n\x07options\x18\x03 \x03(\x0b\x32\x32.spark.connect.ExecuteExternalCommand.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01*\x85\x01\n\x17StreamingQueryEventType\x12\x1e\n\x1aQUERY_PROGRESS_UNSPECIFIED\x10\x00\x12\x18\n\x14QUERY_PROGRESS_EVENT\x10\x01\x12\x1a\n\x16QUERY_TERMINATED_EVENT\x10\x02\x12\x14\n\x10QUERY_IDLE_EVENT\x10\x03\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xca\x10\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeView\x12O\n\x12write_operation_v2\x18\x04 \x01(\x0b\x32\x1f.spark.connect.WriteOperationV2H\x00R\x10writeOperationV2\x12<\n\x0bsql_command\x18\x05 \x01(\x0b\x32\x19.spark.connect.SqlCommandH\x00R\nsqlCommand\x12k\n\x1cwrite_stream_operation_start\x18\x06 \x01(\x0b\x32(.spark.connect.WriteStreamOperationStartH\x00R\x19writeStreamOperationStart\x12^\n\x17streaming_query_command\x18\x07 \x01(\x0b\x32$.spark.connect.StreamingQueryCommandH\x00R\x15streamingQueryCommand\x12X\n\x15get_resources_command\x18\x08 \x01(\x0b\x32".spark.connect.GetResourcesCommandH\x00R\x13getResourcesCommand\x12t\n\x1fstreaming_query_manager_command\x18\t \x01(\x0b\x32+.spark.connect.StreamingQueryManagerCommandH\x00R\x1cstreamingQueryManagerCommand\x12m\n\x17register_table_function\x18\n \x01(\x0b\x32\x33.spark.connect.CommonInlineUserDefinedTableFunctionH\x00R\x15registerTableFunction\x12\x81\x01\n$streaming_query_listener_bus_command\x18\x0b \x01(\x0b\x32/.spark.connect.StreamingQueryListenerBusCommandH\x00R streamingQueryListenerBusCommand\x12\x64\n\x14register_data_source\x18\x0c \x01(\x0b\x32\x30.spark.connect.CommonInlineUserDefinedDataSourceH\x00R\x12registerDataSource\x12t\n\x1f\x63reate_resource_profile_command\x18\r \x01(\x0b\x32+.spark.connect.CreateResourceProfileCommandH\x00R\x1c\x63reateResourceProfileCommand\x12Q\n\x12\x63heckpoint_command\x18\x0e \x01(\x0b\x32 .spark.connect.CheckpointCommandH\x00R\x11\x63heckpointCommand\x12\x84\x01\n%remove_cached_remote_relation_command\x18\x0f \x01(\x0b\x32\x30.spark.connect.RemoveCachedRemoteRelationCommandH\x00R!removeCachedRemoteRelationCommand\x12_\n\x18merge_into_table_command\x18\x10 \x01(\x0b\x32$.spark.connect.MergeIntoTableCommandH\x00R\x15mergeIntoTableCommand\x12\x39\n\nml_command\x18\x11 \x01(\x0b\x32\x18.spark.connect.MlCommandH\x00R\tmlCommand\x12\x61\n\x18\x65xecute_external_command\x18\x12 \x01(\x0b\x32%.spark.connect.ExecuteExternalCommandH\x00R\x16\x65xecuteExternalCommand\x12K\n\x10pipeline_command\x18\x13 \x01(\x0b\x32\x1e.spark.connect.PipelineCommandH\x00R\x0fpipelineCommand\x12\x61\n\x18\x63reate_broadcast_command\x18\x14 \x01(\x0b\x32%.spark.connect.CreateBroadcastCommandH\x00R\x16\x63reateBroadcastCommand\x12j\n\x1bunpersist_broadcast_command\x18\x15 \x01(\x0b\x32(.spark.connect.UnpersistBroadcastCommandH\x00R\x19unpersistBroadcastCommand\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x0e\n\x0c\x63ommand_type"\\\n\x16\x43reateBroadcastCommand\x12#\n\rartifact_hash\x18\x01 \x01(\tR\x0c\x61rtifactHash\x12\x1d\n\nsize_bytes\x18\x02 \x01(\x03R\tsizeBytes"t\n\x19UnpersistBroadcastCommand\x12!\n\x0c\x62roadcast_id\x18\x01 \x01(\x03R\x0b\x62roadcastId\x12\x1a\n\x08\x62locking\x18\x02 \x01(\x08R\x08\x62locking\x12\x18\n\x07\x64\x65stroy\x18\x03 \x01(\x08R\x07\x64\x65stroy"\xaa\x04\n\nSqlCommand\x12\x14\n\x03sql\x18\x01 \x01(\tB\x02\x18\x01R\x03sql\x12;\n\x04\x61rgs\x18\x02 \x03(\x0b\x32#.spark.connect.SqlCommand.ArgsEntryB\x02\x18\x01R\x04\x61rgs\x12@\n\x08pos_args\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralB\x02\x18\x01R\x07posArgs\x12Z\n\x0fnamed_arguments\x18\x04 \x03(\x0b\x32-.spark.connect.SqlCommand.NamedArgumentsEntryB\x02\x18\x01R\x0enamedArguments\x12\x42\n\rpos_arguments\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionB\x02\x18\x01R\x0cposArguments\x12-\n\x05input\x18\x06 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x1aZ\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01\x1a\\\n\x13NamedArgumentsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value:\x02\x38\x01"\x96\x01\n\x1a\x43reateDataFrameViewCommand\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n\tis_global\x18\x03 \x01(\x08R\x08isGlobal\x12\x18\n\x07replace\x18\x04 \x01(\x08R\x07replace"\xfe\x08\n\x0eWriteOperation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1b\n\x06source\x18\x02 \x01(\tH\x01R\x06source\x88\x01\x01\x12\x14\n\x04path\x18\x03 \x01(\tH\x00R\x04path\x12?\n\x05table\x18\x04 \x01(\x0b\x32\'.spark.connect.WriteOperation.SaveTableH\x00R\x05table\x12:\n\x04mode\x18\x05 \x01(\x0e\x32&.spark.connect.WriteOperation.SaveModeR\x04mode\x12*\n\x11sort_column_names\x18\x06 \x03(\tR\x0fsortColumnNames\x12\x31\n\x14partitioning_columns\x18\x07 \x03(\tR\x13partitioningColumns\x12\x43\n\tbucket_by\x18\x08 \x01(\x0b\x32&.spark.connect.WriteOperation.BucketByR\x08\x62ucketBy\x12\x44\n\x07options\x18\t \x03(\x0b\x32*.spark.connect.WriteOperation.OptionsEntryR\x07options\x12-\n\x12\x63lustering_columns\x18\n \x03(\tR\x11\x63lusteringColumns\x12\x32\n\x15with_schema_evolution\x18\x0b \x01(\x08R\x13withSchemaEvolution\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x82\x02\n\tSaveTable\x12\x1d\n\ntable_name\x18\x01 \x01(\tR\ttableName\x12X\n\x0bsave_method\x18\x02 \x01(\x0e\x32\x37.spark.connect.WriteOperation.SaveTable.TableSaveMethodR\nsaveMethod"|\n\x0fTableSaveMethod\x12!\n\x1dTABLE_SAVE_METHOD_UNSPECIFIED\x10\x00\x12#\n\x1fTABLE_SAVE_METHOD_SAVE_AS_TABLE\x10\x01\x12!\n\x1dTABLE_SAVE_METHOD_INSERT_INTO\x10\x02\x1a[\n\x08\x42ucketBy\x12.\n\x13\x62ucket_column_names\x18\x01 \x03(\tR\x11\x62ucketColumnNames\x12\x1f\n\x0bnum_buckets\x18\x02 \x01(\x05R\nnumBuckets"\x89\x01\n\x08SaveMode\x12\x19\n\x15SAVE_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10SAVE_MODE_APPEND\x10\x01\x12\x17\n\x13SAVE_MODE_OVERWRITE\x10\x02\x12\x1d\n\x19SAVE_MODE_ERROR_IF_EXISTS\x10\x03\x12\x14\n\x10SAVE_MODE_IGNORE\x10\x04\x42\x0b\n\tsave_typeB\t\n\x07_source"\x90\x07\n\x10WriteOperationV2\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1d\n\ntable_name\x18\x02 \x01(\tR\ttableName\x12\x1f\n\x08provider\x18\x03 \x01(\tH\x00R\x08provider\x88\x01\x01\x12L\n\x14partitioning_columns\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13partitioningColumns\x12\x46\n\x07options\x18\x05 \x03(\x0b\x32,.spark.connect.WriteOperationV2.OptionsEntryR\x07options\x12_\n\x10table_properties\x18\x06 \x03(\x0b\x32\x34.spark.connect.WriteOperationV2.TablePropertiesEntryR\x0ftableProperties\x12\x38\n\x04mode\x18\x07 \x01(\x0e\x32$.spark.connect.WriteOperationV2.ModeR\x04mode\x12J\n\x13overwrite_condition\x18\x08 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x12overwriteCondition\x12-\n\x12\x63lustering_columns\x18\t \x03(\tR\x11\x63lusteringColumns\x12\x32\n\x15with_schema_evolution\x18\n \x01(\x08R\x13withSchemaEvolution\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x42\n\x14TablePropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"\x9f\x01\n\x04Mode\x12\x14\n\x10MODE_UNSPECIFIED\x10\x00\x12\x0f\n\x0bMODE_CREATE\x10\x01\x12\x12\n\x0eMODE_OVERWRITE\x10\x02\x12\x1d\n\x19MODE_OVERWRITE_PARTITIONS\x10\x03\x12\x0f\n\x0bMODE_APPEND\x10\x04\x12\x10\n\x0cMODE_REPLACE\x10\x05\x12\x1a\n\x16MODE_CREATE_OR_REPLACE\x10\x06\x42\x0b\n\t_provider"\x93\x07\n\x19WriteStreamOperationStart\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06\x66ormat\x18\x02 \x01(\tR\x06\x66ormat\x12O\n\x07options\x18\x03 \x03(\x0b\x32\x35.spark.connect.WriteStreamOperationStart.OptionsEntryR\x07options\x12:\n\x19partitioning_column_names\x18\x04 \x03(\tR\x17partitioningColumnNames\x12:\n\x18processing_time_interval\x18\x05 \x01(\tH\x00R\x16processingTimeInterval\x12%\n\ravailable_now\x18\x06 \x01(\x08H\x00R\x0c\x61vailableNow\x12\x14\n\x04once\x18\x07 \x01(\x08H\x00R\x04once\x12\x46\n\x1e\x63ontinuous_checkpoint_interval\x18\x08 \x01(\tH\x00R\x1c\x63ontinuousCheckpointInterval\x12\x39\n\x18real_time_batch_duration\x18\x64 \x01(\tH\x00R\x15realTimeBatchDuration\x12\x1f\n\x0boutput_mode\x18\t \x01(\tR\noutputMode\x12\x1d\n\nquery_name\x18\n \x01(\tR\tqueryName\x12\x14\n\x04path\x18\x0b \x01(\tH\x01R\x04path\x12\x1f\n\ntable_name\x18\x0c \x01(\tH\x01R\ttableName\x12N\n\x0e\x66oreach_writer\x18\r \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\rforeachWriter\x12L\n\rforeach_batch\x18\x0e \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\x0c\x66oreachBatch\x12\x36\n\x17\x63lustering_column_names\x18\x0f \x03(\tR\x15\x63lusteringColumnNames\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07triggerB\x12\n\x10sink_destination"\xb3\x01\n\x18StreamingForeachFunction\x12\x43\n\x0fpython_function\x18\x01 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x0epythonFunction\x12\x46\n\x0escala_function\x18\x02 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\rscalaFunctionB\n\n\x08\x66unction"\xd4\x01\n\x1fWriteStreamOperationStartResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12<\n\x18query_started_event_json\x18\x03 \x01(\tH\x00R\x15queryStartedEventJson\x88\x01\x01\x42\x1b\n\x19_query_started_event_json"A\n\x18StreamingQueryInstanceId\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x15\n\x06run_id\x18\x02 \x01(\tR\x05runId"\xf8\x04\n\x15StreamingQueryCommand\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x18\n\x06status\x18\x02 \x01(\x08H\x00R\x06status\x12%\n\rlast_progress\x18\x03 \x01(\x08H\x00R\x0clastProgress\x12)\n\x0frecent_progress\x18\x04 \x01(\x08H\x00R\x0erecentProgress\x12\x14\n\x04stop\x18\x05 \x01(\x08H\x00R\x04stop\x12\x34\n\x15process_all_available\x18\x06 \x01(\x08H\x00R\x13processAllAvailable\x12O\n\x07\x65xplain\x18\x07 \x01(\x0b\x32\x33.spark.connect.StreamingQueryCommand.ExplainCommandH\x00R\x07\x65xplain\x12\x1e\n\texception\x18\x08 \x01(\x08H\x00R\texception\x12k\n\x11\x61wait_termination\x18\t \x01(\x0b\x32<.spark.connect.StreamingQueryCommand.AwaitTerminationCommandH\x00R\x10\x61waitTermination\x1a,\n\x0e\x45xplainCommand\x12\x1a\n\x08\x65xtended\x18\x01 \x01(\x08R\x08\x65xtended\x1aL\n\x17\x41waitTerminationCommand\x12"\n\ntimeout_ms\x18\x02 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_msB\t\n\x07\x63ommand"\xf5\x08\n\x1bStreamingQueryCommandResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12Q\n\x06status\x18\x02 \x01(\x0b\x32\x37.spark.connect.StreamingQueryCommandResult.StatusResultH\x00R\x06status\x12j\n\x0frecent_progress\x18\x03 \x01(\x0b\x32?.spark.connect.StreamingQueryCommandResult.RecentProgressResultH\x00R\x0erecentProgress\x12T\n\x07\x65xplain\x18\x04 \x01(\x0b\x32\x38.spark.connect.StreamingQueryCommandResult.ExplainResultH\x00R\x07\x65xplain\x12Z\n\texception\x18\x05 \x01(\x0b\x32:.spark.connect.StreamingQueryCommandResult.ExceptionResultH\x00R\texception\x12p\n\x11\x61wait_termination\x18\x06 \x01(\x0b\x32\x41.spark.connect.StreamingQueryCommandResult.AwaitTerminationResultH\x00R\x10\x61waitTermination\x1a\xaa\x01\n\x0cStatusResult\x12%\n\x0estatus_message\x18\x01 \x01(\tR\rstatusMessage\x12*\n\x11is_data_available\x18\x02 \x01(\x08R\x0fisDataAvailable\x12*\n\x11is_trigger_active\x18\x03 \x01(\x08R\x0fisTriggerActive\x12\x1b\n\tis_active\x18\x04 \x01(\x08R\x08isActive\x1aH\n\x14RecentProgressResult\x12\x30\n\x14recent_progress_json\x18\x05 \x03(\tR\x12recentProgressJson\x1a\'\n\rExplainResult\x12\x16\n\x06result\x18\x01 \x01(\tR\x06result\x1a\xc5\x01\n\x0f\x45xceptionResult\x12\x30\n\x11\x65xception_message\x18\x01 \x01(\tH\x00R\x10\x65xceptionMessage\x88\x01\x01\x12$\n\x0b\x65rror_class\x18\x02 \x01(\tH\x01R\nerrorClass\x88\x01\x01\x12$\n\x0bstack_trace\x18\x03 \x01(\tH\x02R\nstackTrace\x88\x01\x01\x42\x14\n\x12_exception_messageB\x0e\n\x0c_error_classB\x0e\n\x0c_stack_trace\x1a\x38\n\x16\x41waitTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminatedB\r\n\x0bresult_type"\xbd\x06\n\x1cStreamingQueryManagerCommand\x12\x18\n\x06\x61\x63tive\x18\x01 \x01(\x08H\x00R\x06\x61\x63tive\x12\x1d\n\tget_query\x18\x02 \x01(\tH\x00R\x08getQuery\x12|\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32\x46.spark.connect.StreamingQueryManagerCommand.AwaitAnyTerminationCommandH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12n\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0b\x61\x64\x64Listener\x12t\n\x0fremove_listener\x18\x06 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0eremoveListener\x12\'\n\x0elist_listeners\x18\x07 \x01(\x08H\x00R\rlistListeners\x1aO\n\x1a\x41waitAnyTerminationCommand\x12"\n\ntimeout_ms\x18\x01 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_ms\x1a\xcd\x01\n\x1dStreamingQueryListenerCommand\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x12U\n\x17python_listener_payload\x18\x02 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x15pythonListenerPayload\x88\x01\x01\x12\x0e\n\x02id\x18\x03 \x01(\tR\x02idB\x1a\n\x18_python_listener_payloadB\t\n\x07\x63ommand"\xb4\x08\n"StreamingQueryManagerCommandResult\x12X\n\x06\x61\x63tive\x18\x01 \x01(\x0b\x32>.spark.connect.StreamingQueryManagerCommandResult.ActiveResultH\x00R\x06\x61\x63tive\x12`\n\x05query\x18\x02 \x01(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceH\x00R\x05query\x12\x81\x01\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32K.spark.connect.StreamingQueryManagerCommandResult.AwaitAnyTerminationResultH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12#\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x08H\x00R\x0b\x61\x64\x64Listener\x12)\n\x0fremove_listener\x18\x06 \x01(\x08H\x00R\x0eremoveListener\x12{\n\x0elist_listeners\x18\x07 \x01(\x0b\x32R.spark.connect.StreamingQueryManagerCommandResult.ListStreamingQueryListenerResultH\x00R\rlistListeners\x1a\x7f\n\x0c\x41\x63tiveResult\x12o\n\x0e\x61\x63tive_queries\x18\x01 \x03(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceR\ractiveQueries\x1as\n\x16StreamingQueryInstance\x12\x37\n\x02id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x02id\x12\x17\n\x04name\x18\x02 \x01(\tH\x00R\x04name\x88\x01\x01\x42\x07\n\x05_name\x1a;\n\x19\x41waitAnyTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminated\x1aK\n\x1eStreamingQueryListenerInstance\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x1a\x45\n ListStreamingQueryListenerResult\x12!\n\x0clistener_ids\x18\x01 \x03(\tR\x0blistenerIdsB\r\n\x0bresult_type"\xad\x01\n StreamingQueryListenerBusCommand\x12;\n\x19\x61\x64\x64_listener_bus_listener\x18\x01 \x01(\x08H\x00R\x16\x61\x64\x64ListenerBusListener\x12\x41\n\x1cremove_listener_bus_listener\x18\x02 \x01(\x08H\x00R\x19removeListenerBusListenerB\t\n\x07\x63ommand"\x83\x01\n\x1bStreamingQueryListenerEvent\x12\x1d\n\nevent_json\x18\x01 \x01(\tR\teventJson\x12\x45\n\nevent_type\x18\x02 \x01(\x0e\x32&.spark.connect.StreamingQueryEventTypeR\teventType"\xcc\x01\n"StreamingQueryListenerEventsResult\x12\x42\n\x06\x65vents\x18\x01 \x03(\x0b\x32*.spark.connect.StreamingQueryListenerEventR\x06\x65vents\x12\x42\n\x1blistener_bus_listener_added\x18\x02 \x01(\x08H\x00R\x18listenerBusListenerAdded\x88\x01\x01\x42\x1e\n\x1c_listener_bus_listener_added"\x15\n\x13GetResourcesCommand"\xd4\x01\n\x19GetResourcesCommandResult\x12U\n\tresources\x18\x01 \x03(\x0b\x32\x37.spark.connect.GetResourcesCommandResult.ResourcesEntryR\tresources\x1a`\n\x0eResourcesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32".spark.connect.ResourceInformationR\x05value:\x02\x38\x01"X\n\x1c\x43reateResourceProfileCommand\x12\x38\n\x07profile\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ResourceProfileR\x07profile"C\n"CreateResourceProfileCommandResult\x12\x1d\n\nprofile_id\x18\x01 \x01(\x05R\tprofileId"d\n!RemoveCachedRemoteRelationCommand\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation"\xcd\x01\n\x11\x43heckpointCommand\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x14\n\x05local\x18\x02 \x01(\x08R\x05local\x12\x14\n\x05\x65\x61ger\x18\x03 \x01(\x08R\x05\x65\x61ger\x12\x45\n\rstorage_level\x18\x04 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xe8\x03\n\x15MergeIntoTableCommand\x12*\n\x11target_table_name\x18\x01 \x01(\tR\x0ftargetTableName\x12\x43\n\x11source_table_plan\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x0fsourceTablePlan\x12\x42\n\x0fmerge_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0emergeCondition\x12>\n\rmatch_actions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0cmatchActions\x12I\n\x13not_matched_actions\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x11notMatchedActions\x12[\n\x1dnot_matched_by_source_actions\x18\x06 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x19notMatchedBySourceActions\x12\x32\n\x15with_schema_evolution\x18\x07 \x01(\x08R\x13withSchemaEvolution"\xd4\x01\n\x16\x45xecuteExternalCommand\x12\x16\n\x06runner\x18\x01 \x01(\tR\x06runner\x12\x18\n\x07\x63ommand\x18\x02 \x01(\tR\x07\x63ommand\x12L\n\x07options\x18\x03 \x03(\x0b\x32\x32.spark.connect.ExecuteExternalCommand.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01*\x85\x01\n\x17StreamingQueryEventType\x12\x1e\n\x1aQUERY_PROGRESS_UNSPECIFIED\x10\x00\x12\x18\n\x14QUERY_PROGRESS_EVENT\x10\x01\x12\x1a\n\x16QUERY_TERMINATED_EVENT\x10\x02\x12\x14\n\x10QUERY_IDLE_EVENT\x10\x03\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -83,118 +83,124 @@ _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_options = b"8\001" _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._loaded_options = None _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_options = b"8\001" - _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_start = 11920 - _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_end = 12053 + _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_start = 12339 + _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_end = 12472 _globals["_COMMAND"]._serialized_start = 222 - _globals["_COMMAND"]._serialized_end = 2137 - _globals["_SQLCOMMAND"]._serialized_start = 2140 - _globals["_SQLCOMMAND"]._serialized_end = 2694 - _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_start = 2510 - _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_end = 2600 - _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_start = 2602 - _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_end = 2694 - _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_start = 2697 - _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_end = 2847 - _globals["_WRITEOPERATION"]._serialized_start = 2850 - _globals["_WRITEOPERATION"]._serialized_end = 4000 - _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_start = 3424 - _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_end = 3482 - _globals["_WRITEOPERATION_SAVETABLE"]._serialized_start = 3485 - _globals["_WRITEOPERATION_SAVETABLE"]._serialized_end = 3743 - _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_start = 3619 - _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_end = 3743 - _globals["_WRITEOPERATION_BUCKETBY"]._serialized_start = 3745 - _globals["_WRITEOPERATION_BUCKETBY"]._serialized_end = 3836 - _globals["_WRITEOPERATION_SAVEMODE"]._serialized_start = 3839 - _globals["_WRITEOPERATION_SAVEMODE"]._serialized_end = 3976 - _globals["_WRITEOPERATIONV2"]._serialized_start = 4003 - _globals["_WRITEOPERATIONV2"]._serialized_end = 4915 - _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_start = 3424 - _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_end = 3482 - _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_start = 4674 - _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_end = 4740 - _globals["_WRITEOPERATIONV2_MODE"]._serialized_start = 4743 - _globals["_WRITEOPERATIONV2_MODE"]._serialized_end = 4902 - _globals["_WRITESTREAMOPERATIONSTART"]._serialized_start = 4918 - _globals["_WRITESTREAMOPERATIONSTART"]._serialized_end = 5833 - _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_start = 3424 - _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_end = 3482 - _globals["_STREAMINGFOREACHFUNCTION"]._serialized_start = 5836 - _globals["_STREAMINGFOREACHFUNCTION"]._serialized_end = 6015 - _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_start = 6018 - _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_end = 6230 - _globals["_STREAMINGQUERYINSTANCEID"]._serialized_start = 6232 - _globals["_STREAMINGQUERYINSTANCEID"]._serialized_end = 6297 - _globals["_STREAMINGQUERYCOMMAND"]._serialized_start = 6300 - _globals["_STREAMINGQUERYCOMMAND"]._serialized_end = 6932 - _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_start = 6799 - _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_end = 6843 - _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_start = 6845 - _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_end = 6921 - _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_start = 6935 - _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_end = 8076 - _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_start = 7518 - _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_end = 7688 - _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_start = 7690 - _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_end = 7762 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_start = 7764 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_end = 7803 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_start = 7806 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_end = 8003 - _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_start = 8005 - _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_end = 8061 - _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_start = 8079 - _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_end = 8908 - _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_start = 8610 - _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_end = 8689 - _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_start = 8692 - _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_end = 8897 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_start = 8911 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_end = 9987 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_start = 9519 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_end = 9646 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_start = 9648 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_end = 9763 + _globals["_COMMAND"]._serialized_end = 2344 + _globals["_CREATEBROADCASTCOMMAND"]._serialized_start = 2346 + _globals["_CREATEBROADCASTCOMMAND"]._serialized_end = 2438 + _globals["_UNPERSISTBROADCASTCOMMAND"]._serialized_start = 2440 + _globals["_UNPERSISTBROADCASTCOMMAND"]._serialized_end = 2556 + _globals["_SQLCOMMAND"]._serialized_start = 2559 + _globals["_SQLCOMMAND"]._serialized_end = 3113 + _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_start = 2929 + _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_end = 3019 + _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_start = 3021 + _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_end = 3113 + _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_start = 3116 + _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_end = 3266 + _globals["_WRITEOPERATION"]._serialized_start = 3269 + _globals["_WRITEOPERATION"]._serialized_end = 4419 + _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_start = 3843 + _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_end = 3901 + _globals["_WRITEOPERATION_SAVETABLE"]._serialized_start = 3904 + _globals["_WRITEOPERATION_SAVETABLE"]._serialized_end = 4162 + _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_start = 4038 + _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_end = 4162 + _globals["_WRITEOPERATION_BUCKETBY"]._serialized_start = 4164 + _globals["_WRITEOPERATION_BUCKETBY"]._serialized_end = 4255 + _globals["_WRITEOPERATION_SAVEMODE"]._serialized_start = 4258 + _globals["_WRITEOPERATION_SAVEMODE"]._serialized_end = 4395 + _globals["_WRITEOPERATIONV2"]._serialized_start = 4422 + _globals["_WRITEOPERATIONV2"]._serialized_end = 5334 + _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_start = 3843 + _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_end = 3901 + _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_start = 5093 + _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_end = 5159 + _globals["_WRITEOPERATIONV2_MODE"]._serialized_start = 5162 + _globals["_WRITEOPERATIONV2_MODE"]._serialized_end = 5321 + _globals["_WRITESTREAMOPERATIONSTART"]._serialized_start = 5337 + _globals["_WRITESTREAMOPERATIONSTART"]._serialized_end = 6252 + _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_start = 3843 + _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_end = 3901 + _globals["_STREAMINGFOREACHFUNCTION"]._serialized_start = 6255 + _globals["_STREAMINGFOREACHFUNCTION"]._serialized_end = 6434 + _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_start = 6437 + _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_end = 6649 + _globals["_STREAMINGQUERYINSTANCEID"]._serialized_start = 6651 + _globals["_STREAMINGQUERYINSTANCEID"]._serialized_end = 6716 + _globals["_STREAMINGQUERYCOMMAND"]._serialized_start = 6719 + _globals["_STREAMINGQUERYCOMMAND"]._serialized_end = 7351 + _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_start = 7218 + _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_end = 7262 + _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_start = 7264 + _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_end = 7340 + _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_start = 7354 + _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_end = 8495 + _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_start = 7937 + _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_end = 8107 + _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_start = 8109 + _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_end = 8181 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_start = 8183 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_end = 8222 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_start = 8225 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_end = 8422 + _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_start = 8424 + _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_end = 8480 + _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_start = 8498 + _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_end = 9327 + _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_start = 9029 + _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_end = 9108 + _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_start = 9111 + _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_end = 9316 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_start = 9330 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_end = 10406 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_start = 9938 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_end = 10065 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_start = 10067 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_end = 10182 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT" - ]._serialized_start = 9765 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT"]._serialized_end = 9824 + ]._serialized_start = 10184 + _globals[ + "_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT" + ]._serialized_end = 10243 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE" - ]._serialized_start = 9826 + ]._serialized_start = 10245 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE" - ]._serialized_end = 9901 + ]._serialized_end = 10320 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_LISTSTREAMINGQUERYLISTENERRESULT" - ]._serialized_start = 9903 + ]._serialized_start = 10322 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_LISTSTREAMINGQUERYLISTENERRESULT" - ]._serialized_end = 9972 - _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_start = 9990 - _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_end = 10163 - _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_start = 10166 - _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_end = 10297 - _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_start = 10300 - _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_end = 10504 - _globals["_GETRESOURCESCOMMAND"]._serialized_start = 10506 - _globals["_GETRESOURCESCOMMAND"]._serialized_end = 10527 - _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_start = 10530 - _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_end = 10742 - _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_start = 10646 - _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_end = 10742 - _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_start = 10744 - _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_end = 10832 - _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_start = 10834 - _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_end = 10901 - _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_start = 10903 - _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_end = 11003 - _globals["_CHECKPOINTCOMMAND"]._serialized_start = 11006 - _globals["_CHECKPOINTCOMMAND"]._serialized_end = 11211 - _globals["_MERGEINTOTABLECOMMAND"]._serialized_start = 11214 - _globals["_MERGEINTOTABLECOMMAND"]._serialized_end = 11702 - _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_start = 11705 - _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_end = 11917 - _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_start = 3424 - _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_end = 3482 + ]._serialized_end = 10391 + _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_start = 10409 + _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_end = 10582 + _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_start = 10585 + _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_end = 10716 + _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_start = 10719 + _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_end = 10923 + _globals["_GETRESOURCESCOMMAND"]._serialized_start = 10925 + _globals["_GETRESOURCESCOMMAND"]._serialized_end = 10946 + _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_start = 10949 + _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_end = 11161 + _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_start = 11065 + _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_end = 11161 + _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_start = 11163 + _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_end = 11251 + _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_start = 11253 + _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_end = 11320 + _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_start = 11322 + _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_end = 11422 + _globals["_CHECKPOINTCOMMAND"]._serialized_start = 11425 + _globals["_CHECKPOINTCOMMAND"]._serialized_end = 11630 + _globals["_MERGEINTOTABLECOMMAND"]._serialized_start = 11633 + _globals["_MERGEINTOTABLECOMMAND"]._serialized_end = 12121 + _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_start = 12124 + _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_end = 12336 + _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_start = 3843 + _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_end = 3901 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi index cc1330f11f8ac..43ba7795bbd4f 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.pyi +++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi @@ -110,6 +110,8 @@ class Command(google.protobuf.message.Message): ML_COMMAND_FIELD_NUMBER: builtins.int EXECUTE_EXTERNAL_COMMAND_FIELD_NUMBER: builtins.int PIPELINE_COMMAND_FIELD_NUMBER: builtins.int + CREATE_BROADCAST_COMMAND_FIELD_NUMBER: builtins.int + UNPERSIST_BROADCAST_COMMAND_FIELD_NUMBER: builtins.int EXTENSION_FIELD_NUMBER: builtins.int @property def register_function( @@ -158,6 +160,10 @@ class Command(google.protobuf.message.Message): @property def pipeline_command(self) -> pyspark.sql.connect.proto.pipelines_pb2.PipelineCommand: ... @property + def create_broadcast_command(self) -> global___CreateBroadcastCommand: ... + @property + def unpersist_broadcast_command(self) -> global___UnpersistBroadcastCommand: ... + @property def extension(self) -> google.protobuf.any_pb2.Any: """This field is used to mark extensions to the protocol. When plugins generate arbitrary Commands they can add them here. During the planning the correct resolution is done. @@ -189,6 +195,8 @@ class Command(google.protobuf.message.Message): ml_command: pyspark.sql.connect.proto.ml_pb2.MlCommand | None = ..., execute_external_command: global___ExecuteExternalCommand | None = ..., pipeline_command: pyspark.sql.connect.proto.pipelines_pb2.PipelineCommand | None = ..., + create_broadcast_command: global___CreateBroadcastCommand | None = ..., + unpersist_broadcast_command: global___UnpersistBroadcastCommand | None = ..., extension: google.protobuf.any_pb2.Any | None = ..., ) -> None: ... def HasField( @@ -198,6 +206,8 @@ class Command(google.protobuf.message.Message): b"checkpoint_command", "command_type", b"command_type", + "create_broadcast_command", + b"create_broadcast_command", "create_dataframe_view", b"create_dataframe_view", "create_resource_profile_command", @@ -230,6 +240,8 @@ class Command(google.protobuf.message.Message): b"streaming_query_listener_bus_command", "streaming_query_manager_command", b"streaming_query_manager_command", + "unpersist_broadcast_command", + b"unpersist_broadcast_command", "write_operation", b"write_operation", "write_operation_v2", @@ -245,6 +257,8 @@ class Command(google.protobuf.message.Message): b"checkpoint_command", "command_type", b"command_type", + "create_broadcast_command", + b"create_broadcast_command", "create_dataframe_view", b"create_dataframe_view", "create_resource_profile_command", @@ -277,6 +291,8 @@ class Command(google.protobuf.message.Message): b"streaming_query_listener_bus_command", "streaming_query_manager_command", b"streaming_query_manager_command", + "unpersist_broadcast_command", + b"unpersist_broadcast_command", "write_operation", b"write_operation", "write_operation_v2", @@ -308,6 +324,8 @@ class Command(google.protobuf.message.Message): "ml_command", "execute_external_command", "pipeline_command", + "create_broadcast_command", + "unpersist_broadcast_command", "extension", ] | None @@ -315,6 +333,66 @@ class Command(google.protobuf.message.Message): global___Command = Command +class CreateBroadcastCommand(google.protobuf.message.Message): + """(SPARK-51705) Create a broadcast variable from an already-uploaded cache/ artifact. + The client uploads cloudpickle(value) through the existing cache artifact channel and then + sends this command with the returned hash. The server materializes a PythonBroadcast on the + live driver SparkContext and returns a CreateBroadcastResult with the driver-side broadcast id. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ARTIFACT_HASH_FIELD_NUMBER: builtins.int + SIZE_BYTES_FIELD_NUMBER: builtins.int + artifact_hash: builtins.str + """(Required) sha256 hash returned by client.cache_artifact(cloudpickle(value)).""" + size_bytes: builtins.int + """(Optional) Uncompressed byte size, used to enforce the BROADCAST_VALUE_TOO_LARGE quota.""" + def __init__( + self, + *, + artifact_hash: builtins.str = ..., + size_bytes: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "artifact_hash", b"artifact_hash", "size_bytes", b"size_bytes" + ], + ) -> None: ... + +global___CreateBroadcastCommand = CreateBroadcastCommand + +class UnpersistBroadcastCommand(google.protobuf.message.Message): + """(SPARK-51705) Release a broadcast variable created over Spark Connect.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + BROADCAST_ID_FIELD_NUMBER: builtins.int + BLOCKING_FIELD_NUMBER: builtins.int + DESTROY_FIELD_NUMBER: builtins.int + broadcast_id: builtins.int + """(Required) The driver-side broadcast id returned by CreateBroadcastResult.""" + blocking: builtins.bool + """(Optional) Whether to block until unpersisting has completed.""" + destroy: builtins.bool + """(Optional) Whether to destroy (not just unpersist) all data and metadata of the broadcast.""" + def __init__( + self, + *, + broadcast_id: builtins.int = ..., + blocking: builtins.bool = ..., + destroy: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "blocking", b"blocking", "broadcast_id", b"broadcast_id", "destroy", b"destroy" + ], + ) -> None: ... + +global___UnpersistBroadcastCommand = UnpersistBroadcastCommand + class SqlCommand(google.protobuf.message.Message): """A SQL Command is used to trigger the eager evaluation of SQL commands in Spark. diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.py b/python/pyspark/sql/connect/proto/expressions_pb2.py index aa51c393c043f..28e96fb3a00bc 100644 --- a/python/pyspark/sql/connect/proto/expressions_pb2.py +++ b/python/pyspark/sql/connect/proto/expressions_pb2.py @@ -41,7 +41,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\x90<\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFunction\x12Y\n\x11\x65xpression_string\x18\x04 \x01(\x0b\x32*.spark.connect.Expression.ExpressionStringH\x00R\x10\x65xpressionString\x12S\n\x0funresolved_star\x18\x05 \x01(\x0b\x32(.spark.connect.Expression.UnresolvedStarH\x00R\x0eunresolvedStar\x12\x37\n\x05\x61lias\x18\x06 \x01(\x0b\x32\x1f.spark.connect.Expression.AliasH\x00R\x05\x61lias\x12\x34\n\x04\x63\x61st\x18\x07 \x01(\x0b\x32\x1e.spark.connect.Expression.CastH\x00R\x04\x63\x61st\x12V\n\x10unresolved_regex\x18\x08 \x01(\x0b\x32).spark.connect.Expression.UnresolvedRegexH\x00R\x0funresolvedRegex\x12\x44\n\nsort_order\x18\t \x01(\x0b\x32#.spark.connect.Expression.SortOrderH\x00R\tsortOrder\x12S\n\x0flambda_function\x18\n \x01(\x0b\x32(.spark.connect.Expression.LambdaFunctionH\x00R\x0elambdaFunction\x12:\n\x06window\x18\x0b \x01(\x0b\x32 .spark.connect.Expression.WindowH\x00R\x06window\x12l\n\x18unresolved_extract_value\x18\x0c \x01(\x0b\x32\x30.spark.connect.Expression.UnresolvedExtractValueH\x00R\x16unresolvedExtractValue\x12M\n\rupdate_fields\x18\r \x01(\x0b\x32&.spark.connect.Expression.UpdateFieldsH\x00R\x0cupdateFields\x12\x82\x01\n unresolved_named_lambda_variable\x18\x0e \x01(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableH\x00R\x1dunresolvedNamedLambdaVariable\x12~\n#common_inline_user_defined_function\x18\x0f \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x1f\x63ommonInlineUserDefinedFunction\x12\x42\n\rcall_function\x18\x10 \x01(\x0b\x32\x1b.spark.connect.CallFunctionH\x00R\x0c\x63\x61llFunction\x12\x64\n\x19named_argument_expression\x18\x11 \x01(\x0b\x32&.spark.connect.NamedArgumentExpressionH\x00R\x17namedArgumentExpression\x12?\n\x0cmerge_action\x18\x13 \x01(\x0b\x32\x1a.spark.connect.MergeActionH\x00R\x0bmergeAction\x12g\n\x1atyped_aggregate_expression\x18\x14 \x01(\x0b\x32\'.spark.connect.TypedAggregateExpressionH\x00R\x18typedAggregateExpression\x12T\n\x13subquery_expression\x18\x15 \x01(\x0b\x32!.spark.connect.SubqueryExpressionH\x00R\x12subqueryExpression\x12s\n\x1b\x64irect_shuffle_partition_id\x18\x16 \x01(\x0b\x32\x32.spark.connect.Expression.DirectShufflePartitionIDH\x00R\x18\x64irectShufflePartitionId\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x1a\x8f\x06\n\x06Window\x12\x42\n\x0fwindow_function\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0ewindowFunction\x12@\n\x0epartition_spec\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\rpartitionSpec\x12\x42\n\norder_spec\x18\x03 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\torderSpec\x12K\n\nframe_spec\x18\x04 \x01(\x0b\x32,.spark.connect.Expression.Window.WindowFrameR\tframeSpec\x1a\xed\x03\n\x0bWindowFrame\x12U\n\nframe_type\x18\x01 \x01(\x0e\x32\x36.spark.connect.Expression.Window.WindowFrame.FrameTypeR\tframeType\x12P\n\x05lower\x18\x02 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05lower\x12P\n\x05upper\x18\x03 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05upper\x1a\x91\x01\n\rFrameBoundary\x12!\n\x0b\x63urrent_row\x18\x01 \x01(\x08H\x00R\ncurrentRow\x12\x1e\n\tunbounded\x18\x02 \x01(\x08H\x00R\tunbounded\x12\x31\n\x05value\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\x05valueB\n\n\x08\x62oundary"O\n\tFrameType\x12\x18\n\x14\x46RAME_TYPE_UNDEFINED\x10\x00\x12\x12\n\x0e\x46RAME_TYPE_ROW\x10\x01\x12\x14\n\x10\x46RAME_TYPE_RANGE\x10\x02\x1a\xa9\x03\n\tSortOrder\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12O\n\tdirection\x18\x02 \x01(\x0e\x32\x31.spark.connect.Expression.SortOrder.SortDirectionR\tdirection\x12U\n\rnull_ordering\x18\x03 \x01(\x0e\x32\x30.spark.connect.Expression.SortOrder.NullOrderingR\x0cnullOrdering"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"U\n\x0cNullOrdering\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02\x1aK\n\x18\x44irectShufflePartitionID\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x1a\xbb\x02\n\x04\x43\x61st\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12-\n\x04type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04type\x12\x1b\n\x08type_str\x18\x03 \x01(\tH\x00R\x07typeStr\x12\x44\n\teval_mode\x18\x04 \x01(\x0e\x32\'.spark.connect.Expression.Cast.EvalModeR\x08\x65valMode"b\n\x08\x45valMode\x12\x19\n\x15\x45VAL_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10\x45VAL_MODE_LEGACY\x10\x01\x12\x12\n\x0e\x45VAL_MODE_ANSI\x10\x02\x12\x11\n\rEVAL_MODE_TRY\x10\x03\x42\x0e\n\x0c\x63\x61st_to_type\x1a\x9c\x15\n\x07Literal\x12-\n\x04null\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04null\x12\x18\n\x06\x62inary\x18\x02 \x01(\x0cH\x00R\x06\x62inary\x12\x1a\n\x07\x62oolean\x18\x03 \x01(\x08H\x00R\x07\x62oolean\x12\x14\n\x04\x62yte\x18\x04 \x01(\x05H\x00R\x04\x62yte\x12\x16\n\x05short\x18\x05 \x01(\x05H\x00R\x05short\x12\x1a\n\x07integer\x18\x06 \x01(\x05H\x00R\x07integer\x12\x14\n\x04long\x18\x07 \x01(\x03H\x00R\x04long\x12\x16\n\x05\x66loat\x18\n \x01(\x02H\x00R\x05\x66loat\x12\x18\n\x06\x64ouble\x18\x0b \x01(\x01H\x00R\x06\x64ouble\x12\x45\n\x07\x64\x65\x63imal\x18\x0c \x01(\x0b\x32).spark.connect.Expression.Literal.DecimalH\x00R\x07\x64\x65\x63imal\x12\x18\n\x06string\x18\r \x01(\tH\x00R\x06string\x12\x14\n\x04\x64\x61te\x18\x10 \x01(\x05H\x00R\x04\x64\x61te\x12\x1e\n\ttimestamp\x18\x11 \x01(\x03H\x00R\ttimestamp\x12%\n\rtimestamp_ntz\x18\x12 \x01(\x03H\x00R\x0ctimestampNtz\x12\x61\n\x11\x63\x61lendar_interval\x18\x13 \x01(\x0b\x32\x32.spark.connect.Expression.Literal.CalendarIntervalH\x00R\x10\x63\x61lendarInterval\x12\x30\n\x13year_month_interval\x18\x14 \x01(\x05H\x00R\x11yearMonthInterval\x12,\n\x11\x64\x61y_time_interval\x18\x15 \x01(\x03H\x00R\x0f\x64\x61yTimeInterval\x12?\n\x05\x61rray\x18\x16 \x01(\x0b\x32\'.spark.connect.Expression.Literal.ArrayH\x00R\x05\x61rray\x12\x39\n\x03map\x18\x17 \x01(\x0b\x32%.spark.connect.Expression.Literal.MapH\x00R\x03map\x12\x42\n\x06struct\x18\x18 \x01(\x0b\x32(.spark.connect.Expression.Literal.StructH\x00R\x06struct\x12\x61\n\x11specialized_array\x18\x19 \x01(\x0b\x32\x32.spark.connect.Expression.Literal.SpecializedArrayH\x00R\x10specializedArray\x12<\n\x04time\x18\x1a \x01(\x0b\x32&.spark.connect.Expression.Literal.TimeH\x00R\x04time\x12\x65\n\x13timestamp_ntz_nanos\x18\x1d \x01(\x0b\x32\x33.spark.connect.Expression.Literal.TimestampNTZNanosH\x00R\x11timestampNtzNanos\x12\x65\n\x13timestamp_ltz_nanos\x18\x1e \x01(\x0b\x32\x33.spark.connect.Expression.Literal.TimestampLTZNanosH\x00R\x11timestampLtzNanos\x12\x34\n\tdata_type\x18\x64 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x08\x64\x61taType\x1au\n\x07\x44\x65\x63imal\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\x12!\n\tprecision\x18\x02 \x01(\x05H\x00R\tprecision\x88\x01\x01\x12\x19\n\x05scale\x18\x03 \x01(\x05H\x01R\x05scale\x88\x01\x01\x42\x0c\n\n_precisionB\x08\n\x06_scale\x1a\x62\n\x10\x43\x61lendarInterval\x12\x16\n\x06months\x18\x01 \x01(\x05R\x06months\x12\x12\n\x04\x64\x61ys\x18\x02 \x01(\x05R\x04\x64\x61ys\x12"\n\x0cmicroseconds\x18\x03 \x01(\x03R\x0cmicroseconds\x1a\x86\x01\n\x05\x41rray\x12>\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\x0b\x65lementType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lements\x1a\xeb\x01\n\x03Map\x12\x36\n\x08key_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\x07keyType\x12:\n\nvalue_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\tvalueType\x12\x35\n\x04keys\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x04keys\x12\x39\n\x06values\x18\x04 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1a\x85\x01\n\x06Struct\x12<\n\x0bstruct_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\nstructType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lements\x1a\xc0\x02\n\x10SpecializedArray\x12,\n\x05\x62ools\x18\x01 \x01(\x0b\x32\x14.spark.connect.BoolsH\x00R\x05\x62ools\x12)\n\x04ints\x18\x02 \x01(\x0b\x32\x13.spark.connect.IntsH\x00R\x04ints\x12,\n\x05longs\x18\x03 \x01(\x0b\x32\x14.spark.connect.LongsH\x00R\x05longs\x12/\n\x06\x66loats\x18\x04 \x01(\x0b\x32\x15.spark.connect.FloatsH\x00R\x06\x66loats\x12\x32\n\x07\x64oubles\x18\x05 \x01(\x0b\x32\x16.spark.connect.DoublesH\x00R\x07\x64oubles\x12\x32\n\x07strings\x18\x06 \x01(\x0b\x32\x16.spark.connect.StringsH\x00R\x07stringsB\x0c\n\nvalue_type\x1aK\n\x04Time\x12\x12\n\x04nano\x18\x01 \x01(\x03R\x04nano\x12!\n\tprecision\x18\x02 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precision\x1a\x95\x01\n\x11TimestampNTZNanos\x12!\n\x0c\x65poch_micros\x18\x01 \x01(\x03R\x0b\x65pochMicros\x12,\n\x12nanos_within_micro\x18\x02 \x01(\x05R\x10nanosWithinMicro\x12!\n\tprecision\x18\x03 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precision\x1a\x95\x01\n\x11TimestampLTZNanos\x12!\n\x0c\x65poch_micros\x18\x01 \x01(\x03R\x0b\x65pochMicros\x12,\n\x12nanos_within_micro\x18\x02 \x01(\x05R\x10nanosWithinMicro\x12!\n\tprecision\x18\x03 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precisionB\x0e\n\x0cliteral_typeJ\x04\x08\x1b\x10\x1cJ\x04\x08\x1c\x10\x1d\x1a\xba\x01\n\x13UnresolvedAttribute\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x12\x31\n\x12is_metadata_column\x18\x03 \x01(\x08H\x01R\x10isMetadataColumn\x88\x01\x01\x42\n\n\x08_plan_idB\x15\n\x13_is_metadata_column\x1a\x82\x02\n\x12UnresolvedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x1f\n\x0bis_distinct\x18\x03 \x01(\x08R\nisDistinct\x12\x37\n\x18is_user_defined_function\x18\x04 \x01(\x08R\x15isUserDefinedFunction\x12$\n\x0bis_internal\x18\x05 \x01(\x08H\x00R\nisInternal\x88\x01\x01\x42\x0e\n\x0c_is_internal\x1a\x32\n\x10\x45xpressionString\x12\x1e\n\nexpression\x18\x01 \x01(\tR\nexpression\x1a|\n\x0eUnresolvedStar\x12,\n\x0funparsed_target\x18\x01 \x01(\tH\x00R\x0eunparsedTarget\x88\x01\x01\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x01R\x06planId\x88\x01\x01\x42\x12\n\x10_unparsed_targetB\n\n\x08_plan_id\x1aV\n\x0fUnresolvedRegex\x12\x19\n\x08\x63ol_name\x18\x01 \x01(\tR\x07\x63olName\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x42\n\n\x08_plan_id\x1a\x84\x01\n\x16UnresolvedExtractValue\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12\x39\n\nextraction\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nextraction\x1a\xbb\x01\n\x0cUpdateFields\x12\x46\n\x11struct_expression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x10structExpression\x12\x1d\n\nfield_name\x18\x02 \x01(\tR\tfieldName\x12\x44\n\x10value_expression\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0fvalueExpression\x1ax\n\x05\x41lias\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12\x12\n\x04name\x18\x02 \x03(\tR\x04name\x12\x1f\n\x08metadata\x18\x03 \x01(\tH\x00R\x08metadata\x88\x01\x01\x42\x0b\n\t_metadata\x1a\x9e\x01\n\x0eLambdaFunction\x12\x35\n\x08\x66unction\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x08\x66unction\x12U\n\targuments\x18\x02 \x03(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableR\targuments\x1a>\n\x1dUnresolvedNamedLambdaVariable\x12\x1d\n\nname_parts\x18\x01 \x03(\tR\tnamePartsB\x0b\n\texpr_type"A\n\x10\x45xpressionCommon\x12-\n\x06origin\x18\x01 \x01(\x0b\x32\x15.spark.connect.OriginR\x06origin"\x8d\x03\n\x1f\x43ommonInlineUserDefinedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12$\n\rdeterministic\x18\x02 \x01(\x08R\rdeterministic\x12\x37\n\targuments\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x39\n\npython_udf\x18\x04 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\tpythonUdf\x12I\n\x10scalar_scala_udf\x18\x05 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\x0escalarScalaUdf\x12\x33\n\x08java_udf\x18\x06 \x01(\x0b\x32\x16.spark.connect.JavaUDFH\x00R\x07javaUdf\x12\x1f\n\x0bis_distinct\x18\x07 \x01(\x08R\nisDistinctB\n\n\x08\x66unction"\xcc\x01\n\tPythonUDF\x12\x38\n\x0boutput_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1b\n\teval_type\x18\x02 \x01(\x05R\x08\x65valType\x12\x18\n\x07\x63ommand\x18\x03 \x01(\x0cR\x07\x63ommand\x12\x1d\n\npython_ver\x18\x04 \x01(\tR\tpythonVer\x12/\n\x13\x61\x64\x64itional_includes\x18\x05 \x03(\tR\x12\x61\x64\x64itionalIncludes"\xd6\x01\n\x0eScalarScalaUDF\x12\x18\n\x07payload\x18\x01 \x01(\x0cR\x07payload\x12\x37\n\ninputTypes\x18\x02 \x03(\x0b\x32\x17.spark.connect.DataTypeR\ninputTypes\x12\x37\n\noutputType\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1a\n\x08nullable\x18\x04 \x01(\x08R\x08nullable\x12\x1c\n\taggregate\x18\x05 \x01(\x08R\taggregate"\x95\x01\n\x07JavaUDF\x12\x1d\n\nclass_name\x18\x01 \x01(\tR\tclassName\x12=\n\x0boutput_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\noutputType\x88\x01\x01\x12\x1c\n\taggregate\x18\x03 \x01(\x08R\taggregateB\x0e\n\x0c_output_type"c\n\x18TypedAggregateExpression\x12G\n\x10scalar_scala_udf\x18\x01 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFR\x0escalarScalaUdf"l\n\x0c\x43\x61llFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"\\\n\x17NamedArgumentExpression\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\x80\x04\n\x0bMergeAction\x12\x46\n\x0b\x61\x63tion_type\x18\x01 \x01(\x0e\x32%.spark.connect.MergeAction.ActionTypeR\nactionType\x12<\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\tcondition\x88\x01\x01\x12G\n\x0b\x61ssignments\x18\x03 \x03(\x0b\x32%.spark.connect.MergeAction.AssignmentR\x0b\x61ssignments\x1aj\n\nAssignment\x12+\n\x03key\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\xa7\x01\n\nActionType\x12\x17\n\x13\x41\x43TION_TYPE_INVALID\x10\x00\x12\x16\n\x12\x41\x43TION_TYPE_DELETE\x10\x01\x12\x16\n\x12\x41\x43TION_TYPE_INSERT\x10\x02\x12\x1b\n\x17\x41\x43TION_TYPE_INSERT_STAR\x10\x03\x12\x16\n\x12\x41\x43TION_TYPE_UPDATE\x10\x04\x12\x1b\n\x17\x41\x43TION_TYPE_UPDATE_STAR\x10\x05\x42\x0c\n\n_condition"\xc5\x05\n\x12SubqueryExpression\x12\x17\n\x07plan_id\x18\x01 \x01(\x03R\x06planId\x12S\n\rsubquery_type\x18\x02 \x01(\x0e\x32..spark.connect.SubqueryExpression.SubqueryTypeR\x0csubqueryType\x12\x62\n\x11table_arg_options\x18\x03 \x01(\x0b\x32\x31.spark.connect.SubqueryExpression.TableArgOptionsH\x00R\x0ftableArgOptions\x88\x01\x01\x12G\n\x12in_subquery_values\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x10inSubqueryValues\x1a\xea\x01\n\x0fTableArgOptions\x12@\n\x0epartition_spec\x18\x01 \x03(\x0b\x32\x19.spark.connect.ExpressionR\rpartitionSpec\x12\x42\n\norder_spec\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\torderSpec\x12\x37\n\x15with_single_partition\x18\x03 \x01(\x08H\x00R\x13withSinglePartition\x88\x01\x01\x42\x18\n\x16_with_single_partition"\x90\x01\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x12\x1b\n\x17SUBQUERY_TYPE_TABLE_ARG\x10\x03\x12\x14\n\x10SUBQUERY_TYPE_IN\x10\x04\x42\x14\n\x12_table_arg_optionsB6\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\x90<\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFunction\x12Y\n\x11\x65xpression_string\x18\x04 \x01(\x0b\x32*.spark.connect.Expression.ExpressionStringH\x00R\x10\x65xpressionString\x12S\n\x0funresolved_star\x18\x05 \x01(\x0b\x32(.spark.connect.Expression.UnresolvedStarH\x00R\x0eunresolvedStar\x12\x37\n\x05\x61lias\x18\x06 \x01(\x0b\x32\x1f.spark.connect.Expression.AliasH\x00R\x05\x61lias\x12\x34\n\x04\x63\x61st\x18\x07 \x01(\x0b\x32\x1e.spark.connect.Expression.CastH\x00R\x04\x63\x61st\x12V\n\x10unresolved_regex\x18\x08 \x01(\x0b\x32).spark.connect.Expression.UnresolvedRegexH\x00R\x0funresolvedRegex\x12\x44\n\nsort_order\x18\t \x01(\x0b\x32#.spark.connect.Expression.SortOrderH\x00R\tsortOrder\x12S\n\x0flambda_function\x18\n \x01(\x0b\x32(.spark.connect.Expression.LambdaFunctionH\x00R\x0elambdaFunction\x12:\n\x06window\x18\x0b \x01(\x0b\x32 .spark.connect.Expression.WindowH\x00R\x06window\x12l\n\x18unresolved_extract_value\x18\x0c \x01(\x0b\x32\x30.spark.connect.Expression.UnresolvedExtractValueH\x00R\x16unresolvedExtractValue\x12M\n\rupdate_fields\x18\r \x01(\x0b\x32&.spark.connect.Expression.UpdateFieldsH\x00R\x0cupdateFields\x12\x82\x01\n unresolved_named_lambda_variable\x18\x0e \x01(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableH\x00R\x1dunresolvedNamedLambdaVariable\x12~\n#common_inline_user_defined_function\x18\x0f \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x1f\x63ommonInlineUserDefinedFunction\x12\x42\n\rcall_function\x18\x10 \x01(\x0b\x32\x1b.spark.connect.CallFunctionH\x00R\x0c\x63\x61llFunction\x12\x64\n\x19named_argument_expression\x18\x11 \x01(\x0b\x32&.spark.connect.NamedArgumentExpressionH\x00R\x17namedArgumentExpression\x12?\n\x0cmerge_action\x18\x13 \x01(\x0b\x32\x1a.spark.connect.MergeActionH\x00R\x0bmergeAction\x12g\n\x1atyped_aggregate_expression\x18\x14 \x01(\x0b\x32\'.spark.connect.TypedAggregateExpressionH\x00R\x18typedAggregateExpression\x12T\n\x13subquery_expression\x18\x15 \x01(\x0b\x32!.spark.connect.SubqueryExpressionH\x00R\x12subqueryExpression\x12s\n\x1b\x64irect_shuffle_partition_id\x18\x16 \x01(\x0b\x32\x32.spark.connect.Expression.DirectShufflePartitionIDH\x00R\x18\x64irectShufflePartitionId\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x1a\x8f\x06\n\x06Window\x12\x42\n\x0fwindow_function\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0ewindowFunction\x12@\n\x0epartition_spec\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\rpartitionSpec\x12\x42\n\norder_spec\x18\x03 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\torderSpec\x12K\n\nframe_spec\x18\x04 \x01(\x0b\x32,.spark.connect.Expression.Window.WindowFrameR\tframeSpec\x1a\xed\x03\n\x0bWindowFrame\x12U\n\nframe_type\x18\x01 \x01(\x0e\x32\x36.spark.connect.Expression.Window.WindowFrame.FrameTypeR\tframeType\x12P\n\x05lower\x18\x02 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05lower\x12P\n\x05upper\x18\x03 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05upper\x1a\x91\x01\n\rFrameBoundary\x12!\n\x0b\x63urrent_row\x18\x01 \x01(\x08H\x00R\ncurrentRow\x12\x1e\n\tunbounded\x18\x02 \x01(\x08H\x00R\tunbounded\x12\x31\n\x05value\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\x05valueB\n\n\x08\x62oundary"O\n\tFrameType\x12\x18\n\x14\x46RAME_TYPE_UNDEFINED\x10\x00\x12\x12\n\x0e\x46RAME_TYPE_ROW\x10\x01\x12\x14\n\x10\x46RAME_TYPE_RANGE\x10\x02\x1a\xa9\x03\n\tSortOrder\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12O\n\tdirection\x18\x02 \x01(\x0e\x32\x31.spark.connect.Expression.SortOrder.SortDirectionR\tdirection\x12U\n\rnull_ordering\x18\x03 \x01(\x0e\x32\x30.spark.connect.Expression.SortOrder.NullOrderingR\x0cnullOrdering"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"U\n\x0cNullOrdering\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02\x1aK\n\x18\x44irectShufflePartitionID\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x1a\xbb\x02\n\x04\x43\x61st\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12-\n\x04type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04type\x12\x1b\n\x08type_str\x18\x03 \x01(\tH\x00R\x07typeStr\x12\x44\n\teval_mode\x18\x04 \x01(\x0e\x32\'.spark.connect.Expression.Cast.EvalModeR\x08\x65valMode"b\n\x08\x45valMode\x12\x19\n\x15\x45VAL_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10\x45VAL_MODE_LEGACY\x10\x01\x12\x12\n\x0e\x45VAL_MODE_ANSI\x10\x02\x12\x11\n\rEVAL_MODE_TRY\x10\x03\x42\x0e\n\x0c\x63\x61st_to_type\x1a\x9c\x15\n\x07Literal\x12-\n\x04null\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04null\x12\x18\n\x06\x62inary\x18\x02 \x01(\x0cH\x00R\x06\x62inary\x12\x1a\n\x07\x62oolean\x18\x03 \x01(\x08H\x00R\x07\x62oolean\x12\x14\n\x04\x62yte\x18\x04 \x01(\x05H\x00R\x04\x62yte\x12\x16\n\x05short\x18\x05 \x01(\x05H\x00R\x05short\x12\x1a\n\x07integer\x18\x06 \x01(\x05H\x00R\x07integer\x12\x14\n\x04long\x18\x07 \x01(\x03H\x00R\x04long\x12\x16\n\x05\x66loat\x18\n \x01(\x02H\x00R\x05\x66loat\x12\x18\n\x06\x64ouble\x18\x0b \x01(\x01H\x00R\x06\x64ouble\x12\x45\n\x07\x64\x65\x63imal\x18\x0c \x01(\x0b\x32).spark.connect.Expression.Literal.DecimalH\x00R\x07\x64\x65\x63imal\x12\x18\n\x06string\x18\r \x01(\tH\x00R\x06string\x12\x14\n\x04\x64\x61te\x18\x10 \x01(\x05H\x00R\x04\x64\x61te\x12\x1e\n\ttimestamp\x18\x11 \x01(\x03H\x00R\ttimestamp\x12%\n\rtimestamp_ntz\x18\x12 \x01(\x03H\x00R\x0ctimestampNtz\x12\x61\n\x11\x63\x61lendar_interval\x18\x13 \x01(\x0b\x32\x32.spark.connect.Expression.Literal.CalendarIntervalH\x00R\x10\x63\x61lendarInterval\x12\x30\n\x13year_month_interval\x18\x14 \x01(\x05H\x00R\x11yearMonthInterval\x12,\n\x11\x64\x61y_time_interval\x18\x15 \x01(\x03H\x00R\x0f\x64\x61yTimeInterval\x12?\n\x05\x61rray\x18\x16 \x01(\x0b\x32\'.spark.connect.Expression.Literal.ArrayH\x00R\x05\x61rray\x12\x39\n\x03map\x18\x17 \x01(\x0b\x32%.spark.connect.Expression.Literal.MapH\x00R\x03map\x12\x42\n\x06struct\x18\x18 \x01(\x0b\x32(.spark.connect.Expression.Literal.StructH\x00R\x06struct\x12\x61\n\x11specialized_array\x18\x19 \x01(\x0b\x32\x32.spark.connect.Expression.Literal.SpecializedArrayH\x00R\x10specializedArray\x12<\n\x04time\x18\x1a \x01(\x0b\x32&.spark.connect.Expression.Literal.TimeH\x00R\x04time\x12\x65\n\x13timestamp_ntz_nanos\x18\x1d \x01(\x0b\x32\x33.spark.connect.Expression.Literal.TimestampNTZNanosH\x00R\x11timestampNtzNanos\x12\x65\n\x13timestamp_ltz_nanos\x18\x1e \x01(\x0b\x32\x33.spark.connect.Expression.Literal.TimestampLTZNanosH\x00R\x11timestampLtzNanos\x12\x34\n\tdata_type\x18\x64 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x08\x64\x61taType\x1au\n\x07\x44\x65\x63imal\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\x12!\n\tprecision\x18\x02 \x01(\x05H\x00R\tprecision\x88\x01\x01\x12\x19\n\x05scale\x18\x03 \x01(\x05H\x01R\x05scale\x88\x01\x01\x42\x0c\n\n_precisionB\x08\n\x06_scale\x1a\x62\n\x10\x43\x61lendarInterval\x12\x16\n\x06months\x18\x01 \x01(\x05R\x06months\x12\x12\n\x04\x64\x61ys\x18\x02 \x01(\x05R\x04\x64\x61ys\x12"\n\x0cmicroseconds\x18\x03 \x01(\x03R\x0cmicroseconds\x1a\x86\x01\n\x05\x41rray\x12>\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\x0b\x65lementType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lements\x1a\xeb\x01\n\x03Map\x12\x36\n\x08key_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\x07keyType\x12:\n\nvalue_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\tvalueType\x12\x35\n\x04keys\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x04keys\x12\x39\n\x06values\x18\x04 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1a\x85\x01\n\x06Struct\x12<\n\x0bstruct_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\nstructType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lements\x1a\xc0\x02\n\x10SpecializedArray\x12,\n\x05\x62ools\x18\x01 \x01(\x0b\x32\x14.spark.connect.BoolsH\x00R\x05\x62ools\x12)\n\x04ints\x18\x02 \x01(\x0b\x32\x13.spark.connect.IntsH\x00R\x04ints\x12,\n\x05longs\x18\x03 \x01(\x0b\x32\x14.spark.connect.LongsH\x00R\x05longs\x12/\n\x06\x66loats\x18\x04 \x01(\x0b\x32\x15.spark.connect.FloatsH\x00R\x06\x66loats\x12\x32\n\x07\x64oubles\x18\x05 \x01(\x0b\x32\x16.spark.connect.DoublesH\x00R\x07\x64oubles\x12\x32\n\x07strings\x18\x06 \x01(\x0b\x32\x16.spark.connect.StringsH\x00R\x07stringsB\x0c\n\nvalue_type\x1aK\n\x04Time\x12\x12\n\x04nano\x18\x01 \x01(\x03R\x04nano\x12!\n\tprecision\x18\x02 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precision\x1a\x95\x01\n\x11TimestampNTZNanos\x12!\n\x0c\x65poch_micros\x18\x01 \x01(\x03R\x0b\x65pochMicros\x12,\n\x12nanos_within_micro\x18\x02 \x01(\x05R\x10nanosWithinMicro\x12!\n\tprecision\x18\x03 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precision\x1a\x95\x01\n\x11TimestampLTZNanos\x12!\n\x0c\x65poch_micros\x18\x01 \x01(\x03R\x0b\x65pochMicros\x12,\n\x12nanos_within_micro\x18\x02 \x01(\x05R\x10nanosWithinMicro\x12!\n\tprecision\x18\x03 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precisionB\x0e\n\x0cliteral_typeJ\x04\x08\x1b\x10\x1cJ\x04\x08\x1c\x10\x1d\x1a\xba\x01\n\x13UnresolvedAttribute\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x12\x31\n\x12is_metadata_column\x18\x03 \x01(\x08H\x01R\x10isMetadataColumn\x88\x01\x01\x42\n\n\x08_plan_idB\x15\n\x13_is_metadata_column\x1a\x82\x02\n\x12UnresolvedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x1f\n\x0bis_distinct\x18\x03 \x01(\x08R\nisDistinct\x12\x37\n\x18is_user_defined_function\x18\x04 \x01(\x08R\x15isUserDefinedFunction\x12$\n\x0bis_internal\x18\x05 \x01(\x08H\x00R\nisInternal\x88\x01\x01\x42\x0e\n\x0c_is_internal\x1a\x32\n\x10\x45xpressionString\x12\x1e\n\nexpression\x18\x01 \x01(\tR\nexpression\x1a|\n\x0eUnresolvedStar\x12,\n\x0funparsed_target\x18\x01 \x01(\tH\x00R\x0eunparsedTarget\x88\x01\x01\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x01R\x06planId\x88\x01\x01\x42\x12\n\x10_unparsed_targetB\n\n\x08_plan_id\x1aV\n\x0fUnresolvedRegex\x12\x19\n\x08\x63ol_name\x18\x01 \x01(\tR\x07\x63olName\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x42\n\n\x08_plan_id\x1a\x84\x01\n\x16UnresolvedExtractValue\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12\x39\n\nextraction\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nextraction\x1a\xbb\x01\n\x0cUpdateFields\x12\x46\n\x11struct_expression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x10structExpression\x12\x1d\n\nfield_name\x18\x02 \x01(\tR\tfieldName\x12\x44\n\x10value_expression\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0fvalueExpression\x1ax\n\x05\x41lias\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12\x12\n\x04name\x18\x02 \x03(\tR\x04name\x12\x1f\n\x08metadata\x18\x03 \x01(\tH\x00R\x08metadata\x88\x01\x01\x42\x0b\n\t_metadata\x1a\x9e\x01\n\x0eLambdaFunction\x12\x35\n\x08\x66unction\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x08\x66unction\x12U\n\targuments\x18\x02 \x03(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableR\targuments\x1a>\n\x1dUnresolvedNamedLambdaVariable\x12\x1d\n\nname_parts\x18\x01 \x03(\tR\tnamePartsB\x0b\n\texpr_type"A\n\x10\x45xpressionCommon\x12-\n\x06origin\x18\x01 \x01(\x0b\x32\x15.spark.connect.OriginR\x06origin"\x8d\x03\n\x1f\x43ommonInlineUserDefinedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12$\n\rdeterministic\x18\x02 \x01(\x08R\rdeterministic\x12\x37\n\targuments\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x39\n\npython_udf\x18\x04 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\tpythonUdf\x12I\n\x10scalar_scala_udf\x18\x05 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\x0escalarScalaUdf\x12\x33\n\x08java_udf\x18\x06 \x01(\x0b\x32\x16.spark.connect.JavaUDFH\x00R\x07javaUdf\x12\x1f\n\x0bis_distinct\x18\x07 \x01(\x08R\nisDistinctB\n\n\x08\x66unction"\xf1\x01\n\tPythonUDF\x12\x38\n\x0boutput_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1b\n\teval_type\x18\x02 \x01(\x05R\x08\x65valType\x12\x18\n\x07\x63ommand\x18\x03 \x01(\x0cR\x07\x63ommand\x12\x1d\n\npython_ver\x18\x04 \x01(\tR\tpythonVer\x12/\n\x13\x61\x64\x64itional_includes\x18\x05 \x03(\tR\x12\x61\x64\x64itionalIncludes\x12#\n\rbroadcast_ids\x18\x06 \x03(\x03R\x0c\x62roadcastIds"\xd6\x01\n\x0eScalarScalaUDF\x12\x18\n\x07payload\x18\x01 \x01(\x0cR\x07payload\x12\x37\n\ninputTypes\x18\x02 \x03(\x0b\x32\x17.spark.connect.DataTypeR\ninputTypes\x12\x37\n\noutputType\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1a\n\x08nullable\x18\x04 \x01(\x08R\x08nullable\x12\x1c\n\taggregate\x18\x05 \x01(\x08R\taggregate"\x95\x01\n\x07JavaUDF\x12\x1d\n\nclass_name\x18\x01 \x01(\tR\tclassName\x12=\n\x0boutput_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\noutputType\x88\x01\x01\x12\x1c\n\taggregate\x18\x03 \x01(\x08R\taggregateB\x0e\n\x0c_output_type"c\n\x18TypedAggregateExpression\x12G\n\x10scalar_scala_udf\x18\x01 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFR\x0escalarScalaUdf"l\n\x0c\x43\x61llFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"\\\n\x17NamedArgumentExpression\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\x80\x04\n\x0bMergeAction\x12\x46\n\x0b\x61\x63tion_type\x18\x01 \x01(\x0e\x32%.spark.connect.MergeAction.ActionTypeR\nactionType\x12<\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\tcondition\x88\x01\x01\x12G\n\x0b\x61ssignments\x18\x03 \x03(\x0b\x32%.spark.connect.MergeAction.AssignmentR\x0b\x61ssignments\x1aj\n\nAssignment\x12+\n\x03key\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\xa7\x01\n\nActionType\x12\x17\n\x13\x41\x43TION_TYPE_INVALID\x10\x00\x12\x16\n\x12\x41\x43TION_TYPE_DELETE\x10\x01\x12\x16\n\x12\x41\x43TION_TYPE_INSERT\x10\x02\x12\x1b\n\x17\x41\x43TION_TYPE_INSERT_STAR\x10\x03\x12\x16\n\x12\x41\x43TION_TYPE_UPDATE\x10\x04\x12\x1b\n\x17\x41\x43TION_TYPE_UPDATE_STAR\x10\x05\x42\x0c\n\n_condition"\xc5\x05\n\x12SubqueryExpression\x12\x17\n\x07plan_id\x18\x01 \x01(\x03R\x06planId\x12S\n\rsubquery_type\x18\x02 \x01(\x0e\x32..spark.connect.SubqueryExpression.SubqueryTypeR\x0csubqueryType\x12\x62\n\x11table_arg_options\x18\x03 \x01(\x0b\x32\x31.spark.connect.SubqueryExpression.TableArgOptionsH\x00R\x0ftableArgOptions\x88\x01\x01\x12G\n\x12in_subquery_values\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x10inSubqueryValues\x1a\xea\x01\n\x0fTableArgOptions\x12@\n\x0epartition_spec\x18\x01 \x03(\x0b\x32\x19.spark.connect.ExpressionR\rpartitionSpec\x12\x42\n\norder_spec\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\torderSpec\x12\x37\n\x15with_single_partition\x18\x03 \x01(\x08H\x00R\x13withSinglePartition\x88\x01\x01\x42\x18\n\x16_with_single_partition"\x90\x01\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x12\x1b\n\x17SUBQUERY_TYPE_TABLE_ARG\x10\x03\x12\x14\n\x10SUBQUERY_TYPE_IN\x10\x04\x42\x14\n\x12_table_arg_optionsB6\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -135,27 +135,27 @@ _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_start = 7899 _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_end = 8296 _globals["_PYTHONUDF"]._serialized_start = 8299 - _globals["_PYTHONUDF"]._serialized_end = 8503 - _globals["_SCALARSCALAUDF"]._serialized_start = 8506 - _globals["_SCALARSCALAUDF"]._serialized_end = 8720 - _globals["_JAVAUDF"]._serialized_start = 8723 - _globals["_JAVAUDF"]._serialized_end = 8872 - _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_start = 8874 - _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_end = 8973 - _globals["_CALLFUNCTION"]._serialized_start = 8975 - _globals["_CALLFUNCTION"]._serialized_end = 9083 - _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_start = 9085 - _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_end = 9177 - _globals["_MERGEACTION"]._serialized_start = 9180 - _globals["_MERGEACTION"]._serialized_end = 9692 - _globals["_MERGEACTION_ASSIGNMENT"]._serialized_start = 9402 - _globals["_MERGEACTION_ASSIGNMENT"]._serialized_end = 9508 - _globals["_MERGEACTION_ACTIONTYPE"]._serialized_start = 9511 - _globals["_MERGEACTION_ACTIONTYPE"]._serialized_end = 9678 - _globals["_SUBQUERYEXPRESSION"]._serialized_start = 9695 - _globals["_SUBQUERYEXPRESSION"]._serialized_end = 10404 - _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_start = 10001 - _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_end = 10235 - _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_start = 10238 - _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_end = 10382 + _globals["_PYTHONUDF"]._serialized_end = 8540 + _globals["_SCALARSCALAUDF"]._serialized_start = 8543 + _globals["_SCALARSCALAUDF"]._serialized_end = 8757 + _globals["_JAVAUDF"]._serialized_start = 8760 + _globals["_JAVAUDF"]._serialized_end = 8909 + _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_start = 8911 + _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_end = 9010 + _globals["_CALLFUNCTION"]._serialized_start = 9012 + _globals["_CALLFUNCTION"]._serialized_end = 9120 + _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_start = 9122 + _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_end = 9214 + _globals["_MERGEACTION"]._serialized_start = 9217 + _globals["_MERGEACTION"]._serialized_end = 9729 + _globals["_MERGEACTION_ASSIGNMENT"]._serialized_start = 9439 + _globals["_MERGEACTION_ASSIGNMENT"]._serialized_end = 9545 + _globals["_MERGEACTION_ACTIONTYPE"]._serialized_start = 9548 + _globals["_MERGEACTION_ACTIONTYPE"]._serialized_end = 9715 + _globals["_SUBQUERYEXPRESSION"]._serialized_start = 9732 + _globals["_SUBQUERYEXPRESSION"]._serialized_end = 10441 + _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_start = 10038 + _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_end = 10272 + _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_start = 10275 + _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_end = 10419 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.pyi b/python/pyspark/sql/connect/proto/expressions_pb2.pyi index c613ade2f43f0..1dad6ff10a814 100644 --- a/python/pyspark/sql/connect/proto/expressions_pb2.pyi +++ b/python/pyspark/sql/connect/proto/expressions_pb2.pyi @@ -1826,6 +1826,7 @@ class PythonUDF(google.protobuf.message.Message): COMMAND_FIELD_NUMBER: builtins.int PYTHON_VER_FIELD_NUMBER: builtins.int ADDITIONAL_INCLUDES_FIELD_NUMBER: builtins.int + BROADCAST_IDS_FIELD_NUMBER: builtins.int @property def output_type(self) -> pyspark.sql.connect.proto.types_pb2.DataType: """(Required) Output type of the Python UDF""" @@ -1840,6 +1841,15 @@ class PythonUDF(google.protobuf.message.Message): self, ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """(Optional) Additional includes for the Python UDF.""" + @property + def broadcast_ids( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: + """(Optional) (SPARK-51705) Server-assigned ids of broadcast variables referenced by this UDF. + Resolved by SparkConnectPlanner against the per-session SessionHolder broadcasts registry + into SimplePythonFunction.broadcastVars. The ids are the driver-side Broadcast ids returned + by CreateBroadcastResult (which the executor/worker also key on), not opaque handles. + """ def __init__( self, *, @@ -1848,6 +1858,7 @@ class PythonUDF(google.protobuf.message.Message): command: builtins.bytes = ..., python_ver: builtins.str = ..., additional_includes: collections.abc.Iterable[builtins.str] | None = ..., + broadcast_ids: collections.abc.Iterable[builtins.int] | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["output_type", b"output_type"] @@ -1857,6 +1868,8 @@ class PythonUDF(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "additional_includes", b"additional_includes", + "broadcast_ids", + b"broadcast_ids", "command", b"command", "eval_type", diff --git a/python/pyspark/sql/connect/session.py b/python/pyspark/sql/connect/session.py index 99da7308a1a00..af698304ad106 100644 --- a/python/pyspark/sql/connect/session.py +++ b/python/pyspark/sql/connect/session.py @@ -37,6 +37,7 @@ overload, Iterable, Mapping, + TypeVar, TYPE_CHECKING, ClassVar, ) @@ -103,6 +104,7 @@ if TYPE_CHECKING: import pyspark.sql.connect.proto as pb2 + from pyspark.core.broadcast import Broadcast from pyspark.sql.connect._typing import OptionalPrimitiveType from pyspark.sql.connect.catalog import Catalog from pyspark.sql.connect.udf import UDFRegistration @@ -112,6 +114,9 @@ from pyspark.sql.connect.datasource import DataSourceRegistration +T = TypeVar("T") + + class SparkSession: # The active SparkSession for the current thread _active_session: ClassVar[threading.local] = threading.local() @@ -866,6 +871,20 @@ def range( range.__doc__ = PySparkSession.range.__doc__ + def broadcast(self, value: "T") -> "Broadcast[T]": + # (SPARK-51705) Create a broadcast variable over Spark Connect. The value is cloudpickled + # and uploaded through the existing cache artifact channel; the server materializes a + # PythonBroadcast on the driver SparkContext and returns a driver-side broadcast id. + from pyspark.serializers import CloudPickleSerializer + from pyspark.sql.connect.broadcast import ConnectBroadcast + + blob = CloudPickleSerializer().dumps(value) + artifact_hash = self._client.cache_artifact(blob) + broadcast_id = self._client._create_broadcast(artifact_hash, len(blob)) + return ConnectBroadcast(self, broadcast_id, value) + + broadcast.__doc__ = PySparkSession.broadcast.__doc__ + @functools.cached_property def catalog(self) -> "Catalog": from pyspark.sql.connect.catalog import Catalog diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index c36260b9d13ea..cc2b877cb7518 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -75,6 +75,7 @@ from py4j.java_gateway import JavaClass, JavaObject, JVMView import pyarrow as pa from pyspark.core.context import SparkContext + from pyspark.core.broadcast import Broadcast from pyspark.core.rdd import RDD from pyspark.sql._typing import AtomicValue, RowLike, OptionalPrimitiveType from pyspark.sql.catalog import Catalog @@ -1052,6 +1053,40 @@ def range( return DataFrame(jdf, self) + def broadcast(self, value: "T") -> "Broadcast[T]": + """ + Broadcast a read-only variable to the cluster, returning a + :class:`Broadcast ` object for reading it in distributed + functions. The variable will be sent to each executor only once. + + This is an alias for :meth:`SparkContext.broadcast`, provided so that broadcast variables + can be created from a :class:`SparkSession` in both classic PySpark and Spark Connect + (SPARK-51705). + + .. versionadded:: 4.3.0 + + .. versionchanged:: 4.3.0 + Supports Spark Connect. + + Parameters + ---------- + value : T + value to broadcast to the Spark nodes + + Returns + ------- + :class:`Broadcast ` + :class:`Broadcast ` object, a read-only variable cached + on each machine + + Examples + -------- + >>> bc = spark.broadcast({1: 10001, 2: 10002}) + >>> bc.value + {1: 10001, 2: 10002} + """ + return self._sc.broadcast(value) + def _inferSchemaFromList( self, data: Iterable[Any], names: Optional[List[str]] = None ) -> StructType: diff --git a/python/pyspark/sql/tests/connect/test_connect_broadcast.py b/python/pyspark/sql/tests/connect/test_connect_broadcast.py new file mode 100644 index 0000000000000..960fc9b8afa55 --- /dev/null +++ b/python/pyspark/sql/tests/connect/test_connect_broadcast.py @@ -0,0 +1,106 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +(SPARK-51705) End-to-end tests for ``SparkSession.broadcast()`` over Spark Connect. + +These tests require a running Spark Connect server (they build a real session via +:class:`ReusedConnectTestCase`), so they are gated on ``should_test_connect``. +""" + +import unittest + +from pyspark.errors import PySparkException +from pyspark.testing.connectutils import ( + should_test_connect, + ReusedConnectTestCase, + connect_requirement_message, +) + +if should_test_connect: + from pyspark.sql.connect.broadcast import ConnectBroadcast + from pyspark.sql.functions import udf + from pyspark.sql.types import StringType + + +@unittest.skipIf(not should_test_connect, connect_requirement_message) +class SparkConnectBroadcastTests(ReusedConnectTestCase): + def test_broadcast_returns_connect_broadcast(self): + # spark.broadcast(value) returns a ConnectBroadcast that keeps the value locally for + # driver-side reads. + value = {"a": 1, "b": 2} + bcast = self.spark.broadcast(value) + self.assertIsInstance(bcast, ConnectBroadcast) + self.assertEqual(bcast.value, value) + + def test_broadcast_in_udf(self): + # The exact case that fails today with BROADCAST_VARIABLE_NOT_LOADED on Serverless: + # a dict broadcast referenced from a Python UDF over a DataFrame. + mapping = {i: i * 10 for i in range(1000)} + bcast = self.spark.broadcast(mapping) + + @udf(returnType=StringType()) + def lookup(key): + return str(bcast.value.get(key, -1)) + + df = self.spark.range(0, 100) + rows = df.select(lookup(df.id).alias("v")).collect() + expected = [str(mapping.get(i, -1)) for i in range(100)] + self.assertEqual([r.v for r in rows], expected) + + def test_broadcast_portability(self): + # A UDF referencing a broadcast must produce byte-identical results regardless of how the + # broadcast handle was created; here we just assert repeatable correctness across a large + # broadcast value. + mapping = {i: f"v{i}" for i in range(10_000)} + bcast = self.spark.broadcast(mapping) + + @udf(returnType=StringType()) + def lookup(key): + return bcast.value.get(key, "") + + df = self.spark.range(0, 5000) + rows = df.select(lookup(df.id).alias("v")).collect() + self.assertEqual([r.v for r in rows], [mapping[i] for i in range(5000)]) + + def test_unpersist_and_destroy(self): + # unpersist()/destroy() route to the server; they must not raise. After destroy() the + # broadcast is removed from the session registry (verified server-side / via close sweep). + bcast = self.spark.broadcast(list(range(100))) + bcast.unpersist() + bcast.unpersist(blocking=True) + bcast.destroy() + + def test_broadcast_not_found(self): + # Referencing a broadcast id that was destroyed (or never created on this session) must + # fail loudly with BROADCAST_NOT_FOUND rather than silently emitting an empty broadcast + # list (which would surface as BROADCAST_VARIABLE_NOT_LOADED on the worker). + bcast = self.spark.broadcast({"x": 1}) + bcast.destroy() + + @udf(returnType=StringType()) + def lookup(key): + return str(bcast.value.get("x", -1)) + + df = self.spark.range(0, 10) + with self.assertRaises(PySparkException): + df.select(lookup(df.id).alias("v")).collect() + + +if __name__ == "__main__": + from pyspark.testing import main + + main() diff --git a/python/pyspark/sql/tests/connect/test_parity_udf.py b/python/pyspark/sql/tests/connect/test_parity_udf.py index 33ec11adcf747..87c6da2ea48cf 100644 --- a/python/pyspark/sql/tests/connect/test_parity_udf.py +++ b/python/pyspark/sql/tests/connect/test_parity_udf.py @@ -44,7 +44,11 @@ def test_udf_with_input_file_name_for_hadooprdd(self): def test_same_accumulator_in_udfs(self): super().test_same_accumulator_in_udfs() - @unittest.skip("Spark Connect does not support broadcast but the test depends on it.") + @unittest.skip( + "The test uses SparkContext.broadcast, which Spark Connect does not expose. " + "SparkSession.broadcast over Connect (SPARK-51705) is covered by " + "pyspark.sql.tests.connect.test_connect_broadcast." + ) def test_broadcast_in_udf(self): super().test_broadcast_in_udf() diff --git a/sql/connect/common/src/main/protobuf/spark/connect/base.proto b/sql/connect/common/src/main/protobuf/spark/connect/base.proto index c7247129f1907..14dd889cb944d 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/base.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/base.proto @@ -429,6 +429,9 @@ message ExecutePlanResponse { // register its result with the server. PipelineQueryFunctionExecutionSignal pipeline_query_function_execution_signal = 23; + // (SPARK-51705) Server-assigned handle for a CreateBroadcastCommand. + CreateBroadcastResult create_broadcast_result = 24; + // Support arbitrary result objects. google.protobuf.Any extension = 999; } @@ -1176,6 +1179,14 @@ message CheckpointCommandResult { CachedRemoteRelation relation = 1; } +// (SPARK-51705) Server-assigned broadcast handle (mirrors CheckpointCommandResult). +message CreateBroadcastResult { + // (Required) The driver-side broadcast id. The client embeds this id in the pickled + // Broadcast closure (Broadcast._from_id), which the executor/worker resolves against its + // _broadcastRegistry keyed by the same id. + int64 broadcast_id = 1; +} + message CloneSessionRequest { // (Required) // diff --git a/sql/connect/common/src/main/protobuf/spark/connect/commands.proto b/sql/connect/common/src/main/protobuf/spark/connect/commands.proto index dcf5aff2366f7..394673d50b2ea 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/commands.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/commands.proto @@ -53,6 +53,8 @@ message Command { MlCommand ml_command = 17; ExecuteExternalCommand execute_external_command = 18; PipelineCommand pipeline_command = 19; + CreateBroadcastCommand create_broadcast_command = 20; + UnpersistBroadcastCommand unpersist_broadcast_command = 21; // This field is used to mark extensions to the protocol. When plugins generate arbitrary // Commands they can add them here. During the planning the correct resolution is done. @@ -61,6 +63,27 @@ message Command { } } +// (SPARK-51705) Create a broadcast variable from an already-uploaded cache/ artifact. +// The client uploads cloudpickle(value) through the existing cache artifact channel and then +// sends this command with the returned hash. The server materializes a PythonBroadcast on the +// live driver SparkContext and returns a CreateBroadcastResult with the driver-side broadcast id. +message CreateBroadcastCommand { + // (Required) sha256 hash returned by client.cache_artifact(cloudpickle(value)). + string artifact_hash = 1; + // (Optional) Uncompressed byte size, used to enforce the BROADCAST_VALUE_TOO_LARGE quota. + int64 size_bytes = 2; +} + +// (SPARK-51705) Release a broadcast variable created over Spark Connect. +message UnpersistBroadcastCommand { + // (Required) The driver-side broadcast id returned by CreateBroadcastResult. + int64 broadcast_id = 1; + // (Optional) Whether to block until unpersisting has completed. + bool blocking = 2; + // (Optional) Whether to destroy (not just unpersist) all data and metadata of the broadcast. + bool destroy = 3; +} + // A SQL Command is used to trigger the eager evaluation of SQL commands in Spark. // // When the SQL provide as part of the message is a command it will be immediately evaluated diff --git a/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto b/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto index 18f8f294f0c02..8631d6e6def87 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto @@ -475,6 +475,11 @@ message PythonUDF { string python_ver = 4; // (Optional) Additional includes for the Python UDF. repeated string additional_includes = 5; + // (Optional) (SPARK-51705) Server-assigned ids of broadcast variables referenced by this UDF. + // Resolved by SparkConnectPlanner against the per-session SessionHolder broadcasts registry + // into SimplePythonFunction.broadcastVars. The ids are the driver-side Broadcast ids returned + // by CreateBroadcastResult (which the executor/worker also key on), not opaque handles. + repeated int64 broadcast_ids = 6; } message ScalarScalaUDF { diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/InvalidInputErrors.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/InvalidInputErrors.scala index 604ad38606970..6e3dec3f8df1d 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/InvalidInputErrors.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/InvalidInputErrors.scala @@ -256,4 +256,8 @@ object InvalidInputErrors { InvalidPlanInput( "CONNECT_INVALID_PLAN.CANNOT_FIND_CACHED_LOCAL_RELATION", Map("hash" -> hash)) + + // (SPARK-51705) A Python UDF referenced a broadcast id that is not registered on this session. + def broadcastNotFound(id: Long): InvalidPlanInput = + InvalidPlanInput("CONNECT_INVALID_PLAN.BROADCAST_NOT_FOUND", Map("id" -> id.toString)) } diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala index 841ce26402cf0..1a9d6c5844a88 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.connect.planner +import java.io.{File, FileOutputStream} +import java.nio.file.Files import java.util.{HashMap, Properties, UUID} import scala.collection.mutable @@ -31,7 +33,8 @@ import io.grpc.stub.StreamObserver import org.apache.spark.{SparkClassNotFoundException, SparkEnv, SparkException} import org.apache.spark.annotation.{DeveloperApi, Since} -import org.apache.spark.api.python.{PythonEvalType, SimplePythonFunction} +import org.apache.spark.api.python.{PythonBroadcast, PythonEvalType, SimplePythonFunction} +import org.apache.spark.broadcast.Broadcast import org.apache.spark.connect.proto import org.apache.spark.connect.proto.{CheckpointCommand, CreateResourceProfileCommand, ExecutePlanResponse, SqlCommand, StreamingForeachFunction, StreamingQueryCommand, StreamingQueryCommandResult, StreamingQueryInstanceId, StreamingQueryManagerCommand, StreamingQueryManagerCommandResult, WriteStreamOperationStart, WriteStreamOperationStartResult} import org.apache.spark.connect.proto.ExecutePlanResponse.SqlCommandResult @@ -2227,12 +2230,33 @@ class SparkConnectPlanner( pythonIncludes = (fun.getAdditionalIncludesList.asScala.toSeq ++ sessionHolder.artifactManager.getPythonIncludes).asJava, pythonVer = fun.getPythonVer, - // Empty broadcast variables - broadcastVars = Lists.newArrayList(), + // (SPARK-51705) Resolve broadcast variables referenced by this UDF from the per-session + // registry. Empty list when the UDF references no broadcasts. + broadcastVars = resolveBroadcasts(fun.getBroadcastIdsList), // Accumulator if available accumulator = sessionHolder.pythonAccumulator.orNull) } + /** + * (SPARK-51705) Resolve the broadcast ids referenced by a Python UDF into the driver-side + * Broadcast[PythonBroadcast] handles from the per-session SessionHolder registry. An id that is + * unknown (never created on this session, or belonging to another session) fails loudly with + * BROADCAST_NOT_FOUND rather than silently emitting an empty list -- which would otherwise + * surface as BROADCAST_VARIABLE_NOT_LOADED on the Python worker. + */ + private def resolveBroadcasts(broadcastIds: java.util.List[java.lang.Long]) + : java.util.List[Broadcast[PythonBroadcast]] = { + val resolved = new java.util.ArrayList[Broadcast[PythonBroadcast]](broadcastIds.size()) + broadcastIds.forEach { boxedId => + val id = boxedId.longValue() + val bcast = sessionHolder + .getBroadcast(id) + .getOrElse(throw InvalidInputErrors.broadcastNotFound(id)) + resolved.add(bcast) + } + resolved + } + /** * Translates a LambdaFunction from proto to the Catalyst expression. */ @@ -2932,6 +2956,10 @@ class SparkConnectPlanner( handlePipelineCommand(command.getPipelineCommand, responseObserver) case proto.Command.CommandTypeCase.EXECUTE_EXTERNAL_COMMAND => handleExecuteExternalCommand(command.getExecuteExternalCommand, responseObserver) + case proto.Command.CommandTypeCase.CREATE_BROADCAST_COMMAND => + handleCreateBroadcastCommand(command.getCreateBroadcastCommand, responseObserver) + case proto.Command.CommandTypeCase.UNPERSIST_BROADCAST_COMMAND => + handleUnpersistBroadcastCommand(command.getUnpersistBroadcastCommand) case other => throw InvalidInputErrors.invalidOneOfField(other, command.getDescriptorForType) @@ -4012,6 +4040,81 @@ class SparkConnectPlanner( executeHolder.eventsManager.postFinished() } + /** + * (SPARK-51705) Materialize a broadcast variable from an already-uploaded cache/ + * artifact and register it on this session. + * + * The client uploaded cloudpickle(value) through the existing artifact cache channel; here we + * read the block back (decrypting transparently under spark.io.encryption), stage it to a temp + * file under the local dir (PythonBroadcast needs a path, and the cache block may be memory + * only), wrap it in a PythonBroadcast, and broadcast it on the live driver SparkContext -- the + * same object pythonAccumulator uses. The returned broadcast id is the driver-side + * Broadcast.id, which the client embeds into the pickled UDF closure and which the worker keys + * on. + */ + private def handleCreateBroadcastCommand( + command: proto.CreateBroadcastCommand, + responseObserver: StreamObserver[proto.ExecutePlanResponse]): Unit = { + val hash = command.getArtifactHash + val blockManager = session.sparkContext.env.blockManager + val blockId = sessionHolder.artifactManager.getCachedBlockId(hash).getOrElse { + throw InvalidInputErrors.cannotFindCachedLocalRelation(hash) + } + // Stage the (possibly memory-only, possibly encrypted) cache block to a temp file that + // PythonBroadcast can read. Mirrors the block-read idiom in transformCachedLocalRelation. + val bytes = blockManager + .getLocalBytes(blockId) + .getOrElse { + throw InvalidInputErrors.notFoundCachedLocalRelation(blockId.hash, blockId.sessionUUID) + } + val dir = new File(Utils.getLocalDir(session.sparkContext.conf)) + val file = Files.createTempFile(dir.toPath, "broadcast", "").toFile + try { + val in = bytes.toInputStream() + val out = new FileOutputStream(file) + Utils.tryWithSafeFinally { + Utils.copyStream(in, out) + } { + out.close() + in.close() + } + } finally { + blockManager.releaseLock(blockId) + } + + val pythonBroadcast = new PythonBroadcast(file.getAbsolutePath) + val bcast = session.sparkContext.broadcast(pythonBroadcast) + val broadcastId = sessionHolder.registerBroadcast(bcast) + logInfo(log"Created broadcast with id ${MDC(LogKeys.BROADCAST_ID, broadcastId)}") + + executeHolder.eventsManager.postFinished() + responseObserver.onNext( + proto.ExecutePlanResponse + .newBuilder() + .setSessionId(sessionId) + .setServerSideSessionId(sessionHolder.serverSessionId) + .setCreateBroadcastResult( + proto.CreateBroadcastResult + .newBuilder() + .setBroadcastId(broadcastId) + .build()) + .build()) + } + + /** + * (SPARK-51705) Release a broadcast variable created over Spark Connect. Unknown ids are a + * no-op (idempotent unpersist), consistent with the client possibly retrying. + */ + private def handleUnpersistBroadcastCommand(command: proto.UnpersistBroadcastCommand): Unit = { + val broadcastId = command.getBroadcastId + if (command.getDestroy) { + sessionHolder.removeBroadcast(broadcastId).foreach(_.destroy(command.getBlocking)) + } else { + sessionHolder.getBroadcast(broadcastId).foreach(_.unpersist(command.getBlocking)) + } + executeHolder.eventsManager.postFinished() + } + private def transformMergeIntoTableCommand(cmd: proto.MergeIntoTableCommand)( tracker: QueryPlanningTracker): LogicalPlan = { def transformActions(actions: java.util.List[proto.Expression]): Seq[MergeAction] = diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala index 2276230545e67..070a511fdb670 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala @@ -24,12 +24,15 @@ import scala.collection.mutable import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.util.Try +import scala.util.control.NonFatal import com.google.common.base.Ticker import com.google.common.cache.{Cache, CacheBuilder} import org.apache.spark.{SparkEnv, SparkException, SparkSQLException} +import org.apache.spark.api.python.PythonBroadcast import org.apache.spark.api.python.PythonFunction.PythonAccumulator +import org.apache.spark.broadcast.Broadcast import org.apache.spark.connect.proto import org.apache.spark.internal.{Logging, LogKeys} import org.apache.spark.sql.DataFrame @@ -425,6 +428,21 @@ case class SessionHolder(userId: String, sessionId: String, session: SparkSessio // Clean up ML cache (only if ML models were created) mlCache.close() + // (SPARK-51705) Destroy all broadcast variables created over Spark Connect, freeing the + // driver + executor blocks. Best-effort: a failure to destroy one must not abort the sweep. + broadcasts.forEach { (id, bcast) => + try { + bcast.destroy() + } catch { + case NonFatal(e) => + logWarning( + log"Failed to destroy broadcast ${MDC(LogKeys.BROADCAST_ID, id)} " + + log"while closing session ${MDC(LogKeys.SESSION_ID, sessionId)}", + e) + } + } + broadcasts.clear() + session.cleanupPythonWorkerLogs() eventManager.postClosed() @@ -586,6 +604,29 @@ case class SessionHolder(userId: String, sessionId: String, session: SparkSessio private[connect] val pythonAccumulator: Option[PythonAccumulator] = Try(session.sparkContext.collectionAccumulator[Array[Byte]]).toOption + /** + * (SPARK-51705) Registry of broadcast variables created over Spark Connect for this session, + * keyed by the driver-side broadcast id (Broadcast.id). This id is what the client embeds in + * the pickled UDF closure (via Broadcast._from_id) and what the executor/worker keys on, so it + * is used both as the client handle (CreateBroadcastResult.broadcast_id / + * PythonUDF.broadcast_ids) and as the registry key. Membership in this per-session map enforces + * isolation: SparkConnectPlanner.resolveBroadcasts rejects ids that are absent + * (BROADCAST_NOT_FOUND). Sibling to `pythonAccumulator`. + */ + private[connect] val broadcasts: ConcurrentMap[Long, Broadcast[PythonBroadcast]] = + new ConcurrentHashMap() + + private[connect] def registerBroadcast(bcast: Broadcast[PythonBroadcast]): Long = { + broadcasts.put(bcast.id, bcast) + bcast.id + } + + private[connect] def getBroadcast(id: Long): Option[Broadcast[PythonBroadcast]] = + Option(broadcasts.get(id)) + + private[connect] def removeBroadcast(id: Long): Option[Broadcast[PythonBroadcast]] = + Option(broadcasts.remove(id)) + /** * Transform a relation into a logical plan, using the plan cache if enabled. The plan cache is * enable only if `spark.connect.session.planCache.maxSize` is greater than zero AND diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectSessionHolderSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectSessionHolderSuite.scala index cff5f345d2573..e63bebe0a8752 100644 --- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectSessionHolderSuite.scala +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectSessionHolderSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.connect.service import java.nio.charset.StandardCharsets import java.nio.file.Files +import java.util.UUID import java.util.concurrent.{TimeoutException, TimeUnit} import scala.collection.mutable @@ -31,7 +32,7 @@ import com.google.common.collect.Lists import org.scalatest.time.SpanSugar._ import org.apache.spark.SparkEnv -import org.apache.spark.api.python.SimplePythonFunction +import org.apache.spark.api.python.{PythonBroadcast, SimplePythonFunction} import org.apache.spark.connect.proto import org.apache.spark.sql.IntegratedUDFTestUtils import org.apache.spark.sql.connect.{PythonTestDepsChecker, SparkConnectTestUtils} @@ -106,6 +107,48 @@ class SparkConnectSessionHolderSuite extends SharedSparkSession { } } + test("SPARK-51705: broadcast registry put, get and remove") { + val sessionHolder = SparkConnectTestUtils.createDummySessionHolder(spark) + + val pb1 = new PythonBroadcast(Files.createTempFile("broadcast", "").toString) + val pb2 = new PythonBroadcast(Files.createTempFile("broadcast", "").toString) + val bcast1 = spark.sparkContext.broadcast(pb1) + val bcast2 = spark.sparkContext.broadcast(pb2) + + // The registry is keyed by the driver-side broadcast id, which is the client handle. + val id1 = sessionHolder.registerBroadcast(bcast1) + val id2 = sessionHolder.registerBroadcast(bcast2) + assert(id1 == bcast1.id) + assert(id2 == bcast2.id) + assert(sessionHolder.getBroadcast(id1).contains(bcast1)) + assert(sessionHolder.getBroadcast(id2).contains(bcast2)) + + // Unknown id is None -- resolveBroadcasts turns this into BROADCAST_NOT_FOUND. + assert(sessionHolder.getBroadcast(Long.MaxValue).isEmpty) + + // Remove one, the other survives. + assert(sessionHolder.removeBroadcast(id1).contains(bcast1)) + assert(sessionHolder.getBroadcast(id1).isEmpty) + assert(sessionHolder.getBroadcast(id2).contains(bcast2)) + } + + test("SPARK-51705: close() sweeps registered broadcasts") { + // Use an unregistered holder marked Started (not createDummySessionHolder) so that close() + // runs the full cleanup -- for a still-Pending session in tests close() returns early -- and + // so the closed holder is not left behind in the global session manager store. + val sessionHolder = + SessionHolder(userId = "testUser", sessionId = UUID.randomUUID().toString, session = spark) + sessionHolder.eventManager.status_(SessionStatus.Started) + + val pb = new PythonBroadcast(Files.createTempFile("broadcast", "").toString) + val bcast = spark.sparkContext.broadcast(pb) + val id = sessionHolder.registerBroadcast(bcast) + assert(sessionHolder.getBroadcast(id).contains(bcast)) + + sessionHolder.close() + assert(sessionHolder.getBroadcast(id).isEmpty) + } + private def streamingForeachBatchFunction(pysparkPythonPath: String): Array[Byte] = { var binaryFunc: Array[Byte] = null withTempPath { path =>