diff --git a/common/config/README.md b/common/config/README.md new file mode 100644 index 0000000000000..67c74ac48be28 --- /dev/null +++ b/common/config/README.md @@ -0,0 +1,122 @@ +# Spark Configuration Module + +This module provides a proto-based configuration system for defining Spark configurations. Configurations are defined using Protocol Buffers text format (`.textproto` files) and loaded at runtime by `ConfigRegistry`. + +## Directory Structure + +``` +common/config/src/main/ +├── java/org/apache/spark/config/ +│ └── ConfigRegistry.java # Loads and queries configs +├── protobuf/org/apache/spark/config/ +│ └── config_schema.proto # Defines the config schema +└── resources/org/apache/spark/config/ + ├── cluster_configs/ # CLUSTER scope configs + │ └── sql.textproto + └── session_configs/ # SESSION scope configs + └── sql.textproto +``` + +## How to Define a New Config + +### Step 1: Choose the Right File + +- **CLUSTER scope** (`MUTABILITY_STATIC`): If the config applies to the entire Spark cluster and cannot be changed at runtime, add to a file in `cluster_configs/` directory. +- **SESSION scope** (`MUTABILITY_DYNAMIC`): If the config applies to a Spark session and can be changed at runtime (e.g., via `SET`), add to a file in `session_configs/` directory. + +Currently, SQL configs go in `sql.textproto`. New categories can have their own files. + +### Step 2: Add the Config Entry + +Add a new `configs` block to the appropriate `.textproto` file: + +```protobuf +configs { + key: "spark.sql.myFeature.enabled" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_SESSION + doc: "When true, enables my new feature." + version: "4.0.0" +} +``` + +**Notes**: +- See `config_schema.proto` for field details and requirements +- Configs within each file must be ordered alphabetically by key +- A config that sets `test_default` must also set `default_value` (a test-only override needs a + production default to override); this is validated at load time +- These constraints are validated by tests + +### Step 3: Register the Config File (if new) + +If you created a new `.textproto` file, add it to `ConfigRegistry.java`: + +```java +private static final String[] DEFAULT_CONFIG_FILES = { + "org/apache/spark/config/cluster_configs/sql.textproto", + "org/apache/spark/config/session_configs/sql.textproto", + "org/apache/spark/config/session_configs/your_new_file.textproto" // Add here +}; +``` + +## Multi-line Documentation + +For long documentation strings, use protobuf string concatenation: + +```protobuf +configs { + key: "spark.sql.myFeature.threshold" + value_type: VALUE_TYPE_INT + default_value: "100" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "This is a long documentation string that spans multiple lines. " + "Simply place multiple quoted strings adjacent to each other " + "and protobuf will concatenate them automatically." + version: "4.0.0" +} +``` + +## Using Configs in Spark + +### Reading Config Values + +For configs accessed in only one or a few places, use `getConfByKeyStrict` with the config key directly. This will fail at runtime if the key is not defined in a `.textproto` file: + +```scala +def myFunc(conf: SQLConf): Unit = { + if (conf.getConfByKeyStrict[Boolean]("spark.sql.myFeature.enabled")) { + doSomething() + } +} +``` + +For configs accessed in many places, create a `ConfigEntry` in `object SQLConf` (or its friends) to avoid hardcoding keys: + +```scala +object SQLConf { + val MY_FEATURE_ENABLED = buildConfFromConfigFile[Boolean]("spark.sql.myFeature.enabled") +} + +def myFunc(conf: SQLConf): Unit = { + if (conf.getConf(SQLConf.MY_FEATURE_ENABLED)) { + doSomething() + } +} +``` + +### Adding Config Value Validation + +For configs that need to validate their values, create a `ConfigEntry` and use `checkValue`: + +```scala +val MY_THRESHOLD = buildConfFromConfigFile[Int]("spark.sql.myFeature.threshold") + .checkValue(_ > 0, "Threshold must be positive") +``` + diff --git a/common/config/pom.xml b/common/config/pom.xml new file mode 100644 index 0000000000000..b5dba2fcf9d9e --- /dev/null +++ b/common/config/pom.xml @@ -0,0 +1,187 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.13 + 5.0.0-SNAPSHOT + ../../pom.xml + + + spark-config_2.13 + jar + Spark Project Config + https://spark.apache.org/ + + config + + + + + org.apache.spark + spark-tags_${scala.binary.version} + + + + com.google.protobuf + protobuf-java + compile + + + + + org.apache.spark + spark-tags_${scala.binary.version} + test-jar + test + + + + org.junit.jupiter + junit-jupiter + test + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + + org.apache.maven.plugins + maven-shade-plugin + + + false + false + + + com.google.protobuf:* + + + + + com.google.protobuf + ${spark.shade.packageName}.protobuf + + com.google.protobuf.** + + + + + + *:* + + google/protobuf/** + + + + + + + package + + shade + + + + + + + + + + default-protoc + + + !skipDefaultProtoc + + + + + + com.github.os72 + protoc-jar-maven-plugin + ${protoc-jar-maven-plugin.version} + + + generate-sources + + run + + + com.google.protobuf:protoc:${protobuf.version} + ${protobuf.version} + + src/main/protobuf + + + + + + + + + + user-defined-protoc + + ${env.SPARK_PROTOC_EXEC_PATH} + + + + + com.github.os72 + protoc-jar-maven-plugin + ${protoc-jar-maven-plugin.version} + + + generate-sources + + run + + + com.google.protobuf:protoc:${protobuf.version} + ${protobuf.version} + ${spark.protoc.executable.path} + + src/main/protobuf + + + + + + + + + + diff --git a/common/config/src/main/java/org/apache/spark/config/ConfigRegistry.java b/common/config/src/main/java/org/apache/spark/config/ConfigRegistry.java new file mode 100644 index 0000000000000..08080081762aa --- /dev/null +++ b/common/config/src/main/java/org/apache/spark/config/ConfigRegistry.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.config; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.*; + +import com.google.protobuf.TextFormat; + +import org.apache.spark.config.protobuf.ConfigEntries; +import org.apache.spark.config.protobuf.ConfigEntry; + +/** + * A registry for Spark configuration entries. + * + *

This class loads configuration entries from .textproto files on the classpath + * and provides methods to query them by key. + * + *

For production use, use the static methods which access the default singleton instance. + * For testing, create a new instance with custom config file paths using the constructor. + * + *

Config files are organized into two directories: + *

+ */ +public class ConfigRegistry { + + // Explicit list of config files to load for production (package-private for testing) + static final String[] DEFAULT_CONFIG_FILES = { + "org/apache/spark/config/cluster_configs/sql.textproto", + "org/apache/spark/config/session_configs/sql.textproto" + }; + + // Lazy initialization holder class idiom for thread-safe lazy loading of default instance + private static class DefaultInstanceHolder { + static final ConfigRegistry INSTANCE = new ConfigRegistry(DEFAULT_CONFIG_FILES); + } + + // Instance field: map of config key -> ConfigEntry + private final Map configMap; + + /** + * Create a ConfigRegistry with custom config file paths. + * + * @param configFiles the resource paths of .textproto files to load + */ + public ConfigRegistry(String... configFiles) { + this.configMap = loadConfigs(configFiles); + } + + /** + * Get a config entry by its key. + * + * @param key the config key (e.g., "spark.sql.optimizer.maxIterations") + * @return the ConfigEntry if found, null otherwise + */ + public ConfigEntry get(String key) { + return configMap.get(key); + } + + /** + * Check if a config key exists in the registry. + * + * @param key the config key + * @return true if the key exists, false otherwise + */ + public boolean contains(String key) { + return configMap.containsKey(key); + } + + /** + * Get all registered config keys. + * + * @return an unmodifiable set of all config keys + */ + public Set keys() { + return configMap.keySet(); + } + + /** + * Get all registered config entries. + * + * @return an unmodifiable collection of all config entries + */ + public Collection all() { + return configMap.values(); + } + + // ========================================================================== + // Static methods for production use (delegate to default singleton instance) + // ========================================================================== + + /** + * Get the default singleton instance. + * + * @return the default ConfigRegistry instance + */ + public static ConfigRegistry getInstance() { + return DefaultInstanceHolder.INSTANCE; + } + + /** + * Get a config entry by its key from the default registry. + * + * @param key the config key (e.g., "spark.sql.optimizer.maxIterations") + * @return the ConfigEntry if found, null otherwise + */ + public static ConfigEntry getConfig(String key) { + return getInstance().get(key); + } + + /** + * Check if a config key exists in the default registry. + * + * @param key the config key + * @return true if the key exists, false otherwise + */ + public static boolean containsConfig(String key) { + return getInstance().contains(key); + } + + /** + * Get all registered config keys from the default registry. + * + * @return an unmodifiable set of all config keys + */ + public static Set allKeys() { + return getInstance().keys(); + } + + /** + * Get all registered config entries from the default registry. + * + * @return an unmodifiable collection of all config entries + */ + public static Collection allConfigs() { + return getInstance().all(); + } + + // ========================================================================== + // Private helper methods + // ========================================================================== + + /** + * Load configs from .textproto files. + */ + private static Map loadConfigs(String[] configFiles) { + Map result = new HashMap<>(); + ClassLoader classLoader = ConfigRegistry.class.getClassLoader(); + + for (String file : configFiles) { + List configs = loadConfigFile(classLoader, file); + for (ConfigEntry config : configs) { + result.put(config.getKey(), config); + } + } + + return Collections.unmodifiableMap(result); + } + + /** + * Load configs from a single .textproto file. + * Package-private for testing. + */ + static List loadConfigFile( + ClassLoader classLoader, + String resourcePath) { + InputStream inputStream = classLoader.getResourceAsStream(resourcePath); + if (inputStream == null) { + throw internalError("Config file not found on classpath: " + resourcePath); + } + + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + ConfigEntries.Builder builder = ConfigEntries.newBuilder(); + TextFormat.getParser().merge(reader, builder); + return builder.build().getConfigsList(); + } catch (Exception e) { + throw internalError("Failed to load config file: " + resourcePath, e); + } + } + + private static AssertionError internalError(String message) { + return new AssertionError(message); + } + + private static AssertionError internalError(String message, Throwable cause) { + return new AssertionError(message, cause); + } +} diff --git a/common/config/src/main/protobuf/buf.yaml b/common/config/src/main/protobuf/buf.yaml new file mode 100644 index 0000000000000..3b3903fc97c9f --- /dev/null +++ b/common/config/src/main/protobuf/buf.yaml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: v1 +breaking: + use: + - FILE +lint: + use: + - BASIC + diff --git a/common/config/src/main/protobuf/org/apache/spark/config/config_schema.proto b/common/config/src/main/protobuf/org/apache/spark/config/config_schema.proto new file mode 100644 index 0000000000000..c46dbdbb97d4d --- /dev/null +++ b/common/config/src/main/protobuf/org/apache/spark/config/config_schema.proto @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +package org.apache.spark.config; + +option java_multiple_files = true; +option java_package = "org.apache.spark.config.protobuf"; + +// The scope of the config +enum Scope { + SCOPE_UNSPECIFIED = 0; + // The config is effective for the entire Spark cluster. + SCOPE_CLUSTER = 1; + // The config is effective for the current session. + SCOPE_SESSION = 2; +} + +// The type of the config value +enum ValueType { + VALUE_TYPE_UNSPECIFIED = 0; + VALUE_TYPE_BOOL = 1; + VALUE_TYPE_INT = 2; + VALUE_TYPE_LONG = 3; + VALUE_TYPE_DOUBLE = 4; + VALUE_TYPE_STRING = 5; +} + +// The visibility of the config +enum Visibility { + VISIBILITY_UNSPECIFIED = 0; + VISIBILITY_PUBLIC = 1; + VISIBILITY_INTERNAL = 2; +} + +// The binding policy of the config for SQL views, UDFs, and procedures. +enum BindingPolicy { + BINDING_POLICY_UNSPECIFIED = 0; + // The config value propagates from the active session to views/UDFs/procedures. + BINDING_POLICY_SESSION = 1; + // The config uses the value saved at view/UDF/procedure creation time, + // or the Spark default if none was saved. + BINDING_POLICY_PERSISTED = 2; + // The config does not apply to views/UDFs/procedures. At runtime, behaves like SESSION. + BINDING_POLICY_NOT_APPLICABLE = 3; +} + +// Whether the config can be changed after system initialization. +enum Mutability { + MUTABILITY_UNSPECIFIED = 0; + // The config is fixed once the Spark application starts. + MUTABILITY_STATIC = 1; + // The config can be changed at runtime (e.g., via SET). + MUTABILITY_DYNAMIC = 2; +} + +// A single config entry +message ConfigEntry { + // [Required] The config key. + optional string key = 1; + + // [Required] The type of the config value + optional ValueType value_type = 2; + + // The default value as a string representation + optional string default_value = 3; + + // The default value to use during testing, if different from default_value. + // Requires default_value to be set, since it only overrides the production default in tests. + optional string test_default = 4; + + // [Required] The scope of the config (CLUSTER or SESSION). + optional Scope scope = 5; + + // [Required] Whether the config can be changed after system initialization. + optional Mutability mutability = 6; + + // [Required] The visibility of the config (PUBLIC or INTERNAL). + optional Visibility visibility = 7; + + // [Required] The binding policy for SQL views, UDFs, and procedures. + optional BindingPolicy binding_policy = 8; + + // [Required] Documentation for the config. + optional string doc = 9; + + // The Spark version when this config was introduced. + optional string version = 10; +} + +// A collection of config entries +message ConfigEntries { + repeated ConfigEntry configs = 1; +} diff --git a/common/config/src/main/resources/org/apache/spark/config/cluster_configs/sql.textproto b/common/config/src/main/resources/org/apache/spark/config/cluster_configs/sql.textproto new file mode 100644 index 0000000000000..f599ed52d794b --- /dev/null +++ b/common/config/src/main/resources/org/apache/spark/config/cluster_configs/sql.textproto @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SQL configuration entries (CLUSTER scope) + +configs { + key: "spark.sql.ui.retainedExecutions" + value_type: VALUE_TYPE_INT + default_value: "1000" + scope: SCOPE_CLUSTER + mutability: MUTABILITY_STATIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "Number of executions to retain in the Spark UI." + version: "1.5.0" +} diff --git a/common/config/src/main/resources/org/apache/spark/config/session_configs/sql.textproto b/common/config/src/main/resources/org/apache/spark/config/session_configs/sql.textproto new file mode 100644 index 0000000000000..06eaf7ee02779 --- /dev/null +++ b/common/config/src/main/resources/org/apache/spark/config/session_configs/sql.textproto @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SQL configuration entries (SESSION scope) + +configs { + key: "spark.sql.optimizer.datasourceV2ExprFolding" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_INTERNAL + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "When this config is set to true, do safe constant folding for the " + "expressions before translation and pushdown." + version: "4.1.0" +} + +configs { + key: "spark.sql.optimizer.maxIterations" + value_type: VALUE_TYPE_INT + default_value: "100" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_INTERNAL + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "The max number of iterations the optimizer runs." + version: "2.0.0" +} + +configs { + key: "spark.sql.shuffledHashJoinFactor" + value_type: VALUE_TYPE_INT + default_value: "3" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "The shuffle hash join can be selected if the data size of small " + "side multiplied by this factor is still smaller than the large side." + version: "3.3.0" +} diff --git a/common/config/src/test/java/org/apache/spark/config/ConfigRegistrySuite.java b/common/config/src/test/java/org/apache/spark/config/ConfigRegistrySuite.java new file mode 100644 index 0000000000000..9ec05c1a83287 --- /dev/null +++ b/common/config/src/test/java/org/apache/spark/config/ConfigRegistrySuite.java @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.config; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import org.apache.spark.config.protobuf.BindingPolicy; +import org.apache.spark.config.protobuf.ConfigEntry; +import org.apache.spark.config.protobuf.Mutability; +import org.apache.spark.config.protobuf.Scope; +import org.apache.spark.config.protobuf.ValueType; +import org.apache.spark.config.protobuf.Visibility; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for {@link ConfigRegistry}. + */ +public class ConfigRegistrySuite { + + private static final String TEST_CONFIG_FILE = + "org/apache/spark/config/test_configs.textproto"; + + private static final ConfigRegistry registry = new ConfigRegistry(TEST_CONFIG_FILE); + + private static final ClassLoader CLASS_LOADER = ConfigRegistrySuite.class.getClassLoader(); + + // ========================================================================== + // Validation helpers + // ========================================================================== + + /** + * Validate configs in one file: required fields and alphabetical ordering. + * Convenience method that loads the file first. + * + * @param file the resource path of the .textproto file + */ + private static void validateConfigsInOneFile(String file) { + List configs = ConfigRegistry.loadConfigFile(CLASS_LOADER, file); + validateConfigsInOneFile(configs, file); + } + + /** + * Validate configs: required fields and alphabetical ordering. + * + * @param configs the list of config entries + * @param file the file path (for error messages) + */ + private static void validateConfigsInOneFile(List configs, String file) { + String previousKey = null; + for (ConfigEntry config : configs) { + // Validate required fields + List missingFields = new ArrayList<>(); + if (config.getKey().isEmpty()) { + missingFields.add("key"); + } + if (config.getValueType() == ValueType.VALUE_TYPE_UNSPECIFIED) { + missingFields.add("value_type"); + } + if (config.getScope() == Scope.SCOPE_UNSPECIFIED) { + missingFields.add("scope"); + } + if (config.getVisibility() == Visibility.VISIBILITY_UNSPECIFIED) { + missingFields.add("visibility"); + } + if (config.getBindingPolicy() == BindingPolicy.BINDING_POLICY_UNSPECIFIED) { + missingFields.add("binding_policy"); + } + if (config.getMutability() == Mutability.MUTABILITY_UNSPECIFIED) { + missingFields.add("mutability"); + } + if (config.getDoc().isEmpty()) { + missingFields.add("doc"); + } + if (!missingFields.isEmpty()) { + String keyInfo = config.getKey().isEmpty() ? "" : " for key '" + config.getKey() + "'"; + fail("Missing required fields " + missingFields + keyInfo + " in " + file); + } + + // TODO: Relax this constraint once the config framework fully supports + // cluster-scoped dynamic configs and session-scoped static configs. + if (config.getScope() == Scope.SCOPE_CLUSTER + && config.getMutability() != Mutability.MUTABILITY_STATIC) { + fail("CLUSTER scope config must be STATIC" + + " for key '" + config.getKey() + "' in " + file); + } + if (config.getScope() == Scope.SCOPE_SESSION + && config.getMutability() != Mutability.MUTABILITY_DYNAMIC) { + fail("SESSION scope config must be DYNAMIC" + + " for key '" + config.getKey() + "' in " + file); + } + + // A test_default only overrides default_value in test environments, so it requires a + // default_value to override. Otherwise the entry would have a default under test but none + // in production, diverging its shape between the two. + if (config.hasTestDefault() && !config.hasDefaultValue()) { + fail("test_default requires default_value" + + " for key '" + config.getKey() + "' in " + file); + } + + // Validate alphabetical ordering + String key = config.getKey(); + if (previousKey != null && key.compareTo(previousKey) < 0) { + fail("Config keys must be ordered alphabetically in " + file + + ": '" + key + "' should come before '" + previousKey + "'"); + } + previousKey = key; + } + } + + /** + * Validate all config files: required fields, ordering, and no duplicate keys. + * Each file is loaded only once. + * + * @param configFiles the list of config file paths to check + */ + private static void validateAllConfigFiles(String[] configFiles) { + Map keyToFile = new HashMap<>(); + for (String file : configFiles) { + List configs = ConfigRegistry.loadConfigFile(CLASS_LOADER, file); + // Validate required fields and ordering + validateConfigsInOneFile(configs, file); + // Validate no duplicate keys across files + for (ConfigEntry config : configs) { + String key = config.getKey(); + if (keyToFile.containsKey(key)) { + fail("Duplicate config key '" + key + "': " + + "first defined in " + keyToFile.get(key) + ", duplicated in " + file); + } + keyToFile.put(key, file); + } + } + } + + // ========================================================================== + // Basic functionality tests + // ========================================================================== + + @Test + public void testLoadConfigs() { + assertEquals(6, registry.keys().size()); + assertTrue(registry.contains("spark.test.bool.config")); + assertTrue(registry.contains("spark.test.int.config")); + assertTrue(registry.contains("spark.test.string.config")); + assertTrue(registry.contains("spark.test.long.doc.config")); + assertTrue(registry.contains("spark.test.optional.config")); + assertTrue(registry.contains("spark.test.testdefault.config")); + } + + @Test + public void testBoolConfig() { + ConfigEntry config = registry.get("spark.test.bool.config"); + assertNotNull(config); + assertEquals("spark.test.bool.config", config.getKey()); + assertEquals(ValueType.VALUE_TYPE_BOOL, config.getValueType()); + assertEquals("true", config.getDefaultValue()); + assertEquals(Scope.SCOPE_SESSION, config.getScope()); + assertEquals(Visibility.VISIBILITY_PUBLIC, config.getVisibility()); + assertEquals("A test boolean config", config.getDoc()); + assertEquals("4.0.0", config.getVersion()); + assertEquals(BindingPolicy.BINDING_POLICY_SESSION, config.getBindingPolicy()); + } + + @Test + public void testIntConfig() { + ConfigEntry config = registry.get("spark.test.int.config"); + assertNotNull(config); + assertEquals("spark.test.int.config", config.getKey()); + assertEquals(ValueType.VALUE_TYPE_INT, config.getValueType()); + assertEquals("42", config.getDefaultValue()); + assertEquals(Scope.SCOPE_CLUSTER, config.getScope()); + assertEquals(Visibility.VISIBILITY_INTERNAL, config.getVisibility()); + assertEquals("A test integer config", config.getDoc()); + assertEquals("4.0.0", config.getVersion()); + assertEquals(BindingPolicy.BINDING_POLICY_NOT_APPLICABLE, config.getBindingPolicy()); + } + + @Test + public void testStringConfig() { + ConfigEntry config = registry.get("spark.test.string.config"); + assertNotNull(config); + assertEquals("spark.test.string.config", config.getKey()); + assertEquals(ValueType.VALUE_TYPE_STRING, config.getValueType()); + assertEquals("default_value", config.getDefaultValue()); + assertEquals(Scope.SCOPE_SESSION, config.getScope()); + assertEquals(Visibility.VISIBILITY_PUBLIC, config.getVisibility()); + assertEquals("A test string config", config.getDoc()); + assertEquals("4.0.0", config.getVersion()); + assertEquals(BindingPolicy.BINDING_POLICY_NOT_APPLICABLE, config.getBindingPolicy()); + } + + @Test + public void testGetNonExistent() { + assertNull(registry.get("spark.nonexistent.config")); + assertFalse(registry.contains("spark.nonexistent.config")); + } + + @Test + public void testAll() { + assertEquals(6, registry.all().size()); + } + + @Test + public void testOptionalConfig() { + // A config without a default_value: hasDefaultValue is false and the field is empty. + ConfigEntry config = registry.get("spark.test.optional.config"); + assertNotNull(config); + assertEquals("spark.test.optional.config", config.getKey()); + assertEquals(ValueType.VALUE_TYPE_STRING, config.getValueType()); + assertFalse(config.hasDefaultValue()); + assertEquals("", config.getDefaultValue()); + } + + @Test + public void testTestDefaultConfig() { + // A config with a test_default distinct from default_value. + ConfigEntry config = registry.get("spark.test.testdefault.config"); + assertNotNull(config); + assertEquals("spark.test.testdefault.config", config.getKey()); + assertEquals("10", config.getDefaultValue()); + assertTrue(config.hasTestDefault()); + assertEquals("20", config.getTestDefault()); + } + + @Test + public void testMultiLineDoc() { + ConfigEntry config = registry.get("spark.test.long.doc.config"); + assertNotNull(config); + assertEquals("spark.test.long.doc.config", config.getKey()); + assertEquals(ValueType.VALUE_TYPE_STRING, config.getValueType()); + assertEquals("test", config.getDefaultValue()); + assertEquals(Scope.SCOPE_SESSION, config.getScope()); + assertEquals(Visibility.VISIBILITY_PUBLIC, config.getVisibility()); + // Verify multi-line string concatenation works + String expectedDoc = "This is a very long documentation string that spans multiple lines. " + + "It demonstrates the prototext multi-line string concatenation feature. " + + "Each quoted segment will be concatenated together into a single string, " + + "which is useful for configs with lengthy descriptions."; + assertEquals(expectedDoc, config.getDoc()); + assertEquals("4.0.0", config.getVersion()); + } + + // ========================================================================== + // Validation of production config files + // ========================================================================== + + @Test + public void testProductionConfigsAreValid() { + validateAllConfigFiles(ConfigRegistry.DEFAULT_CONFIG_FILES); + } + + // ========================================================================== + // Tests for validation logic using invalid config files + // ========================================================================== + + @Test + public void testValidationMissingKey() { + String file = "org/apache/spark/config/invalid_missing_key.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("key")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingValueType() { + String file = "org/apache/spark/config/invalid_missing_value_type.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("value_type")); + assertTrue(error.getMessage().contains("spark.test.missing.value.type")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingScope() { + String file = "org/apache/spark/config/invalid_missing_scope.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("scope")); + assertTrue(error.getMessage().contains("spark.test.missing.scope")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingVisibility() { + String file = "org/apache/spark/config/invalid_missing_visibility.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("visibility")); + assertTrue(error.getMessage().contains("spark.test.missing.visibility")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingDoc() { + String file = "org/apache/spark/config/invalid_missing_doc.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("doc")); + assertTrue(error.getMessage().contains("spark.test.missing.doc")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingBindingPolicy() { + String file = "org/apache/spark/config/invalid_missing_binding_policy.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("binding_policy")); + assertTrue(error.getMessage().contains("spark.test.missing.binding.policy")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationMissingMutability() { + String file = "org/apache/spark/config/invalid_missing_mutability.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("Missing required fields")); + assertTrue(error.getMessage().contains("mutability")); + assertTrue(error.getMessage().contains("spark.test.missing.mutability")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationTestDefaultWithoutDefault() { + String file = "org/apache/spark/config/invalid_testdefault_without_default.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("test_default requires default_value")); + assertTrue(error.getMessage().contains("spark.test.testdefault.without.default")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationUnorderedConfigs() { + String file = "org/apache/spark/config/invalid_unordered.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateConfigsInOneFile(file)); + assertTrue(error.getMessage().contains("must be ordered alphabetically")); + assertTrue(error.getMessage().contains(file)); + } + + @Test + public void testValidationDuplicateKeys() { + String file1 = TEST_CONFIG_FILE; + String file2 = "org/apache/spark/config/duplicate_key_file2.textproto"; + Throwable error = assertThrows(Throwable.class, + () -> validateAllConfigFiles(new String[]{file1, file2})); + assertTrue(error.getMessage().contains("Duplicate config key")); + assertTrue(error.getMessage().contains("spark.test.bool.config")); + assertTrue(error.getMessage().contains(file1)); + assertTrue(error.getMessage().contains(file2)); + } +} diff --git a/common/config/src/test/resources/org/apache/spark/config/duplicate_key_file2.textproto b/common/config/src/test/resources/org/apache/spark/config/duplicate_key_file2.textproto new file mode 100644 index 0000000000000..5221f87af8839 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/duplicate_key_file2.textproto @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Second test file with a key that duplicates one in test_configs.textproto + +configs { + key: "spark.test.bool.config" + value_type: VALUE_TYPE_BOOL + default_value: "false" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_SESSION + doc: "Duplicate of spark.test.bool.config" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_binding_policy.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_binding_policy.textproto new file mode 100644 index 0000000000000..3f99ec3facb6f --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_binding_policy.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing binding_policy field + +configs { + key: "spark.test.missing.binding.policy" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + doc: "A config with missing binding_policy" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_doc.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_doc.textproto new file mode 100644 index 0000000000000..a0dcbfd6afb64 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_doc.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing doc field + +configs { + key: "spark.test.missing.doc" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_key.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_key.textproto new file mode 100644 index 0000000000000..14f561da7c501 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_key.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing key field + +configs { + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config with missing key" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_mutability.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_mutability.textproto new file mode 100644 index 0000000000000..e8b759941515b --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_mutability.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing mutability field + +configs { + key: "spark.test.missing.mutability" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config with missing mutability" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_scope.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_scope.textproto new file mode 100644 index 0000000000000..56f3661f4d032 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_scope.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing scope field + +configs { + key: "spark.test.missing.scope" + value_type: VALUE_TYPE_BOOL + default_value: "true" + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config with missing scope" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_value_type.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_value_type.textproto new file mode 100644 index 0000000000000..7652ac680f66d --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_value_type.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing value_type field + +configs { + key: "spark.test.missing.value.type" + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config with missing value_type" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_missing_visibility.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_visibility.textproto new file mode 100644 index 0000000000000..e1a5a731e6167 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_missing_visibility.textproto @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with missing visibility field + +configs { + key: "spark.test.missing.visibility" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config with missing visibility" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_testdefault_without_default.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_testdefault_without_default.textproto new file mode 100644 index 0000000000000..a9abe3b60a133 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_testdefault_without_default.textproto @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with test_default but no default_value + +configs { + key: "spark.test.testdefault.without.default" + value_type: VALUE_TYPE_INT + test_default: "20" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A test config with a test default but no default value" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/invalid_unordered.textproto b/common/config/src/test/resources/org/apache/spark/config/invalid_unordered.textproto new file mode 100644 index 0000000000000..99a6c983717d6 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/invalid_unordered.textproto @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test file with unordered config keys (z comes before a) + +configs { + key: "spark.test.z.config" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "Z config" + version: "4.0.0" +} + +configs { + key: "spark.test.a.config" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A config - should come before Z" + version: "4.0.0" +} diff --git a/common/config/src/test/resources/org/apache/spark/config/test_configs.textproto b/common/config/src/test/resources/org/apache/spark/config/test_configs.textproto new file mode 100644 index 0000000000000..3ce51f2d0b4c6 --- /dev/null +++ b/common/config/src/test/resources/org/apache/spark/config/test_configs.textproto @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test configuration entries + +configs { + key: "spark.test.bool.config" + value_type: VALUE_TYPE_BOOL + default_value: "true" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_SESSION + doc: "A test boolean config" + version: "4.0.0" +} + +configs { + key: "spark.test.int.config" + value_type: VALUE_TYPE_INT + default_value: "42" + scope: SCOPE_CLUSTER + mutability: MUTABILITY_STATIC + visibility: VISIBILITY_INTERNAL + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A test integer config" + version: "4.0.0" +} + +configs { + key: "spark.test.long.doc.config" + value_type: VALUE_TYPE_STRING + default_value: "test" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_PERSISTED + doc: "This is a very long documentation string that spans multiple lines. " + "It demonstrates the prototext multi-line string concatenation feature. " + "Each quoted segment will be concatenated together into a single string, " + "which is useful for configs with lengthy descriptions." + version: "4.0.0" +} + +configs { + key: "spark.test.optional.config" + value_type: VALUE_TYPE_STRING + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A test config without a default value" + version: "4.0.0" +} + +configs { + key: "spark.test.string.config" + value_type: VALUE_TYPE_STRING + default_value: "default_value" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A test string config" + version: "4.0.0" +} + +configs { + key: "spark.test.testdefault.config" + value_type: VALUE_TYPE_INT + default_value: "10" + test_default: "20" + scope: SCOPE_SESSION + mutability: MUTABILITY_DYNAMIC + visibility: VISIBILITY_PUBLIC + binding_policy: BINDING_POLICY_NOT_APPLICABLE + doc: "A test config with a distinct test default" + version: "4.0.0" +} diff --git a/common/utils/pom.xml b/common/utils/pom.xml index e09467141c700..17d1186201e71 100644 --- a/common/utils/pom.xml +++ b/common/utils/pom.xml @@ -39,6 +39,11 @@ org.apache.spark spark-tags_${scala.binary.version} + + org.apache.spark + spark-config_${scala.binary.version} + ${project.version} + org.apache.spark spark-common-utils-java_${scala.binary.version} diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index eca32c03d3e0f..7dd73d6efef62 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -331,6 +331,12 @@ ], "sqlState" : "22023" }, + "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT" : { + "message" : [ + "The column `` in the schema collides with a reserved AutoCDC column name (using column name comparison). The following column names are reserved by AutoCDC and cannot appear in the source: . Rename or remove the column." + ], + "sqlState" : "42710" + }, "AUTOCDC_RESERVED_COLUMN_NAME_PREFIX_CONFLICT" : { "message" : [ "The column `` in the schema collides with the reserved AutoCDC column name prefix `` (using column name comparison). Rename or remove the column." @@ -557,6 +563,44 @@ ], "sqlState" : "0A000" }, + "CANNOT_LOAD_CATALOG" : { + "message" : [ + "Cannot load catalog '' with the plugin class '':" + ], + "subClass" : { + "ABSTRACT_CLASS" : { + "message" : [ + "the class is abstract and cannot be instantiated." + ] + }, + "CONSTRUCTOR_FAILURE" : { + "message" : [ + "the constructor threw an exception during instantiation." + ] + }, + "CONSTRUCTOR_NOT_ACCESSIBLE" : { + "message" : [ + "failed to call the public no-arg constructor." + ] + }, + "CONSTRUCTOR_NOT_FOUND" : { + "message" : [ + "failed to find the public no-arg constructor." + ] + }, + "NOT_A_CATALOG_PLUGIN" : { + "message" : [ + "the class does not implement CatalogPlugin." + ] + }, + "PLUGIN_CLASS_NOT_FOUND" : { + "message" : [ + "cannot find the plugin class." + ] + } + }, + "sqlState" : "46103" + }, "CANNOT_LOAD_CHECKPOINT_FILE_MANAGER" : { "message" : [ "Error loading streaming checkpoint file manager for path=." @@ -1254,6 +1298,11 @@ "" ] }, + "BROADCAST_NOT_FOUND" : { + "message" : [ + "Cannot find a broadcast variable with id: . It may never have been created on this session, may belong to another session, or may have been unpersisted or destroyed." + ] + }, "CANNOT_FIND_CACHED_LOCAL_RELATION" : { "message" : [ "Cannot find a cached local relation for hash: " @@ -10957,36 +11006,6 @@ "Invalid catalog name: ." ] }, - "_LEGACY_ERROR_TEMP_2214" : { - "message" : [ - "Plugin class for catalog '' does not implement CatalogPlugin: ." - ] - }, - "_LEGACY_ERROR_TEMP_2215" : { - "message" : [ - "Cannot find catalog plugin class for catalog '': ." - ] - }, - "_LEGACY_ERROR_TEMP_2216" : { - "message" : [ - "Failed to find public no-arg constructor for catalog '': )." - ] - }, - "_LEGACY_ERROR_TEMP_2217" : { - "message" : [ - "Failed to call public no-arg constructor for catalog '': )." - ] - }, - "_LEGACY_ERROR_TEMP_2218" : { - "message" : [ - "Cannot instantiate abstract catalog plugin class for catalog '': ." - ] - }, - "_LEGACY_ERROR_TEMP_2219" : { - "message" : [ - "Failed during instantiating constructor for catalog '': ." - ] - }, "_LEGACY_ERROR_TEMP_2220" : { "message" : [ "" diff --git a/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala b/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala index 97fddf632ed23..abede2d57d519 100644 --- a/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala +++ b/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala @@ -17,6 +17,11 @@ package org.apache.spark.internal.config +import org.apache.spark.SparkException +import org.apache.spark.config.ConfigRegistry +import org.apache.spark.config.protobuf.{BindingPolicy, ConfigEntry => ProtoConfigEntry, ValueType, Visibility} +import org.apache.spark.util.SparkEnvUtils + // ==================================================================================== // The guideline for naming configurations // ==================================================================================== @@ -283,6 +288,107 @@ private[spark] class FallbackConfigEntry[T] ( } } +/** + * Marker trait for config entries backed by proto definitions. + */ +private[spark] sealed trait ProtoBackedBase + +/** + * A proto-backed config entry with a default value. + */ +private[spark] class ProtoBackedConfigEntry[T]( + protoEntry: ProtoConfigEntry, + valueConverter: String => T, + stringConverter: T => String) + extends ConfigEntry[T]( + key = protoEntry.getKey, + prependedKey = None, + prependSeparator = "", + alternatives = Nil, + valueConverter = valueConverter, + stringConverter = stringConverter, + doc = protoEntry.getDoc, + isPublic = protoEntry.getVisibility != Visibility.VISIBILITY_INTERNAL, + version = protoEntry.getVersion, + bindingPolicy = ProtoBackedConfigEntry.toBindingPolicy(protoEntry) + ) with ProtoBackedBase { + + override def defaultValueString: String = + defaultValue.map(stringConverter).getOrElse(ConfigEntry.UNDEFINED) + + // The default is derived from immutable proto fields (and the fixed testing flag), so convert it + // once and cache rather than re-parsing the string on every default-fallback read. + private lazy val cachedDefaultValue: Option[T] = { + val defaultStrOpt = if (SparkEnvUtils.isTesting && protoEntry.hasTestDefault) { + Some(protoEntry.getTestDefault) + } else if (protoEntry.hasDefaultValue) { + Some(protoEntry.getDefaultValue) + } else { + None + } + defaultStrOpt.map(valueConverter) + } + + override def defaultValue: Option[T] = cachedDefaultValue + + override def readFrom(reader: ConfigReader): T = { + readString(reader).map(valueConverter).getOrElse(defaultValue.get) + } + + def checkValue(validator: T => Boolean, errorMsg: String): ProtoBackedConfigEntry[T] = { + new ProtoBackedConfigEntry[T]( + protoEntry, + str => { + val v = valueConverter(str) + if (!validator(v)) { + throw ConfigHelpers.configRequirementError(key, str, errorMsg) + } + v + }, + stringConverter + ) + } +} + +/** + * A proto-backed config entry that does not have a default value. + */ +private[spark] class ProtoBackedOptionalConfigEntry[T]( + protoEntry: ProtoConfigEntry, + rawValueConverter: String => T, + rawStringConverter: T => String) + extends ConfigEntry[Option[T]]( + key = protoEntry.getKey, + prependedKey = None, + prependSeparator = "", + alternatives = Nil, + valueConverter = s => Some(rawValueConverter(s)), + stringConverter = v => v.map(rawStringConverter).orNull, + doc = protoEntry.getDoc, + isPublic = protoEntry.getVisibility != Visibility.VISIBILITY_INTERNAL, + version = protoEntry.getVersion, + bindingPolicy = ProtoBackedConfigEntry.toBindingPolicy(protoEntry) + ) with ProtoBackedBase { + + override def defaultValueString: String = ConfigEntry.UNDEFINED + + override def readFrom(reader: ConfigReader): Option[T] = { + readString(reader).map(rawValueConverter) + } +} + +private[spark] object ProtoBackedConfigEntry { + def toBindingPolicy( + protoEntry: ProtoConfigEntry): Option[ConfigBindingPolicy.Value] = { + protoEntry.getBindingPolicy match { + case BindingPolicy.BINDING_POLICY_SESSION => Some(ConfigBindingPolicy.SESSION) + case BindingPolicy.BINDING_POLICY_PERSISTED => Some(ConfigBindingPolicy.PERSISTED) + case BindingPolicy.BINDING_POLICY_NOT_APPLICABLE => Some(ConfigBindingPolicy.NOT_APPLICABLE) + case _ => None + } + } +} + private[spark] object ConfigEntry { val UNDEFINED = "" @@ -290,13 +396,94 @@ private[spark] object ConfigEntry { private[spark] val knownConfigs = new java.util.concurrent.ConcurrentHashMap[String, ConfigEntry[_]]() + // Register all proto-backed configs at object initialization. + // These can be overwritten later by modules that need to add validation. + ConfigRegistry.allConfigs().forEach { protoEntry => + createProtoBackedConfigEntry(protoEntry) + } + def registerEntry(entry: ConfigEntry[_]): Unit = { val existing = knownConfigs.putIfAbsent(entry.key, entry) - require(existing == null, s"Config entry ${entry.key} already registered!") + if (existing != null) { + // A key registered twice is normally a bug (typo or accidental duplicate), so we fail loudly. + // The one exception is the enhancement pattern: a proto-backed config is registered eagerly + // at object init, then a Scala entry built via `buildConfFromConfigFile` (e.g. to add + // `checkValue`) may intentionally replace it. We allow the overwrite only when the existing + // entry is proto-backed; this deliberately trades the duplicate-detection net for that key, + // so the Scala side must still reference a key that is genuinely defined in a .textproto + // file. + require(existing.isInstanceOf[ProtoBackedBase] && entry.isInstanceOf[ProtoBackedBase], + s"Config entry ${entry.key} already registered!") + knownConfigs.put(entry.key, entry) + } } def findEntry(key: String): ConfigEntry[_] = knownConfigs.get(key) def listAllEntries(): java.util.Collection[ConfigEntry[_]] = knownConfigs.values() + def findProtoBackedEntry(key: String): ProtoBackedConfigEntry[_] = { + knownConfigs.get(key) match { + case entry: ProtoBackedConfigEntry[_] => entry + case _ => null + } + } + + def findProtoDefinedEntry(key: String): ConfigEntry[_] = { + if (ConfigRegistry.containsConfig(key)) knownConfigs.get(key) else null + } + + def listAllProtoDefinedConfigs(): java.util.Collection[ConfigEntry[_]] = { + new java.util.AbstractCollection[ConfigEntry[_]]() { + override def iterator(): java.util.Iterator[ConfigEntry[_]] = { + val keys = ConfigRegistry.allKeys().iterator() + new java.util.Iterator[ConfigEntry[_]]() { + override def hasNext: Boolean = keys.hasNext + override def next(): ConfigEntry[_] = knownConfigs.get(keys.next()) + } + } + + override def size(): Int = ConfigRegistry.allKeys().size() + } + } + + private def createProtoBackedConfigEntry( + protoEntry: ProtoConfigEntry): ConfigEntry[_] = { + // `test_default` only overrides `default_value` in test environments; a config that sets + // `test_default` without `default_value` would be a has-default entry under test but an + // optional entry in production, diverging the entry's shape (and `buildConfFromConfigFile`'s + // success/failure) between the two. Reject it here so the misconfiguration fails consistently. + require(!protoEntry.hasTestDefault || protoEntry.hasDefaultValue, + s"Config entry ${protoEntry.getKey} sets test_default without default_value; " + + "a config with a test default must also declare a default value.") + // With the invariant above, `test_default` implies `default_value`, so having a default is + // equivalent to having a `default_value` in every environment. + val hasDefault = protoEntry.hasDefaultValue + protoEntry.getValueType match { + case ValueType.VALUE_TYPE_BOOL => + createTypedEntry[Boolean](protoEntry, hasDefault, _.toBoolean, _.toString) + case ValueType.VALUE_TYPE_INT => + createTypedEntry[Int](protoEntry, hasDefault, _.toInt, _.toString) + case ValueType.VALUE_TYPE_LONG => + createTypedEntry[Long](protoEntry, hasDefault, _.toLong, _.toString) + case ValueType.VALUE_TYPE_DOUBLE => + createTypedEntry[Double](protoEntry, hasDefault, _.toDouble, _.toString) + case ValueType.VALUE_TYPE_STRING => + createTypedEntry[String](protoEntry, hasDefault, identity[String], identity[String]) + case other => + throw SparkException.internalError(s"Unsupported value type: $other") + } + } + + private def createTypedEntry[T]( + protoEntry: ProtoConfigEntry, + hasDefault: Boolean, + valueConverter: String => T, + stringConverter: T => String): ConfigEntry[_] = { + if (hasDefault) { + new ProtoBackedConfigEntry[T](protoEntry, valueConverter, stringConverter) + } else { + new ProtoBackedOptionalConfigEntry[T](protoEntry, valueConverter, stringConverter) + } + } } diff --git a/dev/mima b/dev/mima index 17558be37fc8e..39be02a1dd557 100755 --- a/dev/mima +++ b/dev/mima @@ -25,8 +25,16 @@ FWDIR="$(cd "`dirname "$0"`"/..; pwd)" cd "$FWDIR" SPARK_PROFILES=${1:-"-Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} + +# Capture sbt output to show errors if the command fails +OLD_DEPS_OUTPUT="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" 2>&1)" || { + echo "ERROR: Failed to export oldDeps classpath:" + echo "$OLD_DEPS_OUTPUT" | grep -E "^\[error\]" + exit 1 +} +OLD_DEPS_CLASSPATH="$(echo "$OLD_DEPS_OUTPUT" | grep jar | tail -n1)" + TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)" -OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)" rm -f .generated-mima* diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 77aa43eb8d947..8805f42d5308b 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -1167,6 +1167,7 @@ def __hash__(self): # sql unittests "pyspark.sql.tests.connect.test_connect_plan", "pyspark.sql.tests.connect.test_connect_basic", + "pyspark.sql.tests.connect.test_connect_broadcast", "pyspark.sql.tests.connect.test_connect_dataframe_property", "pyspark.sql.tests.connect.test_connect_channel", "pyspark.sql.tests.connect.test_connect_clone_session", diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 25a847198bcfc..30851e513a8cb 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -45,7 +45,7 @@ GEM sass-embedded (~> 1.75) jekyll-watch (2.2.1) listen (~> 3.0) - json (2.12.2) + json (2.21.1) kramdown (2.5.1) rexml (>= 3.3.9) kramdown-parser-gfm (1.1.0) diff --git a/pom.xml b/pom.xml index f6354961cf292..0eaf37497f683 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,7 @@ common/utils-java common/variant common/tags + common/config sql/connect/shims core graphx diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 53412c20d8313..ef027de5699f6 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -64,10 +64,10 @@ object BuildCommons { val allProjects@Seq( core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, kvstore, - commonUtils, commonUtilsJava, variant, pipelines, _* + commonUtils, commonUtilsJava, variant, pipelines, sparkConfig, _* ) = Seq( "core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe", - "tags", "sketch", "kvstore", "common-utils", "common-utils-java", "variant", "pipelines" + "tags", "sketch", "kvstore", "common-utils", "common-utils-java", "variant", "pipelines", "config" ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ connectProjects ++ udfWorkerProjects @@ -373,6 +373,7 @@ object SparkBuild extends PomBuild { "-groups", "-skip-packages", Seq( "org.apache.spark.api.python", + "org.apache.spark.config", "org.apache.spark.deploy", "org.apache.spark.kafka010", "org.apache.spark.network", @@ -415,7 +416,7 @@ object SparkBuild extends PomBuild { Seq( spark, hive, hiveThriftServer, repl, networkCommon, networkShuffle, networkYarn, unsafe, tags, tokenProviderKafka010, sqlKafka010, pipelines, connectCommon, connect, - connectJdbc, connectClient, variant, connectShims, profiler, commonUtilsJava, + connectJdbc, connectClient, variant, connectShims, profiler, commonUtilsJava, sparkConfig, udfWorkerProto, udfWorkerCore, udfWorkerGrpc ).contains(x) } @@ -475,6 +476,9 @@ object SparkBuild extends PomBuild { /* UDF Worker gRPC settings */ enable(UDFWorkerGrpc.settings)(udfWorkerGrpc) + /* Config module protobuf settings */ + enable(SparkConfig.settings)(sparkConfig) + enable(DockerIntegrationTests.settings)(dockerIntegrationTests) enable(KubernetesIntegrationTests.settings)(kubernetesIntegrationTests) @@ -682,6 +686,31 @@ object Core { } } +object SparkConfig { + import BuildCommons.protoVersion + lazy val settings = Seq( + // Setting version for the protobuf compiler. + PB.protocVersion := BuildCommons.protoVersion, + libraryDependencies ++= { + Seq( + "com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf" + ) + }, + (Compile / PB.targets) := Seq( + PB.gens.java -> (Compile / sourceManaged).value + ) + ) ++ { + val sparkProtocExecPath = sys.props.get("spark.protoc.executable.path") + if (sparkProtocExecPath.isDefined) { + Seq( + PB.protocExecutable := file(sparkProtocExecPath.get) + ) + } else { + Seq.empty + } + } +} + object SparkConnectCommon { import BuildCommons.protoVersion @@ -1642,6 +1671,7 @@ object Unidoc { protected def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = { packages .map(_.filterNot(_.getName.contains("$"))) + .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/config"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/deploy"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/examples"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/internal"))) diff --git a/python/docs/source/reference/pyspark.sql/spark_session.rst b/python/docs/source/reference/pyspark.sql/spark_session.rst index 9b8dd0ae6564d..4057009096ec0 100644 --- a/python/docs/source/reference/pyspark.sql/spark_session.rst +++ b/python/docs/source/reference/pyspark.sql/spark_session.rst @@ -47,6 +47,7 @@ See also :class:`SparkSession`. SparkSession.addArtifact SparkSession.addArtifacts SparkSession.addTag + SparkSession.broadcast SparkSession.catalog SparkSession.clearTags SparkSession.conf diff --git a/python/pyspark/errors/error-conditions.json b/python/pyspark/errors/error-conditions.json index 07f47a0c91009..fdda2e9e71a48 100644 --- a/python/pyspark/errors/error-conditions.json +++ b/python/pyspark/errors/error-conditions.json @@ -30,6 +30,16 @@ "Length mismatch: Expected axis has elements, new values have elements." ] }, + "BROADCAST_NOT_FOUND": { + "message": [ + "Cannot find a broadcast variable with id: . It may never have been created on this session, may belong to another session, or may have been unpersisted or destroyed." + ] + }, + "BROADCAST_VALUE_TOO_LARGE": { + "message": [ + "The broadcast value of bytes exceeds the maximum allowed size of bytes." + ] + }, "BROADCAST_VARIABLE_NOT_LOADED": { "message": [ "Broadcast variable `` not loaded." diff --git a/python/pyspark/sql/connect/broadcast.py b/python/pyspark/sql/connect/broadcast.py new file mode 100644 index 0000000000000..4bd90d2d43652 --- /dev/null +++ b/python/pyspark/sql/connect/broadcast.py @@ -0,0 +1,114 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +(SPARK-51705) Client-side broadcast proxy for Spark Connect. + +``ConnectBroadcast`` mirrors the classic :class:`pyspark.core.broadcast.Broadcast` pickling +contract so that a UDF closure can reference a broadcast variable transparently. The value was +already uploaded to the server (cloudpickle -> cache artifact -> CreateBroadcastCommand) and a +driver-side broadcast id was returned; this proxy keeps a local copy of the value for driver-side +``.value`` reads, and on pickling emits ``(_from_id, (broadcast_id,))`` -- exactly what the classic +:class:`Broadcast` emits -- so the executor/worker resolves it from its ``_broadcastRegistry`` +keyed by that same id. No JVM is available on the Connect client, so all lifecycle operations are +routed to the server as commands. +""" + +import threading +from typing import Any, Generic, Tuple, TYPE_CHECKING, TypeVar + +# NOTE: This reuses the classic ``pyspark.core.broadcast`` machinery so the pickled UDF closure is +# byte-identical to classic PySpark (``_from_id`` + the driver-side broadcast id). ``pyspark.core`` +# is not available in a Spark Connect-only install (``pyspark-client``); this module is therefore +# imported lazily (from ``SparkSession.broadcast`` and ``PythonUDF.to_plan``), never at package +# import time, so the base Connect package still imports without the classic core present. +from pyspark.core.broadcast import Broadcast, _from_id + +if TYPE_CHECKING: + from pyspark.sql.connect.session import SparkSession + +T = TypeVar("T") + + +class _BroadcastCaptureRegistry(threading.local): + """Thread-local registry of broadcast ids captured while pickling a UDF closure. + + Thread-local is correct because ``CloudPickleSerializer().dumps(...)`` in + ``PythonUDF.to_plan`` and the subsequent drain into ``PythonUDF.broadcast_ids`` happen + synchronously on the same plan-building thread. This mirrors the classic + :class:`pyspark.core.broadcast.BroadcastPickleRegistry`. + """ + + def __init__(self) -> None: + self.__dict__.setdefault("_registry", set()) + + def add(self, bid: int) -> None: + self._registry.add(bid) + + def drain(self) -> "list[int]": + """Return the captured ids and clear the registry (dumps-then-drain-then-clear).""" + captured = list(self._registry) + self._registry.clear() + return captured + + +# Module-level thread-local registry, drained by ``PythonUDF.to_plan``. +_broadcast_capture_registry = _BroadcastCaptureRegistry() + + +class ConnectBroadcast(Broadcast, Generic[T]): + """A broadcast variable created with :meth:`SparkSession.broadcast` over Spark Connect. + + See Also + -------- + pyspark.core.broadcast.Broadcast + """ + + def __init__(self, session: "SparkSession", broadcast_id: int, value: T) -> None: + # Intentionally does NOT call Broadcast.__init__ -- there is no live SparkContext on the + # Connect client. We populate only the attributes needed for driver reads and pickling. + self._session = session + self._broadcast_id = broadcast_id + self._value = value + # Mirror the classic executor-side shape: no JVM broadcast / SparkContext handle. + self._jbroadcast = None + self._sc = None + self._python_broadcast = None + + @property + def value(self) -> T: + """Return the broadcasted value (kept locally on the client for driver-side reads).""" + return self._value + + def unpersist(self, blocking: bool = False) -> None: + """Delete cached copies of this broadcast on the executors via the server.""" + self._session._client._unpersist_broadcast( + self._broadcast_id, blocking=blocking, destroy=False + ) + + def destroy(self, blocking: bool = False) -> None: + """Destroy all data and metadata related to this broadcast via the server.""" + self._session._client._unpersist_broadcast( + self._broadcast_id, blocking=blocking, destroy=True + ) + + def __reduce__(self) -> Tuple[Any, Tuple[int]]: + # Verbatim classic contract: reference the shared classic ``_from_id`` and the driver-side + # broadcast id, so the worker resolves it from ``pyspark.core.broadcast._broadcastRegistry`` + # keyed by the same id. Also side-register the id so ``PythonUDF.to_plan`` can attach it to + # the proto ``broadcast_ids`` field. + _broadcast_capture_registry.add(self._broadcast_id) + return _from_id, (self._broadcast_id,) diff --git a/python/pyspark/sql/connect/client/core.py b/python/pyspark/sql/connect/client/core.py index 19618f2d79681..64445bb56c5a3 100644 --- a/python/pyspark/sql/connect/client/core.py +++ b/python/pyspark/sql/connect/client/core.py @@ -1883,6 +1883,8 @@ def handle_response( if b.HasField("create_resource_profile_command_result"): profile_id = b.create_resource_profile_command_result.profile_id yield {"create_resource_profile_command_result": profile_id} + if b.HasField("create_broadcast_result"): + yield {"create_broadcast_result": b.create_broadcast_result.broadcast_id} if b.HasField("checkpoint_command_result"): yield { "checkpoint_command_result": proto_to_remote_cached_dataframe( @@ -2512,6 +2514,27 @@ def _create_profile(self, profile: pb2.ResourceProfile) -> int: profile_id = properties["create_resource_profile_command_result"] return profile_id + def _create_broadcast(self, artifact_hash: str, size_bytes: int) -> int: + """(SPARK-51705) Create a broadcast variable from an already-uploaded cache artifact and + return the server-assigned (driver-side) broadcast id.""" + logger.debug("Creating a broadcast variable") + cmd = pb2.Command() + cmd.create_broadcast_command.artifact_hash = artifact_hash + cmd.create_broadcast_command.size_bytes = size_bytes + _, properties, _ = self.execute_command(cmd) + return properties["create_broadcast_result"] + + def _unpersist_broadcast( + self, broadcast_id: int, blocking: bool = False, destroy: bool = False + ) -> None: + """(SPARK-51705) Unpersist (or destroy) a broadcast variable created over Connect.""" + logger.debug("Unpersisting a broadcast variable") + cmd = pb2.Command() + cmd.unpersist_broadcast_command.broadcast_id = broadcast_id + cmd.unpersist_broadcast_command.blocking = blocking + cmd.unpersist_broadcast_command.destroy = destroy + self.execute_command(cmd) + def _delete_ml_cache(self, cache_ids: List[str], evict_only: bool = False) -> List[str]: # try best to delete the cache try: diff --git a/python/pyspark/sql/connect/expressions.py b/python/pyspark/sql/connect/expressions.py index 57270398118f7..ae304fac4e8c4 100644 --- a/python/pyspark/sql/connect/expressions.py +++ b/python/pyspark/sql/connect/expressions.py @@ -763,6 +763,15 @@ def to_plan(self, session: "SparkConnectClient") -> proto.PythonUDF: expr.eval_type = self._eval_type expr.command = CloudPickleSerializer().dumps((self._func, output_type)) expr.python_ver = self._python_ver + # (SPARK-51705) Drain the thread-local broadcast capture registry populated while pickling + # the command above (ConnectBroadcast.__reduce__ side-registers its id). This mirrors the + # classic dumps-then-drain-then-clear idiom in rdd._prepare_for_python_RDD, and covers both + # inline UDFs and spark.udf.register since both funnel through this to_plan. + from pyspark.sql.connect.broadcast import _broadcast_capture_registry + + broadcast_ids = _broadcast_capture_registry.drain() + if broadcast_ids: + expr.broadcast_ids.extend(broadcast_ids) return expr def __repr__(self) -> str: diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py index a77c61ca6d2b4..0ec973ce5c56b 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.py +++ b/python/pyspark/sql/connect/proto/base_pb2.py @@ -46,7 +46,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xe3\x03\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommand\x12\\\n\x14\x63ompressed_operation\x18\x03 \x01(\x0b\x32\'.spark.connect.Plan.CompressedOperationH\x00R\x13\x63ompressedOperation\x1a\x8e\x02\n\x13\x43ompressedOperation\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12G\n\x07op_type\x18\x02 \x01(\x0e\x32..spark.connect.Plan.CompressedOperation.OpTypeR\x06opType\x12L\n\x11\x63ompression_codec\x18\x03 \x01(\x0e\x32\x1f.spark.connect.CompressionCodecR\x10\x63ompressionCodec"L\n\x06OpType\x12\x17\n\x13OP_TYPE_UNSPECIFIED\x10\x00\x12\x14\n\x10OP_TYPE_RELATION\x10\x01\x12\x13\n\x0fOP_TYPE_COMMAND\x10\x02\x42\t\n\x07op_type"z\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xf5\x14\n\x12\x41nalyzePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x11 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12\x42\n\x06schema\x18\x04 \x01(\x0b\x32(.spark.connect.AnalyzePlanRequest.SchemaH\x00R\x06schema\x12\x45\n\x07\x65xplain\x18\x05 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.ExplainH\x00R\x07\x65xplain\x12O\n\x0btree_string\x18\x06 \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.TreeStringH\x00R\ntreeString\x12\x46\n\x08is_local\x18\x07 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.IsLocalH\x00R\x07isLocal\x12R\n\x0cis_streaming\x18\x08 \x01(\x0b\x32-.spark.connect.AnalyzePlanRequest.IsStreamingH\x00R\x0bisStreaming\x12O\n\x0binput_files\x18\t \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.InputFilesH\x00R\ninputFiles\x12U\n\rspark_version\x18\n \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SparkVersionH\x00R\x0csparkVersion\x12I\n\tddl_parse\x18\x0b \x01(\x0b\x32*.spark.connect.AnalyzePlanRequest.DDLParseH\x00R\x08\x64\x64lParse\x12X\n\x0esame_semantics\x18\x0c \x01(\x0b\x32/.spark.connect.AnalyzePlanRequest.SameSemanticsH\x00R\rsameSemantics\x12U\n\rsemantic_hash\x18\r \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SemanticHashH\x00R\x0csemanticHash\x12\x45\n\x07persist\x18\x0e \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.PersistH\x00R\x07persist\x12K\n\tunpersist\x18\x0f \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.UnpersistH\x00R\tunpersist\x12_\n\x11get_storage_level\x18\x10 \x01(\x0b\x32\x31.spark.connect.AnalyzePlanRequest.GetStorageLevelH\x00R\x0fgetStorageLevel\x12M\n\x0bjson_to_ddl\x18\x12 \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.JsonToDDLH\x00R\tjsonToDdl\x1a\x31\n\x06Schema\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\xbb\x02\n\x07\x45xplain\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12X\n\x0c\x65xplain_mode\x18\x02 \x01(\x0e\x32\x35.spark.connect.AnalyzePlanRequest.Explain.ExplainModeR\x0b\x65xplainMode"\xac\x01\n\x0b\x45xplainMode\x12\x1c\n\x18\x45XPLAIN_MODE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XPLAIN_MODE_SIMPLE\x10\x01\x12\x19\n\x15\x45XPLAIN_MODE_EXTENDED\x10\x02\x12\x18\n\x14\x45XPLAIN_MODE_CODEGEN\x10\x03\x12\x15\n\x11\x45XPLAIN_MODE_COST\x10\x04\x12\x1a\n\x16\x45XPLAIN_MODE_FORMATTED\x10\x05\x1aZ\n\nTreeString\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12\x19\n\x05level\x18\x02 \x01(\x05H\x00R\x05level\x88\x01\x01\x42\x08\n\x06_level\x1a\x32\n\x07IsLocal\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x36\n\x0bIsStreaming\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x35\n\nInputFiles\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x0e\n\x0cSparkVersion\x1a)\n\x08\x44\x44LParse\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlString\x1ay\n\rSameSemantics\x12\x34\n\x0btarget_plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\ntargetPlan\x12\x32\n\nother_plan\x18\x02 \x01(\x0b\x32\x13.spark.connect.PlanR\totherPlan\x1a\x37\n\x0cSemanticHash\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x97\x01\n\x07Persist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x45\n\rstorage_level\x18\x02 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level\x1an\n\tUnpersist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x1f\n\x08\x62locking\x18\x02 \x01(\x08H\x00R\x08\x62locking\x88\x01\x01\x42\x0b\n\t_blocking\x1a\x46\n\x0fGetStorageLevel\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a,\n\tJsonToDDL\x12\x1f\n\x0bjson_string\x18\x01 \x01(\tR\njsonStringB\t\n\x07\x61nalyzeB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xca\x0e\n\x13\x41nalyzePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12\x43\n\x06schema\x18\x02 \x01(\x0b\x32).spark.connect.AnalyzePlanResponse.SchemaH\x00R\x06schema\x12\x46\n\x07\x65xplain\x18\x03 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.ExplainH\x00R\x07\x65xplain\x12P\n\x0btree_string\x18\x04 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.TreeStringH\x00R\ntreeString\x12G\n\x08is_local\x18\x05 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.IsLocalH\x00R\x07isLocal\x12S\n\x0cis_streaming\x18\x06 \x01(\x0b\x32..spark.connect.AnalyzePlanResponse.IsStreamingH\x00R\x0bisStreaming\x12P\n\x0binput_files\x18\x07 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.InputFilesH\x00R\ninputFiles\x12V\n\rspark_version\x18\x08 \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SparkVersionH\x00R\x0csparkVersion\x12J\n\tddl_parse\x18\t \x01(\x0b\x32+.spark.connect.AnalyzePlanResponse.DDLParseH\x00R\x08\x64\x64lParse\x12Y\n\x0esame_semantics\x18\n \x01(\x0b\x32\x30.spark.connect.AnalyzePlanResponse.SameSemanticsH\x00R\rsameSemantics\x12V\n\rsemantic_hash\x18\x0b \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SemanticHashH\x00R\x0csemanticHash\x12\x46\n\x07persist\x18\x0c \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.PersistH\x00R\x07persist\x12L\n\tunpersist\x18\r \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.UnpersistH\x00R\tunpersist\x12`\n\x11get_storage_level\x18\x0e \x01(\x0b\x32\x32.spark.connect.AnalyzePlanResponse.GetStorageLevelH\x00R\x0fgetStorageLevel\x12N\n\x0bjson_to_ddl\x18\x10 \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.JsonToDDLH\x00R\tjsonToDdl\x1a\x39\n\x06Schema\x12/\n\x06schema\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1a\x30\n\x07\x45xplain\x12%\n\x0e\x65xplain_string\x18\x01 \x01(\tR\rexplainString\x1a-\n\nTreeString\x12\x1f\n\x0btree_string\x18\x01 \x01(\tR\ntreeString\x1a$\n\x07IsLocal\x12\x19\n\x08is_local\x18\x01 \x01(\x08R\x07isLocal\x1a\x30\n\x0bIsStreaming\x12!\n\x0cis_streaming\x18\x01 \x01(\x08R\x0bisStreaming\x1a"\n\nInputFiles\x12\x14\n\x05\x66iles\x18\x01 \x03(\tR\x05\x66iles\x1a(\n\x0cSparkVersion\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\x1a;\n\x08\x44\x44LParse\x12/\n\x06parsed\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06parsed\x1a\'\n\rSameSemantics\x12\x16\n\x06result\x18\x01 \x01(\x08R\x06result\x1a&\n\x0cSemanticHash\x12\x16\n\x06result\x18\x01 \x01(\x05R\x06result\x1a\t\n\x07Persist\x1a\x0b\n\tUnpersist\x1aS\n\x0fGetStorageLevel\x12@\n\rstorage_level\x18\x01 \x01(\x0b\x32\x1b.spark.connect.StorageLevelR\x0cstorageLevel\x1a*\n\tJsonToDDL\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlStringB\x08\n\x06result"\x83\x06\n\x12\x45xecutePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12&\n\x0coperation_id\x18\x06 \x01(\tH\x01R\x0boperationId\x88\x01\x01\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12X\n\x0frequest_options\x18\x05 \x03(\x0b\x32/.spark.connect.ExecutePlanRequest.RequestOptionR\x0erequestOptions\x12\x12\n\x04tags\x18\x07 \x03(\tR\x04tags\x1a\x85\x02\n\rRequestOption\x12K\n\x10reattach_options\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ReattachOptionsH\x00R\x0freattachOptions\x12^\n\x17result_chunking_options\x18\x02 \x01(\x0b\x32$.spark.connect.ResultChunkingOptionsH\x00R\x15resultChunkingOptions\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x10\n\x0erequest_optionB)\n\'_client_observed_server_side_session_idB\x0f\n\r_operation_idB\x0e\n\x0c_client_type"\x87\x1c\n\x13\x45xecutePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12!\n\x0coperation_id\x18\x0c \x01(\tR\x0boperationId\x12\x1f\n\x0bresponse_id\x18\r \x01(\tR\nresponseId\x12P\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32-.spark.connect.ExecutePlanResponse.ArrowBatchH\x00R\narrowBatch\x12\x63\n\x12sql_command_result\x18\x05 \x01(\x0b\x32\x33.spark.connect.ExecutePlanResponse.SqlCommandResultH\x00R\x10sqlCommandResult\x12~\n#write_stream_operation_start_result\x18\x08 \x01(\x0b\x32..spark.connect.WriteStreamOperationStartResultH\x00R\x1fwriteStreamOperationStartResult\x12q\n\x1estreaming_query_command_result\x18\t \x01(\x0b\x32*.spark.connect.StreamingQueryCommandResultH\x00R\x1bstreamingQueryCommandResult\x12k\n\x1cget_resources_command_result\x18\n \x01(\x0b\x32(.spark.connect.GetResourcesCommandResultH\x00R\x19getResourcesCommandResult\x12\x87\x01\n&streaming_query_manager_command_result\x18\x0b \x01(\x0b\x32\x31.spark.connect.StreamingQueryManagerCommandResultH\x00R"streamingQueryManagerCommandResult\x12\x87\x01\n&streaming_query_listener_events_result\x18\x10 \x01(\x0b\x32\x31.spark.connect.StreamingQueryListenerEventsResultH\x00R"streamingQueryListenerEventsResult\x12\\\n\x0fresult_complete\x18\x0e \x01(\x0b\x32\x31.spark.connect.ExecutePlanResponse.ResultCompleteH\x00R\x0eresultComplete\x12\x87\x01\n&create_resource_profile_command_result\x18\x11 \x01(\x0b\x32\x31.spark.connect.CreateResourceProfileCommandResultH\x00R"createResourceProfileCommandResult\x12\x65\n\x12\x65xecution_progress\x18\x12 \x01(\x0b\x32\x34.spark.connect.ExecutePlanResponse.ExecutionProgressH\x00R\x11\x65xecutionProgress\x12\x64\n\x19\x63heckpoint_command_result\x18\x13 \x01(\x0b\x32&.spark.connect.CheckpointCommandResultH\x00R\x17\x63heckpointCommandResult\x12L\n\x11ml_command_result\x18\x14 \x01(\x0b\x32\x1e.spark.connect.MlCommandResultH\x00R\x0fmlCommandResult\x12X\n\x15pipeline_event_result\x18\x15 \x01(\x0b\x32".spark.connect.PipelineEventResultH\x00R\x13pipelineEventResult\x12^\n\x17pipeline_command_result\x18\x16 \x01(\x0b\x32$.spark.connect.PipelineCommandResultH\x00R\x15pipelineCommandResult\x12\x8d\x01\n(pipeline_query_function_execution_signal\x18\x17 \x01(\x0b\x32\x33.spark.connect.PipelineQueryFunctionExecutionSignalH\x00R$pipelineQueryFunctionExecutionSignal\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x44\n\x07metrics\x18\x04 \x01(\x0b\x32*.spark.connect.ExecutePlanResponse.MetricsR\x07metrics\x12]\n\x10observed_metrics\x18\x06 \x03(\x0b\x32\x32.spark.connect.ExecutePlanResponse.ObservedMetricsR\x0fobservedMetrics\x12/\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1aG\n\x10SqlCommandResult\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a\xf8\x01\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x12&\n\x0cstart_offset\x18\x03 \x01(\x03H\x00R\x0bstartOffset\x88\x01\x01\x12$\n\x0b\x63hunk_index\x18\x04 \x01(\x03H\x01R\nchunkIndex\x88\x01\x01\x12\x32\n\x13num_chunks_in_batch\x18\x05 \x01(\x03H\x02R\x10numChunksInBatch\x88\x01\x01\x42\x0f\n\r_start_offsetB\x0e\n\x0c_chunk_indexB\x16\n\x14_num_chunks_in_batch\x1a\x85\x04\n\x07Metrics\x12Q\n\x07metrics\x18\x01 \x03(\x0b\x32\x37.spark.connect.ExecutePlanResponse.Metrics.MetricObjectR\x07metrics\x1a\xcc\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12z\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32M.spark.connect.ExecutePlanResponse.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1a{\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ExecutePlanResponse.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricType\x1a\x93\x02\n\x0fObservedMetrics\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x12\x12\n\x04keys\x18\x03 \x03(\tR\x04keys\x12\x17\n\x07plan_id\x18\x04 \x01(\x03R\x06planId\x12)\n\x0eroot_error_idx\x18\x05 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x06 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrorsB\x11\n\x0f_root_error_idx\x1a\x10\n\x0eResultComplete\x1a\xcd\x02\n\x11\x45xecutionProgress\x12V\n\x06stages\x18\x01 \x03(\x0b\x32>.spark.connect.ExecutePlanResponse.ExecutionProgress.StageInfoR\x06stages\x12,\n\x12num_inflight_tasks\x18\x02 \x01(\x03R\x10numInflightTasks\x1a\xb1\x01\n\tStageInfo\x12\x19\n\x08stage_id\x18\x01 \x01(\x03R\x07stageId\x12\x1b\n\tnum_tasks\x18\x02 \x01(\x03R\x08numTasks\x12.\n\x13num_completed_tasks\x18\x03 \x01(\x03R\x11numCompletedTasks\x12(\n\x10input_bytes_read\x18\x04 \x01(\x03R\x0einputBytesRead\x12\x12\n\x04\x64one\x18\x05 \x01(\x08R\x04\x64oneB\x0f\n\rresponse_type"A\n\x08KeyValue\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x19\n\x05value\x18\x02 \x01(\tH\x00R\x05value\x88\x01\x01\x42\x08\n\x06_value"\xaf\t\n\rConfigRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x44\n\toperation\x18\x03 \x01(\x0b\x32&.spark.connect.ConfigRequest.OperationR\toperation\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x1a\xf2\x03\n\tOperation\x12\x34\n\x03set\x18\x01 \x01(\x0b\x32 .spark.connect.ConfigRequest.SetH\x00R\x03set\x12\x34\n\x03get\x18\x02 \x01(\x0b\x32 .spark.connect.ConfigRequest.GetH\x00R\x03get\x12W\n\x10get_with_default\x18\x03 \x01(\x0b\x32+.spark.connect.ConfigRequest.GetWithDefaultH\x00R\x0egetWithDefault\x12G\n\nget_option\x18\x04 \x01(\x0b\x32&.spark.connect.ConfigRequest.GetOptionH\x00R\tgetOption\x12>\n\x07get_all\x18\x05 \x01(\x0b\x32#.spark.connect.ConfigRequest.GetAllH\x00R\x06getAll\x12:\n\x05unset\x18\x06 \x01(\x0b\x32".spark.connect.ConfigRequest.UnsetH\x00R\x05unset\x12P\n\ris_modifiable\x18\x07 \x01(\x0b\x32).spark.connect.ConfigRequest.IsModifiableH\x00R\x0cisModifiableB\t\n\x07op_type\x1a\\\n\x03Set\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1b\n\x06silent\x18\x02 \x01(\x08H\x00R\x06silent\x88\x01\x01\x42\t\n\x07_silent\x1a\x19\n\x03Get\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a?\n\x0eGetWithDefault\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x1f\n\tGetOption\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a\x30\n\x06GetAll\x12\x1b\n\x06prefix\x18\x01 \x01(\tH\x00R\x06prefix\x88\x01\x01\x42\t\n\x07_prefix\x1a\x1b\n\x05Unset\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a"\n\x0cIsModifiable\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keysB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xaf\x01\n\x0e\x43onfigResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x04 \x01(\tR\x13serverSideSessionId\x12-\n\x05pairs\x18\x02 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1a\n\x08warnings\x18\x03 \x03(\tR\x08warnings"\xea\x07\n\x13\x41\x64\x64\x41rtifactsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12$\n\x0b\x63lient_type\x18\x06 \x01(\tH\x02R\nclientType\x88\x01\x01\x12@\n\x05\x62\x61tch\x18\x03 \x01(\x0b\x32(.spark.connect.AddArtifactsRequest.BatchH\x00R\x05\x62\x61tch\x12Z\n\x0b\x62\x65gin_chunk\x18\x04 \x01(\x0b\x32\x37.spark.connect.AddArtifactsRequest.BeginChunkedArtifactH\x00R\nbeginChunk\x12H\n\x05\x63hunk\x18\x05 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkH\x00R\x05\x63hunk\x1a\x35\n\rArtifactChunk\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12\x10\n\x03\x63rc\x18\x02 \x01(\x03R\x03\x63rc\x1ao\n\x13SingleChunkArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x04\x64\x61ta\x1a]\n\x05\x42\x61tch\x12T\n\tartifacts\x18\x01 \x03(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactR\tartifacts\x1a\xc1\x01\n\x14\x42\x65ginChunkedArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0btotal_bytes\x18\x02 \x01(\x03R\ntotalBytes\x12\x1d\n\nnum_chunks\x18\x03 \x01(\x03R\tnumChunks\x12U\n\rinitial_chunk\x18\x04 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x0cinitialChunkB\t\n\x07payloadB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x02\n\x14\x41\x64\x64\x41rtifactsResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\tartifacts\x18\x01 \x03(\x0b\x32\x33.spark.connect.AddArtifactsResponse.ArtifactSummaryR\tartifacts\x1aQ\n\x0f\x41rtifactSummary\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x11is_crc_successful\x18\x02 \x01(\x08R\x0fisCrcSuccessful"\xc6\x02\n\x17\x41rtifactStatusesRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12\x14\n\x05names\x18\x04 \x03(\tR\x05namesB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xe0\x02\n\x18\x41rtifactStatusesResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\x08statuses\x18\x01 \x03(\x0b\x32\x35.spark.connect.ArtifactStatusesResponse.StatusesEntryR\x08statuses\x1as\n\rStatusesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ArtifactStatusesResponse.ArtifactStatusR\x05value:\x02\x38\x01\x1a(\n\x0e\x41rtifactStatus\x12\x16\n\x06\x65xists\x18\x01 \x01(\x08R\x06\x65xists"\xdb\x04\n\x10InterruptRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12T\n\x0einterrupt_type\x18\x04 \x01(\x0e\x32-.spark.connect.InterruptRequest.InterruptTypeR\rinterruptType\x12%\n\roperation_tag\x18\x05 \x01(\tH\x00R\x0coperationTag\x12#\n\x0coperation_id\x18\x06 \x01(\tH\x00R\x0boperationId"\x80\x01\n\rInterruptType\x12\x1e\n\x1aINTERRUPT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12INTERRUPT_TYPE_ALL\x10\x01\x12\x16\n\x12INTERRUPT_TYPE_TAG\x10\x02\x12\x1f\n\x1bINTERRUPT_TYPE_OPERATION_ID\x10\x03\x42\x0b\n\tinterruptB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x01\n\x11InterruptResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\'\n\x0finterrupted_ids\x18\x02 \x03(\tR\x0einterruptedIds"5\n\x0fReattachOptions\x12"\n\x0creattachable\x18\x01 \x01(\x08R\x0creattachable"\xb5\x01\n\x15ResultChunkingOptions\x12;\n\x1a\x61llow_arrow_batch_chunking\x18\x01 \x01(\x08R\x17\x61llowArrowBatchChunking\x12@\n\x1apreferred_arrow_chunk_size\x18\x02 \x01(\x03H\x00R\x17preferredArrowChunkSize\x88\x01\x01\x42\x1d\n\x1b_preferred_arrow_chunk_size"\x96\x03\n\x16ReattachExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x06 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x12-\n\x10last_response_id\x18\x05 \x01(\tH\x02R\x0elastResponseId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x13\n\x11_last_response_id"\xc9\x04\n\x15ReleaseExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12R\n\x0brelease_all\x18\x05 \x01(\x0b\x32/.spark.connect.ReleaseExecuteRequest.ReleaseAllH\x00R\nreleaseAll\x12X\n\rrelease_until\x18\x06 \x01(\x0b\x32\x31.spark.connect.ReleaseExecuteRequest.ReleaseUntilH\x00R\x0creleaseUntil\x1a\x0c\n\nReleaseAll\x1a/\n\x0cReleaseUntil\x12\x1f\n\x0bresponse_id\x18\x01 \x01(\tR\nresponseIdB\t\n\x07releaseB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x01\n\x16ReleaseExecuteResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12&\n\x0coperation_id\x18\x02 \x01(\tH\x00R\x0boperationId\x88\x01\x01\x42\x0f\n\r_operation_id"\xd4\x01\n\x15ReleaseSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12\'\n\x0f\x61llow_reconnect\x18\x04 \x01(\x08R\x0e\x61llowReconnectB\x0e\n\x0c_client_type"l\n\x16ReleaseSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId"\xcc\x02\n\x18\x46\x65tchErrorDetailsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x19\n\x08\x65rror_id\x18\x03 \x01(\tR\x07\x65rrorId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xd9\x0f\n\x19\x46\x65tchErrorDetailsResponse\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\x1d\n\nsession_id\x18\x04 \x01(\tR\tsessionId\x12)\n\x0eroot_error_idx\x18\x01 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x02 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrors\x1a\xae\x01\n\x11StackTraceElement\x12\'\n\x0f\x64\x65\x63laring_class\x18\x01 \x01(\tR\x0e\x64\x65\x63laringClass\x12\x1f\n\x0bmethod_name\x18\x02 \x01(\tR\nmethodName\x12 \n\tfile_name\x18\x03 \x01(\tH\x00R\x08\x66ileName\x88\x01\x01\x12\x1f\n\x0bline_number\x18\x04 \x01(\x05R\nlineNumberB\x0c\n\n_file_name\x1a\xf0\x02\n\x0cQueryContext\x12\x64\n\x0c\x63ontext_type\x18\n \x01(\x0e\x32\x41.spark.connect.FetchErrorDetailsResponse.QueryContext.ContextTypeR\x0b\x63ontextType\x12\x1f\n\x0bobject_type\x18\x01 \x01(\tR\nobjectType\x12\x1f\n\x0bobject_name\x18\x02 \x01(\tR\nobjectName\x12\x1f\n\x0bstart_index\x18\x03 \x01(\x05R\nstartIndex\x12\x1d\n\nstop_index\x18\x04 \x01(\x05R\tstopIndex\x12\x1a\n\x08\x66ragment\x18\x05 \x01(\tR\x08\x66ragment\x12\x1b\n\tcall_site\x18\x06 \x01(\tR\x08\x63\x61llSite\x12\x18\n\x07summary\x18\x07 \x01(\tR\x07summary"%\n\x0b\x43ontextType\x12\x07\n\x03SQL\x10\x00\x12\r\n\tDATAFRAME\x10\x01\x1a\xa6\x04\n\x0eSparkThrowable\x12$\n\x0b\x65rror_class\x18\x01 \x01(\tH\x00R\nerrorClass\x88\x01\x01\x12}\n\x12message_parameters\x18\x02 \x03(\x0b\x32N.spark.connect.FetchErrorDetailsResponse.SparkThrowable.MessageParametersEntryR\x11messageParameters\x12\\\n\x0equery_contexts\x18\x03 \x03(\x0b\x32\x35.spark.connect.FetchErrorDetailsResponse.QueryContextR\rqueryContexts\x12 \n\tsql_state\x18\x04 \x01(\tH\x01R\x08sqlState\x88\x01\x01\x12r\n\x14\x62reaking_change_info\x18\x05 \x01(\x0b\x32;.spark.connect.FetchErrorDetailsResponse.BreakingChangeInfoH\x02R\x12\x62reakingChangeInfo\x88\x01\x01\x1a\x44\n\x16MessageParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_error_classB\x0c\n\n_sql_stateB\x17\n\x15_breaking_change_info\x1a\xfa\x01\n\x12\x42reakingChangeInfo\x12+\n\x11migration_message\x18\x01 \x03(\tR\x10migrationMessage\x12k\n\x11mitigation_config\x18\x02 \x01(\x0b\x32\x39.spark.connect.FetchErrorDetailsResponse.MitigationConfigH\x00R\x10mitigationConfig\x88\x01\x01\x12$\n\x0bneeds_audit\x18\x03 \x01(\x08H\x01R\nneedsAudit\x88\x01\x01\x42\x14\n\x12_mitigation_configB\x0e\n\x0c_needs_audit\x1a:\n\x10MitigationConfig\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\x1a\xdb\x02\n\x05\x45rror\x12\x30\n\x14\x65rror_type_hierarchy\x18\x01 \x03(\tR\x12\x65rrorTypeHierarchy\x12\x18\n\x07message\x18\x02 \x01(\tR\x07message\x12[\n\x0bstack_trace\x18\x03 \x03(\x0b\x32:.spark.connect.FetchErrorDetailsResponse.StackTraceElementR\nstackTrace\x12 \n\tcause_idx\x18\x04 \x01(\x05H\x00R\x08\x63\x61useIdx\x88\x01\x01\x12\x65\n\x0fspark_throwable\x18\x05 \x01(\x0b\x32\x37.spark.connect.FetchErrorDetailsResponse.SparkThrowableH\x01R\x0esparkThrowable\x88\x01\x01\x42\x0c\n\n_cause_idxB\x12\n\x10_spark_throwableB\x11\n\x0f_root_error_idx"Z\n\x17\x43heckpointCommandResult\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation"\xea\x02\n\x13\x43loneSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12)\n\x0enew_session_id\x18\x04 \x01(\tH\x02R\x0cnewSessionId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x11\n\x0f_new_session_id"\xcc\x01\n\x14\x43loneSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12$\n\x0enew_session_id\x18\x03 \x01(\tR\x0cnewSessionId\x12:\n\x1anew_server_side_session_id\x18\x04 \x01(\tR\x16newServerSideSessionId"\xd3\x04\n\x10GetStatusRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12V\n&client_observed_server_side_session_id\x18\x04 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12\x66\n\x10operation_status\x18\x05 \x01(\x0b\x32\x36.spark.connect.GetStatusRequest.OperationStatusRequestH\x02R\x0foperationStatus\x88\x01\x01\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1at\n\x16OperationStatusRequest\x12#\n\roperation_ids\x18\x01 \x03(\tR\x0coperationIds\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensionsB\x0e\n\x0c_client_typeB)\n\'_client_observed_server_side_session_idB\x13\n\x11_operation_status"\xad\x05\n\x11GetStatusResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12_\n\x12operation_statuses\x18\x03 \x03(\x0b\x32\x30.spark.connect.GetStatusResponse.OperationStatusR\x11operationStatuses\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1a\xab\x03\n\x0fOperationStatus\x12!\n\x0coperation_id\x18\x01 \x01(\tR\x0boperationId\x12U\n\x05state\x18\x02 \x01(\x0e\x32?.spark.connect.GetStatusResponse.OperationStatus.OperationStateR\x05state\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xe6\x01\n\x0eOperationState\x12\x1f\n\x1bOPERATION_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17OPERATION_STATE_UNKNOWN\x10\x01\x12\x1b\n\x17OPERATION_STATE_RUNNING\x10\x02\x12\x1f\n\x1bOPERATION_STATE_TERMINATING\x10\x03\x12\x1d\n\x19OPERATION_STATE_SUCCEEDED\x10\x04\x12\x1a\n\x16OPERATION_STATE_FAILED\x10\x05\x12\x1d\n\x19OPERATION_STATE_CANCELLED\x10\x06*Q\n\x10\x43ompressionCodec\x12!\n\x1d\x43OMPRESSION_CODEC_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x43OMPRESSION_CODEC_ZSTD\x10\x01\x32\xdf\x08\n\x13SparkConnectService\x12X\n\x0b\x45xecutePlan\x12!.spark.connect.ExecutePlanRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12V\n\x0b\x41nalyzePlan\x12!.spark.connect.AnalyzePlanRequest\x1a".spark.connect.AnalyzePlanResponse"\x00\x12G\n\x06\x43onfig\x12\x1c.spark.connect.ConfigRequest\x1a\x1d.spark.connect.ConfigResponse"\x00\x12[\n\x0c\x41\x64\x64\x41rtifacts\x12".spark.connect.AddArtifactsRequest\x1a#.spark.connect.AddArtifactsResponse"\x00(\x01\x12\x63\n\x0e\x41rtifactStatus\x12&.spark.connect.ArtifactStatusesRequest\x1a\'.spark.connect.ArtifactStatusesResponse"\x00\x12P\n\tInterrupt\x12\x1f.spark.connect.InterruptRequest\x1a .spark.connect.InterruptResponse"\x00\x12`\n\x0fReattachExecute\x12%.spark.connect.ReattachExecuteRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12_\n\x0eReleaseExecute\x12$.spark.connect.ReleaseExecuteRequest\x1a%.spark.connect.ReleaseExecuteResponse"\x00\x12_\n\x0eReleaseSession\x12$.spark.connect.ReleaseSessionRequest\x1a%.spark.connect.ReleaseSessionResponse"\x00\x12h\n\x11\x46\x65tchErrorDetails\x12\'.spark.connect.FetchErrorDetailsRequest\x1a(.spark.connect.FetchErrorDetailsResponse"\x00\x12Y\n\x0c\x43loneSession\x12".spark.connect.CloneSessionRequest\x1a#.spark.connect.CloneSessionResponse"\x00\x12P\n\tGetStatus\x12\x1f.spark.connect.GetStatusRequest\x1a .spark.connect.GetStatusResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xe3\x03\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommand\x12\\\n\x14\x63ompressed_operation\x18\x03 \x01(\x0b\x32\'.spark.connect.Plan.CompressedOperationH\x00R\x13\x63ompressedOperation\x1a\x8e\x02\n\x13\x43ompressedOperation\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12G\n\x07op_type\x18\x02 \x01(\x0e\x32..spark.connect.Plan.CompressedOperation.OpTypeR\x06opType\x12L\n\x11\x63ompression_codec\x18\x03 \x01(\x0e\x32\x1f.spark.connect.CompressionCodecR\x10\x63ompressionCodec"L\n\x06OpType\x12\x17\n\x13OP_TYPE_UNSPECIFIED\x10\x00\x12\x14\n\x10OP_TYPE_RELATION\x10\x01\x12\x13\n\x0fOP_TYPE_COMMAND\x10\x02\x42\t\n\x07op_type"z\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xf5\x14\n\x12\x41nalyzePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x11 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12\x42\n\x06schema\x18\x04 \x01(\x0b\x32(.spark.connect.AnalyzePlanRequest.SchemaH\x00R\x06schema\x12\x45\n\x07\x65xplain\x18\x05 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.ExplainH\x00R\x07\x65xplain\x12O\n\x0btree_string\x18\x06 \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.TreeStringH\x00R\ntreeString\x12\x46\n\x08is_local\x18\x07 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.IsLocalH\x00R\x07isLocal\x12R\n\x0cis_streaming\x18\x08 \x01(\x0b\x32-.spark.connect.AnalyzePlanRequest.IsStreamingH\x00R\x0bisStreaming\x12O\n\x0binput_files\x18\t \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.InputFilesH\x00R\ninputFiles\x12U\n\rspark_version\x18\n \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SparkVersionH\x00R\x0csparkVersion\x12I\n\tddl_parse\x18\x0b \x01(\x0b\x32*.spark.connect.AnalyzePlanRequest.DDLParseH\x00R\x08\x64\x64lParse\x12X\n\x0esame_semantics\x18\x0c \x01(\x0b\x32/.spark.connect.AnalyzePlanRequest.SameSemanticsH\x00R\rsameSemantics\x12U\n\rsemantic_hash\x18\r \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SemanticHashH\x00R\x0csemanticHash\x12\x45\n\x07persist\x18\x0e \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.PersistH\x00R\x07persist\x12K\n\tunpersist\x18\x0f \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.UnpersistH\x00R\tunpersist\x12_\n\x11get_storage_level\x18\x10 \x01(\x0b\x32\x31.spark.connect.AnalyzePlanRequest.GetStorageLevelH\x00R\x0fgetStorageLevel\x12M\n\x0bjson_to_ddl\x18\x12 \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.JsonToDDLH\x00R\tjsonToDdl\x1a\x31\n\x06Schema\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\xbb\x02\n\x07\x45xplain\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12X\n\x0c\x65xplain_mode\x18\x02 \x01(\x0e\x32\x35.spark.connect.AnalyzePlanRequest.Explain.ExplainModeR\x0b\x65xplainMode"\xac\x01\n\x0b\x45xplainMode\x12\x1c\n\x18\x45XPLAIN_MODE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XPLAIN_MODE_SIMPLE\x10\x01\x12\x19\n\x15\x45XPLAIN_MODE_EXTENDED\x10\x02\x12\x18\n\x14\x45XPLAIN_MODE_CODEGEN\x10\x03\x12\x15\n\x11\x45XPLAIN_MODE_COST\x10\x04\x12\x1a\n\x16\x45XPLAIN_MODE_FORMATTED\x10\x05\x1aZ\n\nTreeString\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12\x19\n\x05level\x18\x02 \x01(\x05H\x00R\x05level\x88\x01\x01\x42\x08\n\x06_level\x1a\x32\n\x07IsLocal\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x36\n\x0bIsStreaming\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x35\n\nInputFiles\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x0e\n\x0cSparkVersion\x1a)\n\x08\x44\x44LParse\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlString\x1ay\n\rSameSemantics\x12\x34\n\x0btarget_plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\ntargetPlan\x12\x32\n\nother_plan\x18\x02 \x01(\x0b\x32\x13.spark.connect.PlanR\totherPlan\x1a\x37\n\x0cSemanticHash\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x97\x01\n\x07Persist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x45\n\rstorage_level\x18\x02 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level\x1an\n\tUnpersist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x1f\n\x08\x62locking\x18\x02 \x01(\x08H\x00R\x08\x62locking\x88\x01\x01\x42\x0b\n\t_blocking\x1a\x46\n\x0fGetStorageLevel\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a,\n\tJsonToDDL\x12\x1f\n\x0bjson_string\x18\x01 \x01(\tR\njsonStringB\t\n\x07\x61nalyzeB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xca\x0e\n\x13\x41nalyzePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12\x43\n\x06schema\x18\x02 \x01(\x0b\x32).spark.connect.AnalyzePlanResponse.SchemaH\x00R\x06schema\x12\x46\n\x07\x65xplain\x18\x03 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.ExplainH\x00R\x07\x65xplain\x12P\n\x0btree_string\x18\x04 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.TreeStringH\x00R\ntreeString\x12G\n\x08is_local\x18\x05 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.IsLocalH\x00R\x07isLocal\x12S\n\x0cis_streaming\x18\x06 \x01(\x0b\x32..spark.connect.AnalyzePlanResponse.IsStreamingH\x00R\x0bisStreaming\x12P\n\x0binput_files\x18\x07 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.InputFilesH\x00R\ninputFiles\x12V\n\rspark_version\x18\x08 \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SparkVersionH\x00R\x0csparkVersion\x12J\n\tddl_parse\x18\t \x01(\x0b\x32+.spark.connect.AnalyzePlanResponse.DDLParseH\x00R\x08\x64\x64lParse\x12Y\n\x0esame_semantics\x18\n \x01(\x0b\x32\x30.spark.connect.AnalyzePlanResponse.SameSemanticsH\x00R\rsameSemantics\x12V\n\rsemantic_hash\x18\x0b \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SemanticHashH\x00R\x0csemanticHash\x12\x46\n\x07persist\x18\x0c \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.PersistH\x00R\x07persist\x12L\n\tunpersist\x18\r \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.UnpersistH\x00R\tunpersist\x12`\n\x11get_storage_level\x18\x0e \x01(\x0b\x32\x32.spark.connect.AnalyzePlanResponse.GetStorageLevelH\x00R\x0fgetStorageLevel\x12N\n\x0bjson_to_ddl\x18\x10 \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.JsonToDDLH\x00R\tjsonToDdl\x1a\x39\n\x06Schema\x12/\n\x06schema\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1a\x30\n\x07\x45xplain\x12%\n\x0e\x65xplain_string\x18\x01 \x01(\tR\rexplainString\x1a-\n\nTreeString\x12\x1f\n\x0btree_string\x18\x01 \x01(\tR\ntreeString\x1a$\n\x07IsLocal\x12\x19\n\x08is_local\x18\x01 \x01(\x08R\x07isLocal\x1a\x30\n\x0bIsStreaming\x12!\n\x0cis_streaming\x18\x01 \x01(\x08R\x0bisStreaming\x1a"\n\nInputFiles\x12\x14\n\x05\x66iles\x18\x01 \x03(\tR\x05\x66iles\x1a(\n\x0cSparkVersion\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\x1a;\n\x08\x44\x44LParse\x12/\n\x06parsed\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06parsed\x1a\'\n\rSameSemantics\x12\x16\n\x06result\x18\x01 \x01(\x08R\x06result\x1a&\n\x0cSemanticHash\x12\x16\n\x06result\x18\x01 \x01(\x05R\x06result\x1a\t\n\x07Persist\x1a\x0b\n\tUnpersist\x1aS\n\x0fGetStorageLevel\x12@\n\rstorage_level\x18\x01 \x01(\x0b\x32\x1b.spark.connect.StorageLevelR\x0cstorageLevel\x1a*\n\tJsonToDDL\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlStringB\x08\n\x06result"\x83\x06\n\x12\x45xecutePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12&\n\x0coperation_id\x18\x06 \x01(\tH\x01R\x0boperationId\x88\x01\x01\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12X\n\x0frequest_options\x18\x05 \x03(\x0b\x32/.spark.connect.ExecutePlanRequest.RequestOptionR\x0erequestOptions\x12\x12\n\x04tags\x18\x07 \x03(\tR\x04tags\x1a\x85\x02\n\rRequestOption\x12K\n\x10reattach_options\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ReattachOptionsH\x00R\x0freattachOptions\x12^\n\x17result_chunking_options\x18\x02 \x01(\x0b\x32$.spark.connect.ResultChunkingOptionsH\x00R\x15resultChunkingOptions\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x10\n\x0erequest_optionB)\n\'_client_observed_server_side_session_idB\x0f\n\r_operation_idB\x0e\n\x0c_client_type"\xe7\x1c\n\x13\x45xecutePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12!\n\x0coperation_id\x18\x0c \x01(\tR\x0boperationId\x12\x1f\n\x0bresponse_id\x18\r \x01(\tR\nresponseId\x12P\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32-.spark.connect.ExecutePlanResponse.ArrowBatchH\x00R\narrowBatch\x12\x63\n\x12sql_command_result\x18\x05 \x01(\x0b\x32\x33.spark.connect.ExecutePlanResponse.SqlCommandResultH\x00R\x10sqlCommandResult\x12~\n#write_stream_operation_start_result\x18\x08 \x01(\x0b\x32..spark.connect.WriteStreamOperationStartResultH\x00R\x1fwriteStreamOperationStartResult\x12q\n\x1estreaming_query_command_result\x18\t \x01(\x0b\x32*.spark.connect.StreamingQueryCommandResultH\x00R\x1bstreamingQueryCommandResult\x12k\n\x1cget_resources_command_result\x18\n \x01(\x0b\x32(.spark.connect.GetResourcesCommandResultH\x00R\x19getResourcesCommandResult\x12\x87\x01\n&streaming_query_manager_command_result\x18\x0b \x01(\x0b\x32\x31.spark.connect.StreamingQueryManagerCommandResultH\x00R"streamingQueryManagerCommandResult\x12\x87\x01\n&streaming_query_listener_events_result\x18\x10 \x01(\x0b\x32\x31.spark.connect.StreamingQueryListenerEventsResultH\x00R"streamingQueryListenerEventsResult\x12\\\n\x0fresult_complete\x18\x0e \x01(\x0b\x32\x31.spark.connect.ExecutePlanResponse.ResultCompleteH\x00R\x0eresultComplete\x12\x87\x01\n&create_resource_profile_command_result\x18\x11 \x01(\x0b\x32\x31.spark.connect.CreateResourceProfileCommandResultH\x00R"createResourceProfileCommandResult\x12\x65\n\x12\x65xecution_progress\x18\x12 \x01(\x0b\x32\x34.spark.connect.ExecutePlanResponse.ExecutionProgressH\x00R\x11\x65xecutionProgress\x12\x64\n\x19\x63heckpoint_command_result\x18\x13 \x01(\x0b\x32&.spark.connect.CheckpointCommandResultH\x00R\x17\x63heckpointCommandResult\x12L\n\x11ml_command_result\x18\x14 \x01(\x0b\x32\x1e.spark.connect.MlCommandResultH\x00R\x0fmlCommandResult\x12X\n\x15pipeline_event_result\x18\x15 \x01(\x0b\x32".spark.connect.PipelineEventResultH\x00R\x13pipelineEventResult\x12^\n\x17pipeline_command_result\x18\x16 \x01(\x0b\x32$.spark.connect.PipelineCommandResultH\x00R\x15pipelineCommandResult\x12\x8d\x01\n(pipeline_query_function_execution_signal\x18\x17 \x01(\x0b\x32\x33.spark.connect.PipelineQueryFunctionExecutionSignalH\x00R$pipelineQueryFunctionExecutionSignal\x12^\n\x17\x63reate_broadcast_result\x18\x18 \x01(\x0b\x32$.spark.connect.CreateBroadcastResultH\x00R\x15\x63reateBroadcastResult\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x44\n\x07metrics\x18\x04 \x01(\x0b\x32*.spark.connect.ExecutePlanResponse.MetricsR\x07metrics\x12]\n\x10observed_metrics\x18\x06 \x03(\x0b\x32\x32.spark.connect.ExecutePlanResponse.ObservedMetricsR\x0fobservedMetrics\x12/\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1aG\n\x10SqlCommandResult\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a\xf8\x01\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x12&\n\x0cstart_offset\x18\x03 \x01(\x03H\x00R\x0bstartOffset\x88\x01\x01\x12$\n\x0b\x63hunk_index\x18\x04 \x01(\x03H\x01R\nchunkIndex\x88\x01\x01\x12\x32\n\x13num_chunks_in_batch\x18\x05 \x01(\x03H\x02R\x10numChunksInBatch\x88\x01\x01\x42\x0f\n\r_start_offsetB\x0e\n\x0c_chunk_indexB\x16\n\x14_num_chunks_in_batch\x1a\x85\x04\n\x07Metrics\x12Q\n\x07metrics\x18\x01 \x03(\x0b\x32\x37.spark.connect.ExecutePlanResponse.Metrics.MetricObjectR\x07metrics\x1a\xcc\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12z\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32M.spark.connect.ExecutePlanResponse.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1a{\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ExecutePlanResponse.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricType\x1a\x93\x02\n\x0fObservedMetrics\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x12\x12\n\x04keys\x18\x03 \x03(\tR\x04keys\x12\x17\n\x07plan_id\x18\x04 \x01(\x03R\x06planId\x12)\n\x0eroot_error_idx\x18\x05 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x06 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrorsB\x11\n\x0f_root_error_idx\x1a\x10\n\x0eResultComplete\x1a\xcd\x02\n\x11\x45xecutionProgress\x12V\n\x06stages\x18\x01 \x03(\x0b\x32>.spark.connect.ExecutePlanResponse.ExecutionProgress.StageInfoR\x06stages\x12,\n\x12num_inflight_tasks\x18\x02 \x01(\x03R\x10numInflightTasks\x1a\xb1\x01\n\tStageInfo\x12\x19\n\x08stage_id\x18\x01 \x01(\x03R\x07stageId\x12\x1b\n\tnum_tasks\x18\x02 \x01(\x03R\x08numTasks\x12.\n\x13num_completed_tasks\x18\x03 \x01(\x03R\x11numCompletedTasks\x12(\n\x10input_bytes_read\x18\x04 \x01(\x03R\x0einputBytesRead\x12\x12\n\x04\x64one\x18\x05 \x01(\x08R\x04\x64oneB\x0f\n\rresponse_type"A\n\x08KeyValue\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x19\n\x05value\x18\x02 \x01(\tH\x00R\x05value\x88\x01\x01\x42\x08\n\x06_value"\xaf\t\n\rConfigRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x44\n\toperation\x18\x03 \x01(\x0b\x32&.spark.connect.ConfigRequest.OperationR\toperation\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x1a\xf2\x03\n\tOperation\x12\x34\n\x03set\x18\x01 \x01(\x0b\x32 .spark.connect.ConfigRequest.SetH\x00R\x03set\x12\x34\n\x03get\x18\x02 \x01(\x0b\x32 .spark.connect.ConfigRequest.GetH\x00R\x03get\x12W\n\x10get_with_default\x18\x03 \x01(\x0b\x32+.spark.connect.ConfigRequest.GetWithDefaultH\x00R\x0egetWithDefault\x12G\n\nget_option\x18\x04 \x01(\x0b\x32&.spark.connect.ConfigRequest.GetOptionH\x00R\tgetOption\x12>\n\x07get_all\x18\x05 \x01(\x0b\x32#.spark.connect.ConfigRequest.GetAllH\x00R\x06getAll\x12:\n\x05unset\x18\x06 \x01(\x0b\x32".spark.connect.ConfigRequest.UnsetH\x00R\x05unset\x12P\n\ris_modifiable\x18\x07 \x01(\x0b\x32).spark.connect.ConfigRequest.IsModifiableH\x00R\x0cisModifiableB\t\n\x07op_type\x1a\\\n\x03Set\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1b\n\x06silent\x18\x02 \x01(\x08H\x00R\x06silent\x88\x01\x01\x42\t\n\x07_silent\x1a\x19\n\x03Get\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a?\n\x0eGetWithDefault\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x1f\n\tGetOption\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a\x30\n\x06GetAll\x12\x1b\n\x06prefix\x18\x01 \x01(\tH\x00R\x06prefix\x88\x01\x01\x42\t\n\x07_prefix\x1a\x1b\n\x05Unset\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a"\n\x0cIsModifiable\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keysB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xaf\x01\n\x0e\x43onfigResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x04 \x01(\tR\x13serverSideSessionId\x12-\n\x05pairs\x18\x02 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1a\n\x08warnings\x18\x03 \x03(\tR\x08warnings"\xea\x07\n\x13\x41\x64\x64\x41rtifactsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12$\n\x0b\x63lient_type\x18\x06 \x01(\tH\x02R\nclientType\x88\x01\x01\x12@\n\x05\x62\x61tch\x18\x03 \x01(\x0b\x32(.spark.connect.AddArtifactsRequest.BatchH\x00R\x05\x62\x61tch\x12Z\n\x0b\x62\x65gin_chunk\x18\x04 \x01(\x0b\x32\x37.spark.connect.AddArtifactsRequest.BeginChunkedArtifactH\x00R\nbeginChunk\x12H\n\x05\x63hunk\x18\x05 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkH\x00R\x05\x63hunk\x1a\x35\n\rArtifactChunk\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12\x10\n\x03\x63rc\x18\x02 \x01(\x03R\x03\x63rc\x1ao\n\x13SingleChunkArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x04\x64\x61ta\x1a]\n\x05\x42\x61tch\x12T\n\tartifacts\x18\x01 \x03(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactR\tartifacts\x1a\xc1\x01\n\x14\x42\x65ginChunkedArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0btotal_bytes\x18\x02 \x01(\x03R\ntotalBytes\x12\x1d\n\nnum_chunks\x18\x03 \x01(\x03R\tnumChunks\x12U\n\rinitial_chunk\x18\x04 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x0cinitialChunkB\t\n\x07payloadB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x02\n\x14\x41\x64\x64\x41rtifactsResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\tartifacts\x18\x01 \x03(\x0b\x32\x33.spark.connect.AddArtifactsResponse.ArtifactSummaryR\tartifacts\x1aQ\n\x0f\x41rtifactSummary\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x11is_crc_successful\x18\x02 \x01(\x08R\x0fisCrcSuccessful"\xc6\x02\n\x17\x41rtifactStatusesRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12\x14\n\x05names\x18\x04 \x03(\tR\x05namesB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xe0\x02\n\x18\x41rtifactStatusesResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\x08statuses\x18\x01 \x03(\x0b\x32\x35.spark.connect.ArtifactStatusesResponse.StatusesEntryR\x08statuses\x1as\n\rStatusesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ArtifactStatusesResponse.ArtifactStatusR\x05value:\x02\x38\x01\x1a(\n\x0e\x41rtifactStatus\x12\x16\n\x06\x65xists\x18\x01 \x01(\x08R\x06\x65xists"\xdb\x04\n\x10InterruptRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12T\n\x0einterrupt_type\x18\x04 \x01(\x0e\x32-.spark.connect.InterruptRequest.InterruptTypeR\rinterruptType\x12%\n\roperation_tag\x18\x05 \x01(\tH\x00R\x0coperationTag\x12#\n\x0coperation_id\x18\x06 \x01(\tH\x00R\x0boperationId"\x80\x01\n\rInterruptType\x12\x1e\n\x1aINTERRUPT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12INTERRUPT_TYPE_ALL\x10\x01\x12\x16\n\x12INTERRUPT_TYPE_TAG\x10\x02\x12\x1f\n\x1bINTERRUPT_TYPE_OPERATION_ID\x10\x03\x42\x0b\n\tinterruptB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x01\n\x11InterruptResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\'\n\x0finterrupted_ids\x18\x02 \x03(\tR\x0einterruptedIds"5\n\x0fReattachOptions\x12"\n\x0creattachable\x18\x01 \x01(\x08R\x0creattachable"\xb5\x01\n\x15ResultChunkingOptions\x12;\n\x1a\x61llow_arrow_batch_chunking\x18\x01 \x01(\x08R\x17\x61llowArrowBatchChunking\x12@\n\x1apreferred_arrow_chunk_size\x18\x02 \x01(\x03H\x00R\x17preferredArrowChunkSize\x88\x01\x01\x42\x1d\n\x1b_preferred_arrow_chunk_size"\x96\x03\n\x16ReattachExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x06 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x12-\n\x10last_response_id\x18\x05 \x01(\tH\x02R\x0elastResponseId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x13\n\x11_last_response_id"\xc9\x04\n\x15ReleaseExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12R\n\x0brelease_all\x18\x05 \x01(\x0b\x32/.spark.connect.ReleaseExecuteRequest.ReleaseAllH\x00R\nreleaseAll\x12X\n\rrelease_until\x18\x06 \x01(\x0b\x32\x31.spark.connect.ReleaseExecuteRequest.ReleaseUntilH\x00R\x0creleaseUntil\x1a\x0c\n\nReleaseAll\x1a/\n\x0cReleaseUntil\x12\x1f\n\x0bresponse_id\x18\x01 \x01(\tR\nresponseIdB\t\n\x07releaseB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x01\n\x16ReleaseExecuteResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12&\n\x0coperation_id\x18\x02 \x01(\tH\x00R\x0boperationId\x88\x01\x01\x42\x0f\n\r_operation_id"\xd4\x01\n\x15ReleaseSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12\'\n\x0f\x61llow_reconnect\x18\x04 \x01(\x08R\x0e\x61llowReconnectB\x0e\n\x0c_client_type"l\n\x16ReleaseSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId"\xcc\x02\n\x18\x46\x65tchErrorDetailsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x19\n\x08\x65rror_id\x18\x03 \x01(\tR\x07\x65rrorId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xd9\x0f\n\x19\x46\x65tchErrorDetailsResponse\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\x1d\n\nsession_id\x18\x04 \x01(\tR\tsessionId\x12)\n\x0eroot_error_idx\x18\x01 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x02 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrors\x1a\xae\x01\n\x11StackTraceElement\x12\'\n\x0f\x64\x65\x63laring_class\x18\x01 \x01(\tR\x0e\x64\x65\x63laringClass\x12\x1f\n\x0bmethod_name\x18\x02 \x01(\tR\nmethodName\x12 \n\tfile_name\x18\x03 \x01(\tH\x00R\x08\x66ileName\x88\x01\x01\x12\x1f\n\x0bline_number\x18\x04 \x01(\x05R\nlineNumberB\x0c\n\n_file_name\x1a\xf0\x02\n\x0cQueryContext\x12\x64\n\x0c\x63ontext_type\x18\n \x01(\x0e\x32\x41.spark.connect.FetchErrorDetailsResponse.QueryContext.ContextTypeR\x0b\x63ontextType\x12\x1f\n\x0bobject_type\x18\x01 \x01(\tR\nobjectType\x12\x1f\n\x0bobject_name\x18\x02 \x01(\tR\nobjectName\x12\x1f\n\x0bstart_index\x18\x03 \x01(\x05R\nstartIndex\x12\x1d\n\nstop_index\x18\x04 \x01(\x05R\tstopIndex\x12\x1a\n\x08\x66ragment\x18\x05 \x01(\tR\x08\x66ragment\x12\x1b\n\tcall_site\x18\x06 \x01(\tR\x08\x63\x61llSite\x12\x18\n\x07summary\x18\x07 \x01(\tR\x07summary"%\n\x0b\x43ontextType\x12\x07\n\x03SQL\x10\x00\x12\r\n\tDATAFRAME\x10\x01\x1a\xa6\x04\n\x0eSparkThrowable\x12$\n\x0b\x65rror_class\x18\x01 \x01(\tH\x00R\nerrorClass\x88\x01\x01\x12}\n\x12message_parameters\x18\x02 \x03(\x0b\x32N.spark.connect.FetchErrorDetailsResponse.SparkThrowable.MessageParametersEntryR\x11messageParameters\x12\\\n\x0equery_contexts\x18\x03 \x03(\x0b\x32\x35.spark.connect.FetchErrorDetailsResponse.QueryContextR\rqueryContexts\x12 \n\tsql_state\x18\x04 \x01(\tH\x01R\x08sqlState\x88\x01\x01\x12r\n\x14\x62reaking_change_info\x18\x05 \x01(\x0b\x32;.spark.connect.FetchErrorDetailsResponse.BreakingChangeInfoH\x02R\x12\x62reakingChangeInfo\x88\x01\x01\x1a\x44\n\x16MessageParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_error_classB\x0c\n\n_sql_stateB\x17\n\x15_breaking_change_info\x1a\xfa\x01\n\x12\x42reakingChangeInfo\x12+\n\x11migration_message\x18\x01 \x03(\tR\x10migrationMessage\x12k\n\x11mitigation_config\x18\x02 \x01(\x0b\x32\x39.spark.connect.FetchErrorDetailsResponse.MitigationConfigH\x00R\x10mitigationConfig\x88\x01\x01\x12$\n\x0bneeds_audit\x18\x03 \x01(\x08H\x01R\nneedsAudit\x88\x01\x01\x42\x14\n\x12_mitigation_configB\x0e\n\x0c_needs_audit\x1a:\n\x10MitigationConfig\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\x1a\xdb\x02\n\x05\x45rror\x12\x30\n\x14\x65rror_type_hierarchy\x18\x01 \x03(\tR\x12\x65rrorTypeHierarchy\x12\x18\n\x07message\x18\x02 \x01(\tR\x07message\x12[\n\x0bstack_trace\x18\x03 \x03(\x0b\x32:.spark.connect.FetchErrorDetailsResponse.StackTraceElementR\nstackTrace\x12 \n\tcause_idx\x18\x04 \x01(\x05H\x00R\x08\x63\x61useIdx\x88\x01\x01\x12\x65\n\x0fspark_throwable\x18\x05 \x01(\x0b\x32\x37.spark.connect.FetchErrorDetailsResponse.SparkThrowableH\x01R\x0esparkThrowable\x88\x01\x01\x42\x0c\n\n_cause_idxB\x12\n\x10_spark_throwableB\x11\n\x0f_root_error_idx"Z\n\x17\x43heckpointCommandResult\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation":\n\x15\x43reateBroadcastResult\x12!\n\x0c\x62roadcast_id\x18\x01 \x01(\x03R\x0b\x62roadcastId"\xea\x02\n\x13\x43loneSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12)\n\x0enew_session_id\x18\x04 \x01(\tH\x02R\x0cnewSessionId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x11\n\x0f_new_session_id"\xcc\x01\n\x14\x43loneSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12$\n\x0enew_session_id\x18\x03 \x01(\tR\x0cnewSessionId\x12:\n\x1anew_server_side_session_id\x18\x04 \x01(\tR\x16newServerSideSessionId"\xd3\x04\n\x10GetStatusRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12V\n&client_observed_server_side_session_id\x18\x04 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12\x66\n\x10operation_status\x18\x05 \x01(\x0b\x32\x36.spark.connect.GetStatusRequest.OperationStatusRequestH\x02R\x0foperationStatus\x88\x01\x01\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1at\n\x16OperationStatusRequest\x12#\n\roperation_ids\x18\x01 \x03(\tR\x0coperationIds\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensionsB\x0e\n\x0c_client_typeB)\n\'_client_observed_server_side_session_idB\x13\n\x11_operation_status"\xad\x05\n\x11GetStatusResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12_\n\x12operation_statuses\x18\x03 \x03(\x0b\x32\x30.spark.connect.GetStatusResponse.OperationStatusR\x11operationStatuses\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1a\xab\x03\n\x0fOperationStatus\x12!\n\x0coperation_id\x18\x01 \x01(\tR\x0boperationId\x12U\n\x05state\x18\x02 \x01(\x0e\x32?.spark.connect.GetStatusResponse.OperationStatus.OperationStateR\x05state\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xe6\x01\n\x0eOperationState\x12\x1f\n\x1bOPERATION_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17OPERATION_STATE_UNKNOWN\x10\x01\x12\x1b\n\x17OPERATION_STATE_RUNNING\x10\x02\x12\x1f\n\x1bOPERATION_STATE_TERMINATING\x10\x03\x12\x1d\n\x19OPERATION_STATE_SUCCEEDED\x10\x04\x12\x1a\n\x16OPERATION_STATE_FAILED\x10\x05\x12\x1d\n\x19OPERATION_STATE_CANCELLED\x10\x06*Q\n\x10\x43ompressionCodec\x12!\n\x1d\x43OMPRESSION_CODEC_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x43OMPRESSION_CODEC_ZSTD\x10\x01\x32\xdf\x08\n\x13SparkConnectService\x12X\n\x0b\x45xecutePlan\x12!.spark.connect.ExecutePlanRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12V\n\x0b\x41nalyzePlan\x12!.spark.connect.AnalyzePlanRequest\x1a".spark.connect.AnalyzePlanResponse"\x00\x12G\n\x06\x43onfig\x12\x1c.spark.connect.ConfigRequest\x1a\x1d.spark.connect.ConfigResponse"\x00\x12[\n\x0c\x41\x64\x64\x41rtifacts\x12".spark.connect.AddArtifactsRequest\x1a#.spark.connect.AddArtifactsResponse"\x00(\x01\x12\x63\n\x0e\x41rtifactStatus\x12&.spark.connect.ArtifactStatusesRequest\x1a\'.spark.connect.ArtifactStatusesResponse"\x00\x12P\n\tInterrupt\x12\x1f.spark.connect.InterruptRequest\x1a .spark.connect.InterruptResponse"\x00\x12`\n\x0fReattachExecute\x12%.spark.connect.ReattachExecuteRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12_\n\x0eReleaseExecute\x12$.spark.connect.ReleaseExecuteRequest\x1a%.spark.connect.ReleaseExecuteResponse"\x00\x12_\n\x0eReleaseSession\x12$.spark.connect.ReleaseSessionRequest\x1a%.spark.connect.ReleaseSessionResponse"\x00\x12h\n\x11\x46\x65tchErrorDetails\x12\'.spark.connect.FetchErrorDetailsRequest\x1a(.spark.connect.FetchErrorDetailsResponse"\x00\x12Y\n\x0c\x43loneSession\x12".spark.connect.CloneSessionRequest\x1a#.spark.connect.CloneSessionResponse"\x00\x12P\n\tGetStatus\x12\x1f.spark.connect.GetStatusRequest\x1a .spark.connect.GetStatusResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -71,8 +71,8 @@ _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" ]._serialized_options = b"8\001" - _globals["_COMPRESSIONCODEC"]._serialized_start = 19991 - _globals["_COMPRESSIONCODEC"]._serialized_end = 20072 + _globals["_COMPRESSIONCODEC"]._serialized_start = 20147 + _globals["_COMPRESSIONCODEC"]._serialized_end = 20228 _globals["_PLAN"]._serialized_start = 275 _globals["_PLAN"]._serialized_end = 758 _globals["_PLAN_COMPRESSEDOPERATION"]._serialized_start = 477 @@ -148,139 +148,141 @@ _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_start = 5868 _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_end = 6129 _globals["_EXECUTEPLANRESPONSE"]._serialized_start = 6208 - _globals["_EXECUTEPLANRESPONSE"]._serialized_end = 9799 - _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_start = 8308 - _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_end = 8379 - _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_start = 8382 - _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_end = 8630 - _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_start = 8633 - _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_end = 9150 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_start = 8728 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_end = 9060 + _globals["_EXECUTEPLANRESPONSE"]._serialized_end = 9895 + _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_start = 8404 + _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_end = 8475 + _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_start = 8478 + _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_end = 8726 + _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_start = 8729 + _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_end = 9246 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_start = 8824 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_end = 9156 _globals[ "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY" - ]._serialized_start = 8937 + ]._serialized_start = 9033 _globals[ "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY" - ]._serialized_end = 9060 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 9062 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 9150 - _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 9153 - _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_end = 9428 - _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_start = 9430 - _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_end = 9446 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_start = 9449 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_end = 9782 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_start = 9605 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_end = 9782 - _globals["_KEYVALUE"]._serialized_start = 9801 - _globals["_KEYVALUE"]._serialized_end = 9866 - _globals["_CONFIGREQUEST"]._serialized_start = 9869 - _globals["_CONFIGREQUEST"]._serialized_end = 11068 - _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 10177 - _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 10675 - _globals["_CONFIGREQUEST_SET"]._serialized_start = 10677 - _globals["_CONFIGREQUEST_SET"]._serialized_end = 10769 - _globals["_CONFIGREQUEST_GET"]._serialized_start = 10771 - _globals["_CONFIGREQUEST_GET"]._serialized_end = 10796 - _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 10798 - _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 10861 - _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 10863 - _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 10894 - _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 10896 - _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 10944 - _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 10946 - _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 10973 - _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 10975 - _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 11009 - _globals["_CONFIGRESPONSE"]._serialized_start = 11071 - _globals["_CONFIGRESPONSE"]._serialized_end = 11246 - _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 11249 - _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 12251 - _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 11724 - _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 11777 - _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 11779 - _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 11890 - _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 11892 - _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 11985 - _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 11988 - _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 12181 - _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 12254 - _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 12526 - _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 12445 - _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 12526 - _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 12529 - _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 12855 - _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 12858 - _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 13210 - _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 13053 - _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 13168 - _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 13170 - _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 13210 - _globals["_INTERRUPTREQUEST"]._serialized_start = 13213 - _globals["_INTERRUPTREQUEST"]._serialized_end = 13816 - _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 13616 - _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 13744 - _globals["_INTERRUPTRESPONSE"]._serialized_start = 13819 - _globals["_INTERRUPTRESPONSE"]._serialized_end = 13963 - _globals["_REATTACHOPTIONS"]._serialized_start = 13965 - _globals["_REATTACHOPTIONS"]._serialized_end = 14018 - _globals["_RESULTCHUNKINGOPTIONS"]._serialized_start = 14021 - _globals["_RESULTCHUNKINGOPTIONS"]._serialized_end = 14202 - _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 14205 - _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 14611 - _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 14614 - _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 15199 - _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 15068 - _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 15080 - _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 15082 - _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 15129 - _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 15202 - _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 15367 - _globals["_RELEASESESSIONREQUEST"]._serialized_start = 15370 - _globals["_RELEASESESSIONREQUEST"]._serialized_end = 15582 - _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 15584 - _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 15692 - _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 15695 - _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 16027 - _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 16030 - _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 18039 - _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 16259 - _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 16433 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 16436 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 16804 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 16767 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 16804 - _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 16807 - _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 17357 + ]._serialized_end = 9156 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 9158 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 9246 + _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 9249 + _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_end = 9524 + _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_start = 9526 + _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_end = 9542 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_start = 9545 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_end = 9878 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_start = 9701 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_end = 9878 + _globals["_KEYVALUE"]._serialized_start = 9897 + _globals["_KEYVALUE"]._serialized_end = 9962 + _globals["_CONFIGREQUEST"]._serialized_start = 9965 + _globals["_CONFIGREQUEST"]._serialized_end = 11164 + _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 10273 + _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 10771 + _globals["_CONFIGREQUEST_SET"]._serialized_start = 10773 + _globals["_CONFIGREQUEST_SET"]._serialized_end = 10865 + _globals["_CONFIGREQUEST_GET"]._serialized_start = 10867 + _globals["_CONFIGREQUEST_GET"]._serialized_end = 10892 + _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 10894 + _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 10957 + _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 10959 + _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 10990 + _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 10992 + _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 11040 + _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 11042 + _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 11069 + _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 11071 + _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 11105 + _globals["_CONFIGRESPONSE"]._serialized_start = 11167 + _globals["_CONFIGRESPONSE"]._serialized_end = 11342 + _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 11345 + _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 12347 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 11820 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 11873 + _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 11875 + _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 11986 + _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 11988 + _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 12081 + _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 12084 + _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 12277 + _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 12350 + _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 12622 + _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 12541 + _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 12622 + _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 12625 + _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 12951 + _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 12954 + _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 13306 + _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 13149 + _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 13264 + _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 13266 + _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 13306 + _globals["_INTERRUPTREQUEST"]._serialized_start = 13309 + _globals["_INTERRUPTREQUEST"]._serialized_end = 13912 + _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 13712 + _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 13840 + _globals["_INTERRUPTRESPONSE"]._serialized_start = 13915 + _globals["_INTERRUPTRESPONSE"]._serialized_end = 14059 + _globals["_REATTACHOPTIONS"]._serialized_start = 14061 + _globals["_REATTACHOPTIONS"]._serialized_end = 14114 + _globals["_RESULTCHUNKINGOPTIONS"]._serialized_start = 14117 + _globals["_RESULTCHUNKINGOPTIONS"]._serialized_end = 14298 + _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 14301 + _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 14707 + _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 14710 + _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 15295 + _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 15164 + _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 15176 + _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 15178 + _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 15225 + _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 15298 + _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 15463 + _globals["_RELEASESESSIONREQUEST"]._serialized_start = 15466 + _globals["_RELEASESESSIONREQUEST"]._serialized_end = 15678 + _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 15680 + _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 15788 + _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 15791 + _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 16123 + _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 16126 + _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 18135 + _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 16355 + _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 16529 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 16532 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 16900 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 16863 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 16900 + _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 16903 + _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 17453 _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" - ]._serialized_start = 17234 + ]._serialized_start = 17330 _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" - ]._serialized_end = 17302 - _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_start = 17360 - _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_end = 17610 - _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_start = 17612 - _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_end = 17670 - _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 17673 - _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 18020 - _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 18041 - _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 18131 - _globals["_CLONESESSIONREQUEST"]._serialized_start = 18134 - _globals["_CLONESESSIONREQUEST"]._serialized_end = 18496 - _globals["_CLONESESSIONRESPONSE"]._serialized_start = 18499 - _globals["_CLONESESSIONRESPONSE"]._serialized_end = 18703 - _globals["_GETSTATUSREQUEST"]._serialized_start = 18706 - _globals["_GETSTATUSREQUEST"]._serialized_end = 19301 - _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_start = 19105 - _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_end = 19221 - _globals["_GETSTATUSRESPONSE"]._serialized_start = 19304 - _globals["_GETSTATUSRESPONSE"]._serialized_end = 19989 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_start = 19562 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_end = 19989 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_start = 19759 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_end = 19989 - _globals["_SPARKCONNECTSERVICE"]._serialized_start = 20075 - _globals["_SPARKCONNECTSERVICE"]._serialized_end = 21194 + ]._serialized_end = 17398 + _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_start = 17456 + _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_end = 17706 + _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_start = 17708 + _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_end = 17766 + _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 17769 + _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 18116 + _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 18137 + _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 18227 + _globals["_CREATEBROADCASTRESULT"]._serialized_start = 18229 + _globals["_CREATEBROADCASTRESULT"]._serialized_end = 18287 + _globals["_CLONESESSIONREQUEST"]._serialized_start = 18290 + _globals["_CLONESESSIONREQUEST"]._serialized_end = 18652 + _globals["_CLONESESSIONRESPONSE"]._serialized_start = 18655 + _globals["_CLONESESSIONRESPONSE"]._serialized_end = 18859 + _globals["_GETSTATUSREQUEST"]._serialized_start = 18862 + _globals["_GETSTATUSREQUEST"]._serialized_end = 19457 + _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_start = 19261 + _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_end = 19377 + _globals["_GETSTATUSRESPONSE"]._serialized_start = 19460 + _globals["_GETSTATUSRESPONSE"]._serialized_end = 20145 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_start = 19718 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_end = 20145 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_start = 19915 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_end = 20145 + _globals["_SPARKCONNECTSERVICE"]._serialized_start = 20231 + _globals["_SPARKCONNECTSERVICE"]._serialized_end = 21350 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi index 2db3132cd0c01..1fa31946f00ab 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.pyi +++ b/python/pyspark/sql/connect/proto/base_pb2.pyi @@ -1759,6 +1759,7 @@ class ExecutePlanResponse(google.protobuf.message.Message): PIPELINE_EVENT_RESULT_FIELD_NUMBER: builtins.int PIPELINE_COMMAND_RESULT_FIELD_NUMBER: builtins.int PIPELINE_QUERY_FUNCTION_EXECUTION_SIGNAL_FIELD_NUMBER: builtins.int + CREATE_BROADCAST_RESULT_FIELD_NUMBER: builtins.int EXTENSION_FIELD_NUMBER: builtins.int METRICS_FIELD_NUMBER: builtins.int OBSERVED_METRICS_FIELD_NUMBER: builtins.int @@ -1841,6 +1842,9 @@ class ExecutePlanResponse(google.protobuf.message.Message): register its result with the server. """ @property + def create_broadcast_result(self) -> global___CreateBroadcastResult: + """(SPARK-51705) Server-assigned handle for a CreateBroadcastCommand.""" + @property def extension(self) -> google.protobuf.any_pb2.Any: """Support arbitrary result objects.""" @property @@ -1889,6 +1893,7 @@ class ExecutePlanResponse(google.protobuf.message.Message): | None = ..., pipeline_query_function_execution_signal: pyspark.sql.connect.proto.pipelines_pb2.PipelineQueryFunctionExecutionSignal | None = ..., + create_broadcast_result: global___CreateBroadcastResult | None = ..., extension: google.protobuf.any_pb2.Any | None = ..., metrics: global___ExecutePlanResponse.Metrics | None = ..., observed_metrics: collections.abc.Iterable[global___ExecutePlanResponse.ObservedMetrics] @@ -1902,6 +1907,8 @@ class ExecutePlanResponse(google.protobuf.message.Message): b"arrow_batch", "checkpoint_command_result", b"checkpoint_command_result", + "create_broadcast_result", + b"create_broadcast_result", "create_resource_profile_command_result", b"create_resource_profile_command_result", "execution_progress", @@ -1945,6 +1952,8 @@ class ExecutePlanResponse(google.protobuf.message.Message): b"arrow_batch", "checkpoint_command_result", b"checkpoint_command_result", + "create_broadcast_result", + b"create_broadcast_result", "create_resource_profile_command_result", b"create_resource_profile_command_result", "execution_progress", @@ -2010,6 +2019,7 @@ class ExecutePlanResponse(google.protobuf.message.Message): "pipeline_event_result", "pipeline_command_result", "pipeline_query_function_execution_signal", + "create_broadcast_result", "extension", ] | None @@ -4203,6 +4213,28 @@ class CheckpointCommandResult(google.protobuf.message.Message): global___CheckpointCommandResult = CheckpointCommandResult +class CreateBroadcastResult(google.protobuf.message.Message): + """(SPARK-51705) Server-assigned broadcast handle (mirrors CheckpointCommandResult).""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + BROADCAST_ID_FIELD_NUMBER: builtins.int + broadcast_id: builtins.int + """(Required) The driver-side broadcast id. The client embeds this id in the pickled + Broadcast closure (Broadcast._from_id), which the executor/worker resolves against its + _broadcastRegistry keyed by the same id. + """ + def __init__( + self, + *, + broadcast_id: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["broadcast_id", b"broadcast_id"] + ) -> None: ... + +global___CreateBroadcastResult = CreateBroadcastResult + class CloneSessionRequest(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py index 78af0ddbfe318..f5155853ca0c3 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.py +++ b/python/pyspark/sql/connect/proto/commands_pb2.py @@ -44,7 +44,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xfb\x0e\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeView\x12O\n\x12write_operation_v2\x18\x04 \x01(\x0b\x32\x1f.spark.connect.WriteOperationV2H\x00R\x10writeOperationV2\x12<\n\x0bsql_command\x18\x05 \x01(\x0b\x32\x19.spark.connect.SqlCommandH\x00R\nsqlCommand\x12k\n\x1cwrite_stream_operation_start\x18\x06 \x01(\x0b\x32(.spark.connect.WriteStreamOperationStartH\x00R\x19writeStreamOperationStart\x12^\n\x17streaming_query_command\x18\x07 \x01(\x0b\x32$.spark.connect.StreamingQueryCommandH\x00R\x15streamingQueryCommand\x12X\n\x15get_resources_command\x18\x08 \x01(\x0b\x32".spark.connect.GetResourcesCommandH\x00R\x13getResourcesCommand\x12t\n\x1fstreaming_query_manager_command\x18\t \x01(\x0b\x32+.spark.connect.StreamingQueryManagerCommandH\x00R\x1cstreamingQueryManagerCommand\x12m\n\x17register_table_function\x18\n \x01(\x0b\x32\x33.spark.connect.CommonInlineUserDefinedTableFunctionH\x00R\x15registerTableFunction\x12\x81\x01\n$streaming_query_listener_bus_command\x18\x0b \x01(\x0b\x32/.spark.connect.StreamingQueryListenerBusCommandH\x00R streamingQueryListenerBusCommand\x12\x64\n\x14register_data_source\x18\x0c \x01(\x0b\x32\x30.spark.connect.CommonInlineUserDefinedDataSourceH\x00R\x12registerDataSource\x12t\n\x1f\x63reate_resource_profile_command\x18\r \x01(\x0b\x32+.spark.connect.CreateResourceProfileCommandH\x00R\x1c\x63reateResourceProfileCommand\x12Q\n\x12\x63heckpoint_command\x18\x0e \x01(\x0b\x32 .spark.connect.CheckpointCommandH\x00R\x11\x63heckpointCommand\x12\x84\x01\n%remove_cached_remote_relation_command\x18\x0f \x01(\x0b\x32\x30.spark.connect.RemoveCachedRemoteRelationCommandH\x00R!removeCachedRemoteRelationCommand\x12_\n\x18merge_into_table_command\x18\x10 \x01(\x0b\x32$.spark.connect.MergeIntoTableCommandH\x00R\x15mergeIntoTableCommand\x12\x39\n\nml_command\x18\x11 \x01(\x0b\x32\x18.spark.connect.MlCommandH\x00R\tmlCommand\x12\x61\n\x18\x65xecute_external_command\x18\x12 \x01(\x0b\x32%.spark.connect.ExecuteExternalCommandH\x00R\x16\x65xecuteExternalCommand\x12K\n\x10pipeline_command\x18\x13 \x01(\x0b\x32\x1e.spark.connect.PipelineCommandH\x00R\x0fpipelineCommand\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x0e\n\x0c\x63ommand_type"\xaa\x04\n\nSqlCommand\x12\x14\n\x03sql\x18\x01 \x01(\tB\x02\x18\x01R\x03sql\x12;\n\x04\x61rgs\x18\x02 \x03(\x0b\x32#.spark.connect.SqlCommand.ArgsEntryB\x02\x18\x01R\x04\x61rgs\x12@\n\x08pos_args\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralB\x02\x18\x01R\x07posArgs\x12Z\n\x0fnamed_arguments\x18\x04 \x03(\x0b\x32-.spark.connect.SqlCommand.NamedArgumentsEntryB\x02\x18\x01R\x0enamedArguments\x12\x42\n\rpos_arguments\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionB\x02\x18\x01R\x0cposArguments\x12-\n\x05input\x18\x06 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x1aZ\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01\x1a\\\n\x13NamedArgumentsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value:\x02\x38\x01"\x96\x01\n\x1a\x43reateDataFrameViewCommand\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n\tis_global\x18\x03 \x01(\x08R\x08isGlobal\x12\x18\n\x07replace\x18\x04 \x01(\x08R\x07replace"\xfe\x08\n\x0eWriteOperation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1b\n\x06source\x18\x02 \x01(\tH\x01R\x06source\x88\x01\x01\x12\x14\n\x04path\x18\x03 \x01(\tH\x00R\x04path\x12?\n\x05table\x18\x04 \x01(\x0b\x32\'.spark.connect.WriteOperation.SaveTableH\x00R\x05table\x12:\n\x04mode\x18\x05 \x01(\x0e\x32&.spark.connect.WriteOperation.SaveModeR\x04mode\x12*\n\x11sort_column_names\x18\x06 \x03(\tR\x0fsortColumnNames\x12\x31\n\x14partitioning_columns\x18\x07 \x03(\tR\x13partitioningColumns\x12\x43\n\tbucket_by\x18\x08 \x01(\x0b\x32&.spark.connect.WriteOperation.BucketByR\x08\x62ucketBy\x12\x44\n\x07options\x18\t \x03(\x0b\x32*.spark.connect.WriteOperation.OptionsEntryR\x07options\x12-\n\x12\x63lustering_columns\x18\n \x03(\tR\x11\x63lusteringColumns\x12\x32\n\x15with_schema_evolution\x18\x0b \x01(\x08R\x13withSchemaEvolution\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x82\x02\n\tSaveTable\x12\x1d\n\ntable_name\x18\x01 \x01(\tR\ttableName\x12X\n\x0bsave_method\x18\x02 \x01(\x0e\x32\x37.spark.connect.WriteOperation.SaveTable.TableSaveMethodR\nsaveMethod"|\n\x0fTableSaveMethod\x12!\n\x1dTABLE_SAVE_METHOD_UNSPECIFIED\x10\x00\x12#\n\x1fTABLE_SAVE_METHOD_SAVE_AS_TABLE\x10\x01\x12!\n\x1dTABLE_SAVE_METHOD_INSERT_INTO\x10\x02\x1a[\n\x08\x42ucketBy\x12.\n\x13\x62ucket_column_names\x18\x01 \x03(\tR\x11\x62ucketColumnNames\x12\x1f\n\x0bnum_buckets\x18\x02 \x01(\x05R\nnumBuckets"\x89\x01\n\x08SaveMode\x12\x19\n\x15SAVE_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10SAVE_MODE_APPEND\x10\x01\x12\x17\n\x13SAVE_MODE_OVERWRITE\x10\x02\x12\x1d\n\x19SAVE_MODE_ERROR_IF_EXISTS\x10\x03\x12\x14\n\x10SAVE_MODE_IGNORE\x10\x04\x42\x0b\n\tsave_typeB\t\n\x07_source"\x90\x07\n\x10WriteOperationV2\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1d\n\ntable_name\x18\x02 \x01(\tR\ttableName\x12\x1f\n\x08provider\x18\x03 \x01(\tH\x00R\x08provider\x88\x01\x01\x12L\n\x14partitioning_columns\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13partitioningColumns\x12\x46\n\x07options\x18\x05 \x03(\x0b\x32,.spark.connect.WriteOperationV2.OptionsEntryR\x07options\x12_\n\x10table_properties\x18\x06 \x03(\x0b\x32\x34.spark.connect.WriteOperationV2.TablePropertiesEntryR\x0ftableProperties\x12\x38\n\x04mode\x18\x07 \x01(\x0e\x32$.spark.connect.WriteOperationV2.ModeR\x04mode\x12J\n\x13overwrite_condition\x18\x08 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x12overwriteCondition\x12-\n\x12\x63lustering_columns\x18\t \x03(\tR\x11\x63lusteringColumns\x12\x32\n\x15with_schema_evolution\x18\n \x01(\x08R\x13withSchemaEvolution\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x42\n\x14TablePropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"\x9f\x01\n\x04Mode\x12\x14\n\x10MODE_UNSPECIFIED\x10\x00\x12\x0f\n\x0bMODE_CREATE\x10\x01\x12\x12\n\x0eMODE_OVERWRITE\x10\x02\x12\x1d\n\x19MODE_OVERWRITE_PARTITIONS\x10\x03\x12\x0f\n\x0bMODE_APPEND\x10\x04\x12\x10\n\x0cMODE_REPLACE\x10\x05\x12\x1a\n\x16MODE_CREATE_OR_REPLACE\x10\x06\x42\x0b\n\t_provider"\x93\x07\n\x19WriteStreamOperationStart\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06\x66ormat\x18\x02 \x01(\tR\x06\x66ormat\x12O\n\x07options\x18\x03 \x03(\x0b\x32\x35.spark.connect.WriteStreamOperationStart.OptionsEntryR\x07options\x12:\n\x19partitioning_column_names\x18\x04 \x03(\tR\x17partitioningColumnNames\x12:\n\x18processing_time_interval\x18\x05 \x01(\tH\x00R\x16processingTimeInterval\x12%\n\ravailable_now\x18\x06 \x01(\x08H\x00R\x0c\x61vailableNow\x12\x14\n\x04once\x18\x07 \x01(\x08H\x00R\x04once\x12\x46\n\x1e\x63ontinuous_checkpoint_interval\x18\x08 \x01(\tH\x00R\x1c\x63ontinuousCheckpointInterval\x12\x39\n\x18real_time_batch_duration\x18\x64 \x01(\tH\x00R\x15realTimeBatchDuration\x12\x1f\n\x0boutput_mode\x18\t \x01(\tR\noutputMode\x12\x1d\n\nquery_name\x18\n \x01(\tR\tqueryName\x12\x14\n\x04path\x18\x0b \x01(\tH\x01R\x04path\x12\x1f\n\ntable_name\x18\x0c \x01(\tH\x01R\ttableName\x12N\n\x0e\x66oreach_writer\x18\r \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\rforeachWriter\x12L\n\rforeach_batch\x18\x0e \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\x0c\x66oreachBatch\x12\x36\n\x17\x63lustering_column_names\x18\x0f \x03(\tR\x15\x63lusteringColumnNames\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07triggerB\x12\n\x10sink_destination"\xb3\x01\n\x18StreamingForeachFunction\x12\x43\n\x0fpython_function\x18\x01 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x0epythonFunction\x12\x46\n\x0escala_function\x18\x02 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\rscalaFunctionB\n\n\x08\x66unction"\xd4\x01\n\x1fWriteStreamOperationStartResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12<\n\x18query_started_event_json\x18\x03 \x01(\tH\x00R\x15queryStartedEventJson\x88\x01\x01\x42\x1b\n\x19_query_started_event_json"A\n\x18StreamingQueryInstanceId\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x15\n\x06run_id\x18\x02 \x01(\tR\x05runId"\xf8\x04\n\x15StreamingQueryCommand\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x18\n\x06status\x18\x02 \x01(\x08H\x00R\x06status\x12%\n\rlast_progress\x18\x03 \x01(\x08H\x00R\x0clastProgress\x12)\n\x0frecent_progress\x18\x04 \x01(\x08H\x00R\x0erecentProgress\x12\x14\n\x04stop\x18\x05 \x01(\x08H\x00R\x04stop\x12\x34\n\x15process_all_available\x18\x06 \x01(\x08H\x00R\x13processAllAvailable\x12O\n\x07\x65xplain\x18\x07 \x01(\x0b\x32\x33.spark.connect.StreamingQueryCommand.ExplainCommandH\x00R\x07\x65xplain\x12\x1e\n\texception\x18\x08 \x01(\x08H\x00R\texception\x12k\n\x11\x61wait_termination\x18\t \x01(\x0b\x32<.spark.connect.StreamingQueryCommand.AwaitTerminationCommandH\x00R\x10\x61waitTermination\x1a,\n\x0e\x45xplainCommand\x12\x1a\n\x08\x65xtended\x18\x01 \x01(\x08R\x08\x65xtended\x1aL\n\x17\x41waitTerminationCommand\x12"\n\ntimeout_ms\x18\x02 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_msB\t\n\x07\x63ommand"\xf5\x08\n\x1bStreamingQueryCommandResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12Q\n\x06status\x18\x02 \x01(\x0b\x32\x37.spark.connect.StreamingQueryCommandResult.StatusResultH\x00R\x06status\x12j\n\x0frecent_progress\x18\x03 \x01(\x0b\x32?.spark.connect.StreamingQueryCommandResult.RecentProgressResultH\x00R\x0erecentProgress\x12T\n\x07\x65xplain\x18\x04 \x01(\x0b\x32\x38.spark.connect.StreamingQueryCommandResult.ExplainResultH\x00R\x07\x65xplain\x12Z\n\texception\x18\x05 \x01(\x0b\x32:.spark.connect.StreamingQueryCommandResult.ExceptionResultH\x00R\texception\x12p\n\x11\x61wait_termination\x18\x06 \x01(\x0b\x32\x41.spark.connect.StreamingQueryCommandResult.AwaitTerminationResultH\x00R\x10\x61waitTermination\x1a\xaa\x01\n\x0cStatusResult\x12%\n\x0estatus_message\x18\x01 \x01(\tR\rstatusMessage\x12*\n\x11is_data_available\x18\x02 \x01(\x08R\x0fisDataAvailable\x12*\n\x11is_trigger_active\x18\x03 \x01(\x08R\x0fisTriggerActive\x12\x1b\n\tis_active\x18\x04 \x01(\x08R\x08isActive\x1aH\n\x14RecentProgressResult\x12\x30\n\x14recent_progress_json\x18\x05 \x03(\tR\x12recentProgressJson\x1a\'\n\rExplainResult\x12\x16\n\x06result\x18\x01 \x01(\tR\x06result\x1a\xc5\x01\n\x0f\x45xceptionResult\x12\x30\n\x11\x65xception_message\x18\x01 \x01(\tH\x00R\x10\x65xceptionMessage\x88\x01\x01\x12$\n\x0b\x65rror_class\x18\x02 \x01(\tH\x01R\nerrorClass\x88\x01\x01\x12$\n\x0bstack_trace\x18\x03 \x01(\tH\x02R\nstackTrace\x88\x01\x01\x42\x14\n\x12_exception_messageB\x0e\n\x0c_error_classB\x0e\n\x0c_stack_trace\x1a\x38\n\x16\x41waitTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminatedB\r\n\x0bresult_type"\xbd\x06\n\x1cStreamingQueryManagerCommand\x12\x18\n\x06\x61\x63tive\x18\x01 \x01(\x08H\x00R\x06\x61\x63tive\x12\x1d\n\tget_query\x18\x02 \x01(\tH\x00R\x08getQuery\x12|\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32\x46.spark.connect.StreamingQueryManagerCommand.AwaitAnyTerminationCommandH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12n\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0b\x61\x64\x64Listener\x12t\n\x0fremove_listener\x18\x06 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0eremoveListener\x12\'\n\x0elist_listeners\x18\x07 \x01(\x08H\x00R\rlistListeners\x1aO\n\x1a\x41waitAnyTerminationCommand\x12"\n\ntimeout_ms\x18\x01 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_ms\x1a\xcd\x01\n\x1dStreamingQueryListenerCommand\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x12U\n\x17python_listener_payload\x18\x02 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x15pythonListenerPayload\x88\x01\x01\x12\x0e\n\x02id\x18\x03 \x01(\tR\x02idB\x1a\n\x18_python_listener_payloadB\t\n\x07\x63ommand"\xb4\x08\n"StreamingQueryManagerCommandResult\x12X\n\x06\x61\x63tive\x18\x01 \x01(\x0b\x32>.spark.connect.StreamingQueryManagerCommandResult.ActiveResultH\x00R\x06\x61\x63tive\x12`\n\x05query\x18\x02 \x01(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceH\x00R\x05query\x12\x81\x01\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32K.spark.connect.StreamingQueryManagerCommandResult.AwaitAnyTerminationResultH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12#\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x08H\x00R\x0b\x61\x64\x64Listener\x12)\n\x0fremove_listener\x18\x06 \x01(\x08H\x00R\x0eremoveListener\x12{\n\x0elist_listeners\x18\x07 \x01(\x0b\x32R.spark.connect.StreamingQueryManagerCommandResult.ListStreamingQueryListenerResultH\x00R\rlistListeners\x1a\x7f\n\x0c\x41\x63tiveResult\x12o\n\x0e\x61\x63tive_queries\x18\x01 \x03(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceR\ractiveQueries\x1as\n\x16StreamingQueryInstance\x12\x37\n\x02id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x02id\x12\x17\n\x04name\x18\x02 \x01(\tH\x00R\x04name\x88\x01\x01\x42\x07\n\x05_name\x1a;\n\x19\x41waitAnyTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminated\x1aK\n\x1eStreamingQueryListenerInstance\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x1a\x45\n ListStreamingQueryListenerResult\x12!\n\x0clistener_ids\x18\x01 \x03(\tR\x0blistenerIdsB\r\n\x0bresult_type"\xad\x01\n StreamingQueryListenerBusCommand\x12;\n\x19\x61\x64\x64_listener_bus_listener\x18\x01 \x01(\x08H\x00R\x16\x61\x64\x64ListenerBusListener\x12\x41\n\x1cremove_listener_bus_listener\x18\x02 \x01(\x08H\x00R\x19removeListenerBusListenerB\t\n\x07\x63ommand"\x83\x01\n\x1bStreamingQueryListenerEvent\x12\x1d\n\nevent_json\x18\x01 \x01(\tR\teventJson\x12\x45\n\nevent_type\x18\x02 \x01(\x0e\x32&.spark.connect.StreamingQueryEventTypeR\teventType"\xcc\x01\n"StreamingQueryListenerEventsResult\x12\x42\n\x06\x65vents\x18\x01 \x03(\x0b\x32*.spark.connect.StreamingQueryListenerEventR\x06\x65vents\x12\x42\n\x1blistener_bus_listener_added\x18\x02 \x01(\x08H\x00R\x18listenerBusListenerAdded\x88\x01\x01\x42\x1e\n\x1c_listener_bus_listener_added"\x15\n\x13GetResourcesCommand"\xd4\x01\n\x19GetResourcesCommandResult\x12U\n\tresources\x18\x01 \x03(\x0b\x32\x37.spark.connect.GetResourcesCommandResult.ResourcesEntryR\tresources\x1a`\n\x0eResourcesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32".spark.connect.ResourceInformationR\x05value:\x02\x38\x01"X\n\x1c\x43reateResourceProfileCommand\x12\x38\n\x07profile\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ResourceProfileR\x07profile"C\n"CreateResourceProfileCommandResult\x12\x1d\n\nprofile_id\x18\x01 \x01(\x05R\tprofileId"d\n!RemoveCachedRemoteRelationCommand\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation"\xcd\x01\n\x11\x43heckpointCommand\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x14\n\x05local\x18\x02 \x01(\x08R\x05local\x12\x14\n\x05\x65\x61ger\x18\x03 \x01(\x08R\x05\x65\x61ger\x12\x45\n\rstorage_level\x18\x04 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xe8\x03\n\x15MergeIntoTableCommand\x12*\n\x11target_table_name\x18\x01 \x01(\tR\x0ftargetTableName\x12\x43\n\x11source_table_plan\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x0fsourceTablePlan\x12\x42\n\x0fmerge_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0emergeCondition\x12>\n\rmatch_actions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0cmatchActions\x12I\n\x13not_matched_actions\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x11notMatchedActions\x12[\n\x1dnot_matched_by_source_actions\x18\x06 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x19notMatchedBySourceActions\x12\x32\n\x15with_schema_evolution\x18\x07 \x01(\x08R\x13withSchemaEvolution"\xd4\x01\n\x16\x45xecuteExternalCommand\x12\x16\n\x06runner\x18\x01 \x01(\tR\x06runner\x12\x18\n\x07\x63ommand\x18\x02 \x01(\tR\x07\x63ommand\x12L\n\x07options\x18\x03 \x03(\x0b\x32\x32.spark.connect.ExecuteExternalCommand.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01*\x85\x01\n\x17StreamingQueryEventType\x12\x1e\n\x1aQUERY_PROGRESS_UNSPECIFIED\x10\x00\x12\x18\n\x14QUERY_PROGRESS_EVENT\x10\x01\x12\x1a\n\x16QUERY_TERMINATED_EVENT\x10\x02\x12\x14\n\x10QUERY_IDLE_EVENT\x10\x03\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xca\x10\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeView\x12O\n\x12write_operation_v2\x18\x04 \x01(\x0b\x32\x1f.spark.connect.WriteOperationV2H\x00R\x10writeOperationV2\x12<\n\x0bsql_command\x18\x05 \x01(\x0b\x32\x19.spark.connect.SqlCommandH\x00R\nsqlCommand\x12k\n\x1cwrite_stream_operation_start\x18\x06 \x01(\x0b\x32(.spark.connect.WriteStreamOperationStartH\x00R\x19writeStreamOperationStart\x12^\n\x17streaming_query_command\x18\x07 \x01(\x0b\x32$.spark.connect.StreamingQueryCommandH\x00R\x15streamingQueryCommand\x12X\n\x15get_resources_command\x18\x08 \x01(\x0b\x32".spark.connect.GetResourcesCommandH\x00R\x13getResourcesCommand\x12t\n\x1fstreaming_query_manager_command\x18\t \x01(\x0b\x32+.spark.connect.StreamingQueryManagerCommandH\x00R\x1cstreamingQueryManagerCommand\x12m\n\x17register_table_function\x18\n \x01(\x0b\x32\x33.spark.connect.CommonInlineUserDefinedTableFunctionH\x00R\x15registerTableFunction\x12\x81\x01\n$streaming_query_listener_bus_command\x18\x0b \x01(\x0b\x32/.spark.connect.StreamingQueryListenerBusCommandH\x00R streamingQueryListenerBusCommand\x12\x64\n\x14register_data_source\x18\x0c \x01(\x0b\x32\x30.spark.connect.CommonInlineUserDefinedDataSourceH\x00R\x12registerDataSource\x12t\n\x1f\x63reate_resource_profile_command\x18\r \x01(\x0b\x32+.spark.connect.CreateResourceProfileCommandH\x00R\x1c\x63reateResourceProfileCommand\x12Q\n\x12\x63heckpoint_command\x18\x0e \x01(\x0b\x32 .spark.connect.CheckpointCommandH\x00R\x11\x63heckpointCommand\x12\x84\x01\n%remove_cached_remote_relation_command\x18\x0f \x01(\x0b\x32\x30.spark.connect.RemoveCachedRemoteRelationCommandH\x00R!removeCachedRemoteRelationCommand\x12_\n\x18merge_into_table_command\x18\x10 \x01(\x0b\x32$.spark.connect.MergeIntoTableCommandH\x00R\x15mergeIntoTableCommand\x12\x39\n\nml_command\x18\x11 \x01(\x0b\x32\x18.spark.connect.MlCommandH\x00R\tmlCommand\x12\x61\n\x18\x65xecute_external_command\x18\x12 \x01(\x0b\x32%.spark.connect.ExecuteExternalCommandH\x00R\x16\x65xecuteExternalCommand\x12K\n\x10pipeline_command\x18\x13 \x01(\x0b\x32\x1e.spark.connect.PipelineCommandH\x00R\x0fpipelineCommand\x12\x61\n\x18\x63reate_broadcast_command\x18\x14 \x01(\x0b\x32%.spark.connect.CreateBroadcastCommandH\x00R\x16\x63reateBroadcastCommand\x12j\n\x1bunpersist_broadcast_command\x18\x15 \x01(\x0b\x32(.spark.connect.UnpersistBroadcastCommandH\x00R\x19unpersistBroadcastCommand\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x0e\n\x0c\x63ommand_type"\\\n\x16\x43reateBroadcastCommand\x12#\n\rartifact_hash\x18\x01 \x01(\tR\x0c\x61rtifactHash\x12\x1d\n\nsize_bytes\x18\x02 \x01(\x03R\tsizeBytes"t\n\x19UnpersistBroadcastCommand\x12!\n\x0c\x62roadcast_id\x18\x01 \x01(\x03R\x0b\x62roadcastId\x12\x1a\n\x08\x62locking\x18\x02 \x01(\x08R\x08\x62locking\x12\x18\n\x07\x64\x65stroy\x18\x03 \x01(\x08R\x07\x64\x65stroy"\xaa\x04\n\nSqlCommand\x12\x14\n\x03sql\x18\x01 \x01(\tB\x02\x18\x01R\x03sql\x12;\n\x04\x61rgs\x18\x02 \x03(\x0b\x32#.spark.connect.SqlCommand.ArgsEntryB\x02\x18\x01R\x04\x61rgs\x12@\n\x08pos_args\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralB\x02\x18\x01R\x07posArgs\x12Z\n\x0fnamed_arguments\x18\x04 \x03(\x0b\x32-.spark.connect.SqlCommand.NamedArgumentsEntryB\x02\x18\x01R\x0enamedArguments\x12\x42\n\rpos_arguments\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionB\x02\x18\x01R\x0cposArguments\x12-\n\x05input\x18\x06 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x1aZ\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01\x1a\\\n\x13NamedArgumentsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value:\x02\x38\x01"\x96\x01\n\x1a\x43reateDataFrameViewCommand\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n\tis_global\x18\x03 \x01(\x08R\x08isGlobal\x12\x18\n\x07replace\x18\x04 \x01(\x08R\x07replace"\xfe\x08\n\x0eWriteOperation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1b\n\x06source\x18\x02 \x01(\tH\x01R\x06source\x88\x01\x01\x12\x14\n\x04path\x18\x03 \x01(\tH\x00R\x04path\x12?\n\x05table\x18\x04 \x01(\x0b\x32\'.spark.connect.WriteOperation.SaveTableH\x00R\x05table\x12:\n\x04mode\x18\x05 \x01(\x0e\x32&.spark.connect.WriteOperation.SaveModeR\x04mode\x12*\n\x11sort_column_names\x18\x06 \x03(\tR\x0fsortColumnNames\x12\x31\n\x14partitioning_columns\x18\x07 \x03(\tR\x13partitioningColumns\x12\x43\n\tbucket_by\x18\x08 \x01(\x0b\x32&.spark.connect.WriteOperation.BucketByR\x08\x62ucketBy\x12\x44\n\x07options\x18\t \x03(\x0b\x32*.spark.connect.WriteOperation.OptionsEntryR\x07options\x12-\n\x12\x63lustering_columns\x18\n \x03(\tR\x11\x63lusteringColumns\x12\x32\n\x15with_schema_evolution\x18\x0b \x01(\x08R\x13withSchemaEvolution\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x82\x02\n\tSaveTable\x12\x1d\n\ntable_name\x18\x01 \x01(\tR\ttableName\x12X\n\x0bsave_method\x18\x02 \x01(\x0e\x32\x37.spark.connect.WriteOperation.SaveTable.TableSaveMethodR\nsaveMethod"|\n\x0fTableSaveMethod\x12!\n\x1dTABLE_SAVE_METHOD_UNSPECIFIED\x10\x00\x12#\n\x1fTABLE_SAVE_METHOD_SAVE_AS_TABLE\x10\x01\x12!\n\x1dTABLE_SAVE_METHOD_INSERT_INTO\x10\x02\x1a[\n\x08\x42ucketBy\x12.\n\x13\x62ucket_column_names\x18\x01 \x03(\tR\x11\x62ucketColumnNames\x12\x1f\n\x0bnum_buckets\x18\x02 \x01(\x05R\nnumBuckets"\x89\x01\n\x08SaveMode\x12\x19\n\x15SAVE_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10SAVE_MODE_APPEND\x10\x01\x12\x17\n\x13SAVE_MODE_OVERWRITE\x10\x02\x12\x1d\n\x19SAVE_MODE_ERROR_IF_EXISTS\x10\x03\x12\x14\n\x10SAVE_MODE_IGNORE\x10\x04\x42\x0b\n\tsave_typeB\t\n\x07_source"\x90\x07\n\x10WriteOperationV2\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1d\n\ntable_name\x18\x02 \x01(\tR\ttableName\x12\x1f\n\x08provider\x18\x03 \x01(\tH\x00R\x08provider\x88\x01\x01\x12L\n\x14partitioning_columns\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13partitioningColumns\x12\x46\n\x07options\x18\x05 \x03(\x0b\x32,.spark.connect.WriteOperationV2.OptionsEntryR\x07options\x12_\n\x10table_properties\x18\x06 \x03(\x0b\x32\x34.spark.connect.WriteOperationV2.TablePropertiesEntryR\x0ftableProperties\x12\x38\n\x04mode\x18\x07 \x01(\x0e\x32$.spark.connect.WriteOperationV2.ModeR\x04mode\x12J\n\x13overwrite_condition\x18\x08 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x12overwriteCondition\x12-\n\x12\x63lustering_columns\x18\t \x03(\tR\x11\x63lusteringColumns\x12\x32\n\x15with_schema_evolution\x18\n \x01(\x08R\x13withSchemaEvolution\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x42\n\x14TablePropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"\x9f\x01\n\x04Mode\x12\x14\n\x10MODE_UNSPECIFIED\x10\x00\x12\x0f\n\x0bMODE_CREATE\x10\x01\x12\x12\n\x0eMODE_OVERWRITE\x10\x02\x12\x1d\n\x19MODE_OVERWRITE_PARTITIONS\x10\x03\x12\x0f\n\x0bMODE_APPEND\x10\x04\x12\x10\n\x0cMODE_REPLACE\x10\x05\x12\x1a\n\x16MODE_CREATE_OR_REPLACE\x10\x06\x42\x0b\n\t_provider"\x93\x07\n\x19WriteStreamOperationStart\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06\x66ormat\x18\x02 \x01(\tR\x06\x66ormat\x12O\n\x07options\x18\x03 \x03(\x0b\x32\x35.spark.connect.WriteStreamOperationStart.OptionsEntryR\x07options\x12:\n\x19partitioning_column_names\x18\x04 \x03(\tR\x17partitioningColumnNames\x12:\n\x18processing_time_interval\x18\x05 \x01(\tH\x00R\x16processingTimeInterval\x12%\n\ravailable_now\x18\x06 \x01(\x08H\x00R\x0c\x61vailableNow\x12\x14\n\x04once\x18\x07 \x01(\x08H\x00R\x04once\x12\x46\n\x1e\x63ontinuous_checkpoint_interval\x18\x08 \x01(\tH\x00R\x1c\x63ontinuousCheckpointInterval\x12\x39\n\x18real_time_batch_duration\x18\x64 \x01(\tH\x00R\x15realTimeBatchDuration\x12\x1f\n\x0boutput_mode\x18\t \x01(\tR\noutputMode\x12\x1d\n\nquery_name\x18\n \x01(\tR\tqueryName\x12\x14\n\x04path\x18\x0b \x01(\tH\x01R\x04path\x12\x1f\n\ntable_name\x18\x0c \x01(\tH\x01R\ttableName\x12N\n\x0e\x66oreach_writer\x18\r \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\rforeachWriter\x12L\n\rforeach_batch\x18\x0e \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\x0c\x66oreachBatch\x12\x36\n\x17\x63lustering_column_names\x18\x0f \x03(\tR\x15\x63lusteringColumnNames\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07triggerB\x12\n\x10sink_destination"\xb3\x01\n\x18StreamingForeachFunction\x12\x43\n\x0fpython_function\x18\x01 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x0epythonFunction\x12\x46\n\x0escala_function\x18\x02 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\rscalaFunctionB\n\n\x08\x66unction"\xd4\x01\n\x1fWriteStreamOperationStartResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12<\n\x18query_started_event_json\x18\x03 \x01(\tH\x00R\x15queryStartedEventJson\x88\x01\x01\x42\x1b\n\x19_query_started_event_json"A\n\x18StreamingQueryInstanceId\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x15\n\x06run_id\x18\x02 \x01(\tR\x05runId"\xf8\x04\n\x15StreamingQueryCommand\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x18\n\x06status\x18\x02 \x01(\x08H\x00R\x06status\x12%\n\rlast_progress\x18\x03 \x01(\x08H\x00R\x0clastProgress\x12)\n\x0frecent_progress\x18\x04 \x01(\x08H\x00R\x0erecentProgress\x12\x14\n\x04stop\x18\x05 \x01(\x08H\x00R\x04stop\x12\x34\n\x15process_all_available\x18\x06 \x01(\x08H\x00R\x13processAllAvailable\x12O\n\x07\x65xplain\x18\x07 \x01(\x0b\x32\x33.spark.connect.StreamingQueryCommand.ExplainCommandH\x00R\x07\x65xplain\x12\x1e\n\texception\x18\x08 \x01(\x08H\x00R\texception\x12k\n\x11\x61wait_termination\x18\t \x01(\x0b\x32<.spark.connect.StreamingQueryCommand.AwaitTerminationCommandH\x00R\x10\x61waitTermination\x1a,\n\x0e\x45xplainCommand\x12\x1a\n\x08\x65xtended\x18\x01 \x01(\x08R\x08\x65xtended\x1aL\n\x17\x41waitTerminationCommand\x12"\n\ntimeout_ms\x18\x02 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_msB\t\n\x07\x63ommand"\xf5\x08\n\x1bStreamingQueryCommandResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12Q\n\x06status\x18\x02 \x01(\x0b\x32\x37.spark.connect.StreamingQueryCommandResult.StatusResultH\x00R\x06status\x12j\n\x0frecent_progress\x18\x03 \x01(\x0b\x32?.spark.connect.StreamingQueryCommandResult.RecentProgressResultH\x00R\x0erecentProgress\x12T\n\x07\x65xplain\x18\x04 \x01(\x0b\x32\x38.spark.connect.StreamingQueryCommandResult.ExplainResultH\x00R\x07\x65xplain\x12Z\n\texception\x18\x05 \x01(\x0b\x32:.spark.connect.StreamingQueryCommandResult.ExceptionResultH\x00R\texception\x12p\n\x11\x61wait_termination\x18\x06 \x01(\x0b\x32\x41.spark.connect.StreamingQueryCommandResult.AwaitTerminationResultH\x00R\x10\x61waitTermination\x1a\xaa\x01\n\x0cStatusResult\x12%\n\x0estatus_message\x18\x01 \x01(\tR\rstatusMessage\x12*\n\x11is_data_available\x18\x02 \x01(\x08R\x0fisDataAvailable\x12*\n\x11is_trigger_active\x18\x03 \x01(\x08R\x0fisTriggerActive\x12\x1b\n\tis_active\x18\x04 \x01(\x08R\x08isActive\x1aH\n\x14RecentProgressResult\x12\x30\n\x14recent_progress_json\x18\x05 \x03(\tR\x12recentProgressJson\x1a\'\n\rExplainResult\x12\x16\n\x06result\x18\x01 \x01(\tR\x06result\x1a\xc5\x01\n\x0f\x45xceptionResult\x12\x30\n\x11\x65xception_message\x18\x01 \x01(\tH\x00R\x10\x65xceptionMessage\x88\x01\x01\x12$\n\x0b\x65rror_class\x18\x02 \x01(\tH\x01R\nerrorClass\x88\x01\x01\x12$\n\x0bstack_trace\x18\x03 \x01(\tH\x02R\nstackTrace\x88\x01\x01\x42\x14\n\x12_exception_messageB\x0e\n\x0c_error_classB\x0e\n\x0c_stack_trace\x1a\x38\n\x16\x41waitTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminatedB\r\n\x0bresult_type"\xbd\x06\n\x1cStreamingQueryManagerCommand\x12\x18\n\x06\x61\x63tive\x18\x01 \x01(\x08H\x00R\x06\x61\x63tive\x12\x1d\n\tget_query\x18\x02 \x01(\tH\x00R\x08getQuery\x12|\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32\x46.spark.connect.StreamingQueryManagerCommand.AwaitAnyTerminationCommandH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12n\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0b\x61\x64\x64Listener\x12t\n\x0fremove_listener\x18\x06 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0eremoveListener\x12\'\n\x0elist_listeners\x18\x07 \x01(\x08H\x00R\rlistListeners\x1aO\n\x1a\x41waitAnyTerminationCommand\x12"\n\ntimeout_ms\x18\x01 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_ms\x1a\xcd\x01\n\x1dStreamingQueryListenerCommand\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x12U\n\x17python_listener_payload\x18\x02 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x15pythonListenerPayload\x88\x01\x01\x12\x0e\n\x02id\x18\x03 \x01(\tR\x02idB\x1a\n\x18_python_listener_payloadB\t\n\x07\x63ommand"\xb4\x08\n"StreamingQueryManagerCommandResult\x12X\n\x06\x61\x63tive\x18\x01 \x01(\x0b\x32>.spark.connect.StreamingQueryManagerCommandResult.ActiveResultH\x00R\x06\x61\x63tive\x12`\n\x05query\x18\x02 \x01(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceH\x00R\x05query\x12\x81\x01\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32K.spark.connect.StreamingQueryManagerCommandResult.AwaitAnyTerminationResultH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12#\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x08H\x00R\x0b\x61\x64\x64Listener\x12)\n\x0fremove_listener\x18\x06 \x01(\x08H\x00R\x0eremoveListener\x12{\n\x0elist_listeners\x18\x07 \x01(\x0b\x32R.spark.connect.StreamingQueryManagerCommandResult.ListStreamingQueryListenerResultH\x00R\rlistListeners\x1a\x7f\n\x0c\x41\x63tiveResult\x12o\n\x0e\x61\x63tive_queries\x18\x01 \x03(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceR\ractiveQueries\x1as\n\x16StreamingQueryInstance\x12\x37\n\x02id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x02id\x12\x17\n\x04name\x18\x02 \x01(\tH\x00R\x04name\x88\x01\x01\x42\x07\n\x05_name\x1a;\n\x19\x41waitAnyTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminated\x1aK\n\x1eStreamingQueryListenerInstance\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x1a\x45\n ListStreamingQueryListenerResult\x12!\n\x0clistener_ids\x18\x01 \x03(\tR\x0blistenerIdsB\r\n\x0bresult_type"\xad\x01\n StreamingQueryListenerBusCommand\x12;\n\x19\x61\x64\x64_listener_bus_listener\x18\x01 \x01(\x08H\x00R\x16\x61\x64\x64ListenerBusListener\x12\x41\n\x1cremove_listener_bus_listener\x18\x02 \x01(\x08H\x00R\x19removeListenerBusListenerB\t\n\x07\x63ommand"\x83\x01\n\x1bStreamingQueryListenerEvent\x12\x1d\n\nevent_json\x18\x01 \x01(\tR\teventJson\x12\x45\n\nevent_type\x18\x02 \x01(\x0e\x32&.spark.connect.StreamingQueryEventTypeR\teventType"\xcc\x01\n"StreamingQueryListenerEventsResult\x12\x42\n\x06\x65vents\x18\x01 \x03(\x0b\x32*.spark.connect.StreamingQueryListenerEventR\x06\x65vents\x12\x42\n\x1blistener_bus_listener_added\x18\x02 \x01(\x08H\x00R\x18listenerBusListenerAdded\x88\x01\x01\x42\x1e\n\x1c_listener_bus_listener_added"\x15\n\x13GetResourcesCommand"\xd4\x01\n\x19GetResourcesCommandResult\x12U\n\tresources\x18\x01 \x03(\x0b\x32\x37.spark.connect.GetResourcesCommandResult.ResourcesEntryR\tresources\x1a`\n\x0eResourcesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32".spark.connect.ResourceInformationR\x05value:\x02\x38\x01"X\n\x1c\x43reateResourceProfileCommand\x12\x38\n\x07profile\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ResourceProfileR\x07profile"C\n"CreateResourceProfileCommandResult\x12\x1d\n\nprofile_id\x18\x01 \x01(\x05R\tprofileId"d\n!RemoveCachedRemoteRelationCommand\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation"\xcd\x01\n\x11\x43heckpointCommand\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x14\n\x05local\x18\x02 \x01(\x08R\x05local\x12\x14\n\x05\x65\x61ger\x18\x03 \x01(\x08R\x05\x65\x61ger\x12\x45\n\rstorage_level\x18\x04 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xe8\x03\n\x15MergeIntoTableCommand\x12*\n\x11target_table_name\x18\x01 \x01(\tR\x0ftargetTableName\x12\x43\n\x11source_table_plan\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x0fsourceTablePlan\x12\x42\n\x0fmerge_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0emergeCondition\x12>\n\rmatch_actions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0cmatchActions\x12I\n\x13not_matched_actions\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x11notMatchedActions\x12[\n\x1dnot_matched_by_source_actions\x18\x06 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x19notMatchedBySourceActions\x12\x32\n\x15with_schema_evolution\x18\x07 \x01(\x08R\x13withSchemaEvolution"\xd4\x01\n\x16\x45xecuteExternalCommand\x12\x16\n\x06runner\x18\x01 \x01(\tR\x06runner\x12\x18\n\x07\x63ommand\x18\x02 \x01(\tR\x07\x63ommand\x12L\n\x07options\x18\x03 \x03(\x0b\x32\x32.spark.connect.ExecuteExternalCommand.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01*\x85\x01\n\x17StreamingQueryEventType\x12\x1e\n\x1aQUERY_PROGRESS_UNSPECIFIED\x10\x00\x12\x18\n\x14QUERY_PROGRESS_EVENT\x10\x01\x12\x1a\n\x16QUERY_TERMINATED_EVENT\x10\x02\x12\x14\n\x10QUERY_IDLE_EVENT\x10\x03\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -83,118 +83,124 @@ _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_options = b"8\001" _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._loaded_options = None _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_options = b"8\001" - _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_start = 11920 - _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_end = 12053 + _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_start = 12339 + _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_end = 12472 _globals["_COMMAND"]._serialized_start = 222 - _globals["_COMMAND"]._serialized_end = 2137 - _globals["_SQLCOMMAND"]._serialized_start = 2140 - _globals["_SQLCOMMAND"]._serialized_end = 2694 - _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_start = 2510 - _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_end = 2600 - _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_start = 2602 - _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_end = 2694 - _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_start = 2697 - _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_end = 2847 - _globals["_WRITEOPERATION"]._serialized_start = 2850 - _globals["_WRITEOPERATION"]._serialized_end = 4000 - _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_start = 3424 - _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_end = 3482 - _globals["_WRITEOPERATION_SAVETABLE"]._serialized_start = 3485 - _globals["_WRITEOPERATION_SAVETABLE"]._serialized_end = 3743 - _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_start = 3619 - _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_end = 3743 - _globals["_WRITEOPERATION_BUCKETBY"]._serialized_start = 3745 - _globals["_WRITEOPERATION_BUCKETBY"]._serialized_end = 3836 - _globals["_WRITEOPERATION_SAVEMODE"]._serialized_start = 3839 - _globals["_WRITEOPERATION_SAVEMODE"]._serialized_end = 3976 - _globals["_WRITEOPERATIONV2"]._serialized_start = 4003 - _globals["_WRITEOPERATIONV2"]._serialized_end = 4915 - _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_start = 3424 - _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_end = 3482 - _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_start = 4674 - _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_end = 4740 - _globals["_WRITEOPERATIONV2_MODE"]._serialized_start = 4743 - _globals["_WRITEOPERATIONV2_MODE"]._serialized_end = 4902 - _globals["_WRITESTREAMOPERATIONSTART"]._serialized_start = 4918 - _globals["_WRITESTREAMOPERATIONSTART"]._serialized_end = 5833 - _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_start = 3424 - _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_end = 3482 - _globals["_STREAMINGFOREACHFUNCTION"]._serialized_start = 5836 - _globals["_STREAMINGFOREACHFUNCTION"]._serialized_end = 6015 - _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_start = 6018 - _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_end = 6230 - _globals["_STREAMINGQUERYINSTANCEID"]._serialized_start = 6232 - _globals["_STREAMINGQUERYINSTANCEID"]._serialized_end = 6297 - _globals["_STREAMINGQUERYCOMMAND"]._serialized_start = 6300 - _globals["_STREAMINGQUERYCOMMAND"]._serialized_end = 6932 - _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_start = 6799 - _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_end = 6843 - _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_start = 6845 - _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_end = 6921 - _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_start = 6935 - _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_end = 8076 - _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_start = 7518 - _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_end = 7688 - _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_start = 7690 - _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_end = 7762 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_start = 7764 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_end = 7803 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_start = 7806 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_end = 8003 - _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_start = 8005 - _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_end = 8061 - _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_start = 8079 - _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_end = 8908 - _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_start = 8610 - _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_end = 8689 - _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_start = 8692 - _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_end = 8897 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_start = 8911 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_end = 9987 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_start = 9519 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_end = 9646 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_start = 9648 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_end = 9763 + _globals["_COMMAND"]._serialized_end = 2344 + _globals["_CREATEBROADCASTCOMMAND"]._serialized_start = 2346 + _globals["_CREATEBROADCASTCOMMAND"]._serialized_end = 2438 + _globals["_UNPERSISTBROADCASTCOMMAND"]._serialized_start = 2440 + _globals["_UNPERSISTBROADCASTCOMMAND"]._serialized_end = 2556 + _globals["_SQLCOMMAND"]._serialized_start = 2559 + _globals["_SQLCOMMAND"]._serialized_end = 3113 + _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_start = 2929 + _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_end = 3019 + _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_start = 3021 + _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_end = 3113 + _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_start = 3116 + _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_end = 3266 + _globals["_WRITEOPERATION"]._serialized_start = 3269 + _globals["_WRITEOPERATION"]._serialized_end = 4419 + _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_start = 3843 + _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_end = 3901 + _globals["_WRITEOPERATION_SAVETABLE"]._serialized_start = 3904 + _globals["_WRITEOPERATION_SAVETABLE"]._serialized_end = 4162 + _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_start = 4038 + _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_end = 4162 + _globals["_WRITEOPERATION_BUCKETBY"]._serialized_start = 4164 + _globals["_WRITEOPERATION_BUCKETBY"]._serialized_end = 4255 + _globals["_WRITEOPERATION_SAVEMODE"]._serialized_start = 4258 + _globals["_WRITEOPERATION_SAVEMODE"]._serialized_end = 4395 + _globals["_WRITEOPERATIONV2"]._serialized_start = 4422 + _globals["_WRITEOPERATIONV2"]._serialized_end = 5334 + _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_start = 3843 + _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_end = 3901 + _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_start = 5093 + _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_end = 5159 + _globals["_WRITEOPERATIONV2_MODE"]._serialized_start = 5162 + _globals["_WRITEOPERATIONV2_MODE"]._serialized_end = 5321 + _globals["_WRITESTREAMOPERATIONSTART"]._serialized_start = 5337 + _globals["_WRITESTREAMOPERATIONSTART"]._serialized_end = 6252 + _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_start = 3843 + _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_end = 3901 + _globals["_STREAMINGFOREACHFUNCTION"]._serialized_start = 6255 + _globals["_STREAMINGFOREACHFUNCTION"]._serialized_end = 6434 + _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_start = 6437 + _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_end = 6649 + _globals["_STREAMINGQUERYINSTANCEID"]._serialized_start = 6651 + _globals["_STREAMINGQUERYINSTANCEID"]._serialized_end = 6716 + _globals["_STREAMINGQUERYCOMMAND"]._serialized_start = 6719 + _globals["_STREAMINGQUERYCOMMAND"]._serialized_end = 7351 + _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_start = 7218 + _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_end = 7262 + _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_start = 7264 + _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_end = 7340 + _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_start = 7354 + _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_end = 8495 + _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_start = 7937 + _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_end = 8107 + _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_start = 8109 + _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_end = 8181 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_start = 8183 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_end = 8222 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_start = 8225 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_end = 8422 + _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_start = 8424 + _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_end = 8480 + _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_start = 8498 + _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_end = 9327 + _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_start = 9029 + _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_end = 9108 + _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_start = 9111 + _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_end = 9316 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_start = 9330 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_end = 10406 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_start = 9938 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_end = 10065 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_start = 10067 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_end = 10182 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT" - ]._serialized_start = 9765 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT"]._serialized_end = 9824 + ]._serialized_start = 10184 + _globals[ + "_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT" + ]._serialized_end = 10243 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE" - ]._serialized_start = 9826 + ]._serialized_start = 10245 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE" - ]._serialized_end = 9901 + ]._serialized_end = 10320 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_LISTSTREAMINGQUERYLISTENERRESULT" - ]._serialized_start = 9903 + ]._serialized_start = 10322 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_LISTSTREAMINGQUERYLISTENERRESULT" - ]._serialized_end = 9972 - _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_start = 9990 - _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_end = 10163 - _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_start = 10166 - _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_end = 10297 - _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_start = 10300 - _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_end = 10504 - _globals["_GETRESOURCESCOMMAND"]._serialized_start = 10506 - _globals["_GETRESOURCESCOMMAND"]._serialized_end = 10527 - _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_start = 10530 - _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_end = 10742 - _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_start = 10646 - _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_end = 10742 - _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_start = 10744 - _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_end = 10832 - _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_start = 10834 - _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_end = 10901 - _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_start = 10903 - _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_end = 11003 - _globals["_CHECKPOINTCOMMAND"]._serialized_start = 11006 - _globals["_CHECKPOINTCOMMAND"]._serialized_end = 11211 - _globals["_MERGEINTOTABLECOMMAND"]._serialized_start = 11214 - _globals["_MERGEINTOTABLECOMMAND"]._serialized_end = 11702 - _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_start = 11705 - _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_end = 11917 - _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_start = 3424 - _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_end = 3482 + ]._serialized_end = 10391 + _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_start = 10409 + _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_end = 10582 + _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_start = 10585 + _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_end = 10716 + _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_start = 10719 + _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_end = 10923 + _globals["_GETRESOURCESCOMMAND"]._serialized_start = 10925 + _globals["_GETRESOURCESCOMMAND"]._serialized_end = 10946 + _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_start = 10949 + _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_end = 11161 + _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_start = 11065 + _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_end = 11161 + _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_start = 11163 + _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_end = 11251 + _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_start = 11253 + _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_end = 11320 + _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_start = 11322 + _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_end = 11422 + _globals["_CHECKPOINTCOMMAND"]._serialized_start = 11425 + _globals["_CHECKPOINTCOMMAND"]._serialized_end = 11630 + _globals["_MERGEINTOTABLECOMMAND"]._serialized_start = 11633 + _globals["_MERGEINTOTABLECOMMAND"]._serialized_end = 12121 + _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_start = 12124 + _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_end = 12336 + _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_start = 3843 + _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_end = 3901 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi index cc1330f11f8ac..43ba7795bbd4f 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.pyi +++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi @@ -110,6 +110,8 @@ class Command(google.protobuf.message.Message): ML_COMMAND_FIELD_NUMBER: builtins.int EXECUTE_EXTERNAL_COMMAND_FIELD_NUMBER: builtins.int PIPELINE_COMMAND_FIELD_NUMBER: builtins.int + CREATE_BROADCAST_COMMAND_FIELD_NUMBER: builtins.int + UNPERSIST_BROADCAST_COMMAND_FIELD_NUMBER: builtins.int EXTENSION_FIELD_NUMBER: builtins.int @property def register_function( @@ -158,6 +160,10 @@ class Command(google.protobuf.message.Message): @property def pipeline_command(self) -> pyspark.sql.connect.proto.pipelines_pb2.PipelineCommand: ... @property + def create_broadcast_command(self) -> global___CreateBroadcastCommand: ... + @property + def unpersist_broadcast_command(self) -> global___UnpersistBroadcastCommand: ... + @property def extension(self) -> google.protobuf.any_pb2.Any: """This field is used to mark extensions to the protocol. When plugins generate arbitrary Commands they can add them here. During the planning the correct resolution is done. @@ -189,6 +195,8 @@ class Command(google.protobuf.message.Message): ml_command: pyspark.sql.connect.proto.ml_pb2.MlCommand | None = ..., execute_external_command: global___ExecuteExternalCommand | None = ..., pipeline_command: pyspark.sql.connect.proto.pipelines_pb2.PipelineCommand | None = ..., + create_broadcast_command: global___CreateBroadcastCommand | None = ..., + unpersist_broadcast_command: global___UnpersistBroadcastCommand | None = ..., extension: google.protobuf.any_pb2.Any | None = ..., ) -> None: ... def HasField( @@ -198,6 +206,8 @@ class Command(google.protobuf.message.Message): b"checkpoint_command", "command_type", b"command_type", + "create_broadcast_command", + b"create_broadcast_command", "create_dataframe_view", b"create_dataframe_view", "create_resource_profile_command", @@ -230,6 +240,8 @@ class Command(google.protobuf.message.Message): b"streaming_query_listener_bus_command", "streaming_query_manager_command", b"streaming_query_manager_command", + "unpersist_broadcast_command", + b"unpersist_broadcast_command", "write_operation", b"write_operation", "write_operation_v2", @@ -245,6 +257,8 @@ class Command(google.protobuf.message.Message): b"checkpoint_command", "command_type", b"command_type", + "create_broadcast_command", + b"create_broadcast_command", "create_dataframe_view", b"create_dataframe_view", "create_resource_profile_command", @@ -277,6 +291,8 @@ class Command(google.protobuf.message.Message): b"streaming_query_listener_bus_command", "streaming_query_manager_command", b"streaming_query_manager_command", + "unpersist_broadcast_command", + b"unpersist_broadcast_command", "write_operation", b"write_operation", "write_operation_v2", @@ -308,6 +324,8 @@ class Command(google.protobuf.message.Message): "ml_command", "execute_external_command", "pipeline_command", + "create_broadcast_command", + "unpersist_broadcast_command", "extension", ] | None @@ -315,6 +333,66 @@ class Command(google.protobuf.message.Message): global___Command = Command +class CreateBroadcastCommand(google.protobuf.message.Message): + """(SPARK-51705) Create a broadcast variable from an already-uploaded cache/ artifact. + The client uploads cloudpickle(value) through the existing cache artifact channel and then + sends this command with the returned hash. The server materializes a PythonBroadcast on the + live driver SparkContext and returns a CreateBroadcastResult with the driver-side broadcast id. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ARTIFACT_HASH_FIELD_NUMBER: builtins.int + SIZE_BYTES_FIELD_NUMBER: builtins.int + artifact_hash: builtins.str + """(Required) sha256 hash returned by client.cache_artifact(cloudpickle(value)).""" + size_bytes: builtins.int + """(Optional) Uncompressed byte size, used to enforce the BROADCAST_VALUE_TOO_LARGE quota.""" + def __init__( + self, + *, + artifact_hash: builtins.str = ..., + size_bytes: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "artifact_hash", b"artifact_hash", "size_bytes", b"size_bytes" + ], + ) -> None: ... + +global___CreateBroadcastCommand = CreateBroadcastCommand + +class UnpersistBroadcastCommand(google.protobuf.message.Message): + """(SPARK-51705) Release a broadcast variable created over Spark Connect.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + BROADCAST_ID_FIELD_NUMBER: builtins.int + BLOCKING_FIELD_NUMBER: builtins.int + DESTROY_FIELD_NUMBER: builtins.int + broadcast_id: builtins.int + """(Required) The driver-side broadcast id returned by CreateBroadcastResult.""" + blocking: builtins.bool + """(Optional) Whether to block until unpersisting has completed.""" + destroy: builtins.bool + """(Optional) Whether to destroy (not just unpersist) all data and metadata of the broadcast.""" + def __init__( + self, + *, + broadcast_id: builtins.int = ..., + blocking: builtins.bool = ..., + destroy: builtins.bool = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "blocking", b"blocking", "broadcast_id", b"broadcast_id", "destroy", b"destroy" + ], + ) -> None: ... + +global___UnpersistBroadcastCommand = UnpersistBroadcastCommand + class SqlCommand(google.protobuf.message.Message): """A SQL Command is used to trigger the eager evaluation of SQL commands in Spark. diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.py b/python/pyspark/sql/connect/proto/expressions_pb2.py index aa51c393c043f..28e96fb3a00bc 100644 --- a/python/pyspark/sql/connect/proto/expressions_pb2.py +++ b/python/pyspark/sql/connect/proto/expressions_pb2.py @@ -41,7 +41,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\x90<\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFunction\x12Y\n\x11\x65xpression_string\x18\x04 \x01(\x0b\x32*.spark.connect.Expression.ExpressionStringH\x00R\x10\x65xpressionString\x12S\n\x0funresolved_star\x18\x05 \x01(\x0b\x32(.spark.connect.Expression.UnresolvedStarH\x00R\x0eunresolvedStar\x12\x37\n\x05\x61lias\x18\x06 \x01(\x0b\x32\x1f.spark.connect.Expression.AliasH\x00R\x05\x61lias\x12\x34\n\x04\x63\x61st\x18\x07 \x01(\x0b\x32\x1e.spark.connect.Expression.CastH\x00R\x04\x63\x61st\x12V\n\x10unresolved_regex\x18\x08 \x01(\x0b\x32).spark.connect.Expression.UnresolvedRegexH\x00R\x0funresolvedRegex\x12\x44\n\nsort_order\x18\t \x01(\x0b\x32#.spark.connect.Expression.SortOrderH\x00R\tsortOrder\x12S\n\x0flambda_function\x18\n \x01(\x0b\x32(.spark.connect.Expression.LambdaFunctionH\x00R\x0elambdaFunction\x12:\n\x06window\x18\x0b \x01(\x0b\x32 .spark.connect.Expression.WindowH\x00R\x06window\x12l\n\x18unresolved_extract_value\x18\x0c \x01(\x0b\x32\x30.spark.connect.Expression.UnresolvedExtractValueH\x00R\x16unresolvedExtractValue\x12M\n\rupdate_fields\x18\r \x01(\x0b\x32&.spark.connect.Expression.UpdateFieldsH\x00R\x0cupdateFields\x12\x82\x01\n unresolved_named_lambda_variable\x18\x0e \x01(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableH\x00R\x1dunresolvedNamedLambdaVariable\x12~\n#common_inline_user_defined_function\x18\x0f \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x1f\x63ommonInlineUserDefinedFunction\x12\x42\n\rcall_function\x18\x10 \x01(\x0b\x32\x1b.spark.connect.CallFunctionH\x00R\x0c\x63\x61llFunction\x12\x64\n\x19named_argument_expression\x18\x11 \x01(\x0b\x32&.spark.connect.NamedArgumentExpressionH\x00R\x17namedArgumentExpression\x12?\n\x0cmerge_action\x18\x13 \x01(\x0b\x32\x1a.spark.connect.MergeActionH\x00R\x0bmergeAction\x12g\n\x1atyped_aggregate_expression\x18\x14 \x01(\x0b\x32\'.spark.connect.TypedAggregateExpressionH\x00R\x18typedAggregateExpression\x12T\n\x13subquery_expression\x18\x15 \x01(\x0b\x32!.spark.connect.SubqueryExpressionH\x00R\x12subqueryExpression\x12s\n\x1b\x64irect_shuffle_partition_id\x18\x16 \x01(\x0b\x32\x32.spark.connect.Expression.DirectShufflePartitionIDH\x00R\x18\x64irectShufflePartitionId\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x1a\x8f\x06\n\x06Window\x12\x42\n\x0fwindow_function\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0ewindowFunction\x12@\n\x0epartition_spec\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\rpartitionSpec\x12\x42\n\norder_spec\x18\x03 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\torderSpec\x12K\n\nframe_spec\x18\x04 \x01(\x0b\x32,.spark.connect.Expression.Window.WindowFrameR\tframeSpec\x1a\xed\x03\n\x0bWindowFrame\x12U\n\nframe_type\x18\x01 \x01(\x0e\x32\x36.spark.connect.Expression.Window.WindowFrame.FrameTypeR\tframeType\x12P\n\x05lower\x18\x02 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05lower\x12P\n\x05upper\x18\x03 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05upper\x1a\x91\x01\n\rFrameBoundary\x12!\n\x0b\x63urrent_row\x18\x01 \x01(\x08H\x00R\ncurrentRow\x12\x1e\n\tunbounded\x18\x02 \x01(\x08H\x00R\tunbounded\x12\x31\n\x05value\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\x05valueB\n\n\x08\x62oundary"O\n\tFrameType\x12\x18\n\x14\x46RAME_TYPE_UNDEFINED\x10\x00\x12\x12\n\x0e\x46RAME_TYPE_ROW\x10\x01\x12\x14\n\x10\x46RAME_TYPE_RANGE\x10\x02\x1a\xa9\x03\n\tSortOrder\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12O\n\tdirection\x18\x02 \x01(\x0e\x32\x31.spark.connect.Expression.SortOrder.SortDirectionR\tdirection\x12U\n\rnull_ordering\x18\x03 \x01(\x0e\x32\x30.spark.connect.Expression.SortOrder.NullOrderingR\x0cnullOrdering"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"U\n\x0cNullOrdering\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02\x1aK\n\x18\x44irectShufflePartitionID\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x1a\xbb\x02\n\x04\x43\x61st\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12-\n\x04type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04type\x12\x1b\n\x08type_str\x18\x03 \x01(\tH\x00R\x07typeStr\x12\x44\n\teval_mode\x18\x04 \x01(\x0e\x32\'.spark.connect.Expression.Cast.EvalModeR\x08\x65valMode"b\n\x08\x45valMode\x12\x19\n\x15\x45VAL_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10\x45VAL_MODE_LEGACY\x10\x01\x12\x12\n\x0e\x45VAL_MODE_ANSI\x10\x02\x12\x11\n\rEVAL_MODE_TRY\x10\x03\x42\x0e\n\x0c\x63\x61st_to_type\x1a\x9c\x15\n\x07Literal\x12-\n\x04null\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04null\x12\x18\n\x06\x62inary\x18\x02 \x01(\x0cH\x00R\x06\x62inary\x12\x1a\n\x07\x62oolean\x18\x03 \x01(\x08H\x00R\x07\x62oolean\x12\x14\n\x04\x62yte\x18\x04 \x01(\x05H\x00R\x04\x62yte\x12\x16\n\x05short\x18\x05 \x01(\x05H\x00R\x05short\x12\x1a\n\x07integer\x18\x06 \x01(\x05H\x00R\x07integer\x12\x14\n\x04long\x18\x07 \x01(\x03H\x00R\x04long\x12\x16\n\x05\x66loat\x18\n \x01(\x02H\x00R\x05\x66loat\x12\x18\n\x06\x64ouble\x18\x0b \x01(\x01H\x00R\x06\x64ouble\x12\x45\n\x07\x64\x65\x63imal\x18\x0c \x01(\x0b\x32).spark.connect.Expression.Literal.DecimalH\x00R\x07\x64\x65\x63imal\x12\x18\n\x06string\x18\r \x01(\tH\x00R\x06string\x12\x14\n\x04\x64\x61te\x18\x10 \x01(\x05H\x00R\x04\x64\x61te\x12\x1e\n\ttimestamp\x18\x11 \x01(\x03H\x00R\ttimestamp\x12%\n\rtimestamp_ntz\x18\x12 \x01(\x03H\x00R\x0ctimestampNtz\x12\x61\n\x11\x63\x61lendar_interval\x18\x13 \x01(\x0b\x32\x32.spark.connect.Expression.Literal.CalendarIntervalH\x00R\x10\x63\x61lendarInterval\x12\x30\n\x13year_month_interval\x18\x14 \x01(\x05H\x00R\x11yearMonthInterval\x12,\n\x11\x64\x61y_time_interval\x18\x15 \x01(\x03H\x00R\x0f\x64\x61yTimeInterval\x12?\n\x05\x61rray\x18\x16 \x01(\x0b\x32\'.spark.connect.Expression.Literal.ArrayH\x00R\x05\x61rray\x12\x39\n\x03map\x18\x17 \x01(\x0b\x32%.spark.connect.Expression.Literal.MapH\x00R\x03map\x12\x42\n\x06struct\x18\x18 \x01(\x0b\x32(.spark.connect.Expression.Literal.StructH\x00R\x06struct\x12\x61\n\x11specialized_array\x18\x19 \x01(\x0b\x32\x32.spark.connect.Expression.Literal.SpecializedArrayH\x00R\x10specializedArray\x12<\n\x04time\x18\x1a \x01(\x0b\x32&.spark.connect.Expression.Literal.TimeH\x00R\x04time\x12\x65\n\x13timestamp_ntz_nanos\x18\x1d \x01(\x0b\x32\x33.spark.connect.Expression.Literal.TimestampNTZNanosH\x00R\x11timestampNtzNanos\x12\x65\n\x13timestamp_ltz_nanos\x18\x1e \x01(\x0b\x32\x33.spark.connect.Expression.Literal.TimestampLTZNanosH\x00R\x11timestampLtzNanos\x12\x34\n\tdata_type\x18\x64 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x08\x64\x61taType\x1au\n\x07\x44\x65\x63imal\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\x12!\n\tprecision\x18\x02 \x01(\x05H\x00R\tprecision\x88\x01\x01\x12\x19\n\x05scale\x18\x03 \x01(\x05H\x01R\x05scale\x88\x01\x01\x42\x0c\n\n_precisionB\x08\n\x06_scale\x1a\x62\n\x10\x43\x61lendarInterval\x12\x16\n\x06months\x18\x01 \x01(\x05R\x06months\x12\x12\n\x04\x64\x61ys\x18\x02 \x01(\x05R\x04\x64\x61ys\x12"\n\x0cmicroseconds\x18\x03 \x01(\x03R\x0cmicroseconds\x1a\x86\x01\n\x05\x41rray\x12>\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\x0b\x65lementType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lements\x1a\xeb\x01\n\x03Map\x12\x36\n\x08key_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\x07keyType\x12:\n\nvalue_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\tvalueType\x12\x35\n\x04keys\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x04keys\x12\x39\n\x06values\x18\x04 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1a\x85\x01\n\x06Struct\x12<\n\x0bstruct_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\nstructType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lements\x1a\xc0\x02\n\x10SpecializedArray\x12,\n\x05\x62ools\x18\x01 \x01(\x0b\x32\x14.spark.connect.BoolsH\x00R\x05\x62ools\x12)\n\x04ints\x18\x02 \x01(\x0b\x32\x13.spark.connect.IntsH\x00R\x04ints\x12,\n\x05longs\x18\x03 \x01(\x0b\x32\x14.spark.connect.LongsH\x00R\x05longs\x12/\n\x06\x66loats\x18\x04 \x01(\x0b\x32\x15.spark.connect.FloatsH\x00R\x06\x66loats\x12\x32\n\x07\x64oubles\x18\x05 \x01(\x0b\x32\x16.spark.connect.DoublesH\x00R\x07\x64oubles\x12\x32\n\x07strings\x18\x06 \x01(\x0b\x32\x16.spark.connect.StringsH\x00R\x07stringsB\x0c\n\nvalue_type\x1aK\n\x04Time\x12\x12\n\x04nano\x18\x01 \x01(\x03R\x04nano\x12!\n\tprecision\x18\x02 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precision\x1a\x95\x01\n\x11TimestampNTZNanos\x12!\n\x0c\x65poch_micros\x18\x01 \x01(\x03R\x0b\x65pochMicros\x12,\n\x12nanos_within_micro\x18\x02 \x01(\x05R\x10nanosWithinMicro\x12!\n\tprecision\x18\x03 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precision\x1a\x95\x01\n\x11TimestampLTZNanos\x12!\n\x0c\x65poch_micros\x18\x01 \x01(\x03R\x0b\x65pochMicros\x12,\n\x12nanos_within_micro\x18\x02 \x01(\x05R\x10nanosWithinMicro\x12!\n\tprecision\x18\x03 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precisionB\x0e\n\x0cliteral_typeJ\x04\x08\x1b\x10\x1cJ\x04\x08\x1c\x10\x1d\x1a\xba\x01\n\x13UnresolvedAttribute\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x12\x31\n\x12is_metadata_column\x18\x03 \x01(\x08H\x01R\x10isMetadataColumn\x88\x01\x01\x42\n\n\x08_plan_idB\x15\n\x13_is_metadata_column\x1a\x82\x02\n\x12UnresolvedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x1f\n\x0bis_distinct\x18\x03 \x01(\x08R\nisDistinct\x12\x37\n\x18is_user_defined_function\x18\x04 \x01(\x08R\x15isUserDefinedFunction\x12$\n\x0bis_internal\x18\x05 \x01(\x08H\x00R\nisInternal\x88\x01\x01\x42\x0e\n\x0c_is_internal\x1a\x32\n\x10\x45xpressionString\x12\x1e\n\nexpression\x18\x01 \x01(\tR\nexpression\x1a|\n\x0eUnresolvedStar\x12,\n\x0funparsed_target\x18\x01 \x01(\tH\x00R\x0eunparsedTarget\x88\x01\x01\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x01R\x06planId\x88\x01\x01\x42\x12\n\x10_unparsed_targetB\n\n\x08_plan_id\x1aV\n\x0fUnresolvedRegex\x12\x19\n\x08\x63ol_name\x18\x01 \x01(\tR\x07\x63olName\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x42\n\n\x08_plan_id\x1a\x84\x01\n\x16UnresolvedExtractValue\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12\x39\n\nextraction\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nextraction\x1a\xbb\x01\n\x0cUpdateFields\x12\x46\n\x11struct_expression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x10structExpression\x12\x1d\n\nfield_name\x18\x02 \x01(\tR\tfieldName\x12\x44\n\x10value_expression\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0fvalueExpression\x1ax\n\x05\x41lias\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12\x12\n\x04name\x18\x02 \x03(\tR\x04name\x12\x1f\n\x08metadata\x18\x03 \x01(\tH\x00R\x08metadata\x88\x01\x01\x42\x0b\n\t_metadata\x1a\x9e\x01\n\x0eLambdaFunction\x12\x35\n\x08\x66unction\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x08\x66unction\x12U\n\targuments\x18\x02 \x03(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableR\targuments\x1a>\n\x1dUnresolvedNamedLambdaVariable\x12\x1d\n\nname_parts\x18\x01 \x03(\tR\tnamePartsB\x0b\n\texpr_type"A\n\x10\x45xpressionCommon\x12-\n\x06origin\x18\x01 \x01(\x0b\x32\x15.spark.connect.OriginR\x06origin"\x8d\x03\n\x1f\x43ommonInlineUserDefinedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12$\n\rdeterministic\x18\x02 \x01(\x08R\rdeterministic\x12\x37\n\targuments\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x39\n\npython_udf\x18\x04 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\tpythonUdf\x12I\n\x10scalar_scala_udf\x18\x05 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\x0escalarScalaUdf\x12\x33\n\x08java_udf\x18\x06 \x01(\x0b\x32\x16.spark.connect.JavaUDFH\x00R\x07javaUdf\x12\x1f\n\x0bis_distinct\x18\x07 \x01(\x08R\nisDistinctB\n\n\x08\x66unction"\xcc\x01\n\tPythonUDF\x12\x38\n\x0boutput_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1b\n\teval_type\x18\x02 \x01(\x05R\x08\x65valType\x12\x18\n\x07\x63ommand\x18\x03 \x01(\x0cR\x07\x63ommand\x12\x1d\n\npython_ver\x18\x04 \x01(\tR\tpythonVer\x12/\n\x13\x61\x64\x64itional_includes\x18\x05 \x03(\tR\x12\x61\x64\x64itionalIncludes"\xd6\x01\n\x0eScalarScalaUDF\x12\x18\n\x07payload\x18\x01 \x01(\x0cR\x07payload\x12\x37\n\ninputTypes\x18\x02 \x03(\x0b\x32\x17.spark.connect.DataTypeR\ninputTypes\x12\x37\n\noutputType\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1a\n\x08nullable\x18\x04 \x01(\x08R\x08nullable\x12\x1c\n\taggregate\x18\x05 \x01(\x08R\taggregate"\x95\x01\n\x07JavaUDF\x12\x1d\n\nclass_name\x18\x01 \x01(\tR\tclassName\x12=\n\x0boutput_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\noutputType\x88\x01\x01\x12\x1c\n\taggregate\x18\x03 \x01(\x08R\taggregateB\x0e\n\x0c_output_type"c\n\x18TypedAggregateExpression\x12G\n\x10scalar_scala_udf\x18\x01 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFR\x0escalarScalaUdf"l\n\x0c\x43\x61llFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"\\\n\x17NamedArgumentExpression\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\x80\x04\n\x0bMergeAction\x12\x46\n\x0b\x61\x63tion_type\x18\x01 \x01(\x0e\x32%.spark.connect.MergeAction.ActionTypeR\nactionType\x12<\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\tcondition\x88\x01\x01\x12G\n\x0b\x61ssignments\x18\x03 \x03(\x0b\x32%.spark.connect.MergeAction.AssignmentR\x0b\x61ssignments\x1aj\n\nAssignment\x12+\n\x03key\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\xa7\x01\n\nActionType\x12\x17\n\x13\x41\x43TION_TYPE_INVALID\x10\x00\x12\x16\n\x12\x41\x43TION_TYPE_DELETE\x10\x01\x12\x16\n\x12\x41\x43TION_TYPE_INSERT\x10\x02\x12\x1b\n\x17\x41\x43TION_TYPE_INSERT_STAR\x10\x03\x12\x16\n\x12\x41\x43TION_TYPE_UPDATE\x10\x04\x12\x1b\n\x17\x41\x43TION_TYPE_UPDATE_STAR\x10\x05\x42\x0c\n\n_condition"\xc5\x05\n\x12SubqueryExpression\x12\x17\n\x07plan_id\x18\x01 \x01(\x03R\x06planId\x12S\n\rsubquery_type\x18\x02 \x01(\x0e\x32..spark.connect.SubqueryExpression.SubqueryTypeR\x0csubqueryType\x12\x62\n\x11table_arg_options\x18\x03 \x01(\x0b\x32\x31.spark.connect.SubqueryExpression.TableArgOptionsH\x00R\x0ftableArgOptions\x88\x01\x01\x12G\n\x12in_subquery_values\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x10inSubqueryValues\x1a\xea\x01\n\x0fTableArgOptions\x12@\n\x0epartition_spec\x18\x01 \x03(\x0b\x32\x19.spark.connect.ExpressionR\rpartitionSpec\x12\x42\n\norder_spec\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\torderSpec\x12\x37\n\x15with_single_partition\x18\x03 \x01(\x08H\x00R\x13withSinglePartition\x88\x01\x01\x42\x18\n\x16_with_single_partition"\x90\x01\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x12\x1b\n\x17SUBQUERY_TYPE_TABLE_ARG\x10\x03\x12\x14\n\x10SUBQUERY_TYPE_IN\x10\x04\x42\x14\n\x12_table_arg_optionsB6\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\x90<\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFunction\x12Y\n\x11\x65xpression_string\x18\x04 \x01(\x0b\x32*.spark.connect.Expression.ExpressionStringH\x00R\x10\x65xpressionString\x12S\n\x0funresolved_star\x18\x05 \x01(\x0b\x32(.spark.connect.Expression.UnresolvedStarH\x00R\x0eunresolvedStar\x12\x37\n\x05\x61lias\x18\x06 \x01(\x0b\x32\x1f.spark.connect.Expression.AliasH\x00R\x05\x61lias\x12\x34\n\x04\x63\x61st\x18\x07 \x01(\x0b\x32\x1e.spark.connect.Expression.CastH\x00R\x04\x63\x61st\x12V\n\x10unresolved_regex\x18\x08 \x01(\x0b\x32).spark.connect.Expression.UnresolvedRegexH\x00R\x0funresolvedRegex\x12\x44\n\nsort_order\x18\t \x01(\x0b\x32#.spark.connect.Expression.SortOrderH\x00R\tsortOrder\x12S\n\x0flambda_function\x18\n \x01(\x0b\x32(.spark.connect.Expression.LambdaFunctionH\x00R\x0elambdaFunction\x12:\n\x06window\x18\x0b \x01(\x0b\x32 .spark.connect.Expression.WindowH\x00R\x06window\x12l\n\x18unresolved_extract_value\x18\x0c \x01(\x0b\x32\x30.spark.connect.Expression.UnresolvedExtractValueH\x00R\x16unresolvedExtractValue\x12M\n\rupdate_fields\x18\r \x01(\x0b\x32&.spark.connect.Expression.UpdateFieldsH\x00R\x0cupdateFields\x12\x82\x01\n unresolved_named_lambda_variable\x18\x0e \x01(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableH\x00R\x1dunresolvedNamedLambdaVariable\x12~\n#common_inline_user_defined_function\x18\x0f \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x1f\x63ommonInlineUserDefinedFunction\x12\x42\n\rcall_function\x18\x10 \x01(\x0b\x32\x1b.spark.connect.CallFunctionH\x00R\x0c\x63\x61llFunction\x12\x64\n\x19named_argument_expression\x18\x11 \x01(\x0b\x32&.spark.connect.NamedArgumentExpressionH\x00R\x17namedArgumentExpression\x12?\n\x0cmerge_action\x18\x13 \x01(\x0b\x32\x1a.spark.connect.MergeActionH\x00R\x0bmergeAction\x12g\n\x1atyped_aggregate_expression\x18\x14 \x01(\x0b\x32\'.spark.connect.TypedAggregateExpressionH\x00R\x18typedAggregateExpression\x12T\n\x13subquery_expression\x18\x15 \x01(\x0b\x32!.spark.connect.SubqueryExpressionH\x00R\x12subqueryExpression\x12s\n\x1b\x64irect_shuffle_partition_id\x18\x16 \x01(\x0b\x32\x32.spark.connect.Expression.DirectShufflePartitionIDH\x00R\x18\x64irectShufflePartitionId\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x1a\x8f\x06\n\x06Window\x12\x42\n\x0fwindow_function\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0ewindowFunction\x12@\n\x0epartition_spec\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\rpartitionSpec\x12\x42\n\norder_spec\x18\x03 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\torderSpec\x12K\n\nframe_spec\x18\x04 \x01(\x0b\x32,.spark.connect.Expression.Window.WindowFrameR\tframeSpec\x1a\xed\x03\n\x0bWindowFrame\x12U\n\nframe_type\x18\x01 \x01(\x0e\x32\x36.spark.connect.Expression.Window.WindowFrame.FrameTypeR\tframeType\x12P\n\x05lower\x18\x02 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05lower\x12P\n\x05upper\x18\x03 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05upper\x1a\x91\x01\n\rFrameBoundary\x12!\n\x0b\x63urrent_row\x18\x01 \x01(\x08H\x00R\ncurrentRow\x12\x1e\n\tunbounded\x18\x02 \x01(\x08H\x00R\tunbounded\x12\x31\n\x05value\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\x05valueB\n\n\x08\x62oundary"O\n\tFrameType\x12\x18\n\x14\x46RAME_TYPE_UNDEFINED\x10\x00\x12\x12\n\x0e\x46RAME_TYPE_ROW\x10\x01\x12\x14\n\x10\x46RAME_TYPE_RANGE\x10\x02\x1a\xa9\x03\n\tSortOrder\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12O\n\tdirection\x18\x02 \x01(\x0e\x32\x31.spark.connect.Expression.SortOrder.SortDirectionR\tdirection\x12U\n\rnull_ordering\x18\x03 \x01(\x0e\x32\x30.spark.connect.Expression.SortOrder.NullOrderingR\x0cnullOrdering"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"U\n\x0cNullOrdering\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02\x1aK\n\x18\x44irectShufflePartitionID\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x1a\xbb\x02\n\x04\x43\x61st\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12-\n\x04type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04type\x12\x1b\n\x08type_str\x18\x03 \x01(\tH\x00R\x07typeStr\x12\x44\n\teval_mode\x18\x04 \x01(\x0e\x32\'.spark.connect.Expression.Cast.EvalModeR\x08\x65valMode"b\n\x08\x45valMode\x12\x19\n\x15\x45VAL_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10\x45VAL_MODE_LEGACY\x10\x01\x12\x12\n\x0e\x45VAL_MODE_ANSI\x10\x02\x12\x11\n\rEVAL_MODE_TRY\x10\x03\x42\x0e\n\x0c\x63\x61st_to_type\x1a\x9c\x15\n\x07Literal\x12-\n\x04null\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04null\x12\x18\n\x06\x62inary\x18\x02 \x01(\x0cH\x00R\x06\x62inary\x12\x1a\n\x07\x62oolean\x18\x03 \x01(\x08H\x00R\x07\x62oolean\x12\x14\n\x04\x62yte\x18\x04 \x01(\x05H\x00R\x04\x62yte\x12\x16\n\x05short\x18\x05 \x01(\x05H\x00R\x05short\x12\x1a\n\x07integer\x18\x06 \x01(\x05H\x00R\x07integer\x12\x14\n\x04long\x18\x07 \x01(\x03H\x00R\x04long\x12\x16\n\x05\x66loat\x18\n \x01(\x02H\x00R\x05\x66loat\x12\x18\n\x06\x64ouble\x18\x0b \x01(\x01H\x00R\x06\x64ouble\x12\x45\n\x07\x64\x65\x63imal\x18\x0c \x01(\x0b\x32).spark.connect.Expression.Literal.DecimalH\x00R\x07\x64\x65\x63imal\x12\x18\n\x06string\x18\r \x01(\tH\x00R\x06string\x12\x14\n\x04\x64\x61te\x18\x10 \x01(\x05H\x00R\x04\x64\x61te\x12\x1e\n\ttimestamp\x18\x11 \x01(\x03H\x00R\ttimestamp\x12%\n\rtimestamp_ntz\x18\x12 \x01(\x03H\x00R\x0ctimestampNtz\x12\x61\n\x11\x63\x61lendar_interval\x18\x13 \x01(\x0b\x32\x32.spark.connect.Expression.Literal.CalendarIntervalH\x00R\x10\x63\x61lendarInterval\x12\x30\n\x13year_month_interval\x18\x14 \x01(\x05H\x00R\x11yearMonthInterval\x12,\n\x11\x64\x61y_time_interval\x18\x15 \x01(\x03H\x00R\x0f\x64\x61yTimeInterval\x12?\n\x05\x61rray\x18\x16 \x01(\x0b\x32\'.spark.connect.Expression.Literal.ArrayH\x00R\x05\x61rray\x12\x39\n\x03map\x18\x17 \x01(\x0b\x32%.spark.connect.Expression.Literal.MapH\x00R\x03map\x12\x42\n\x06struct\x18\x18 \x01(\x0b\x32(.spark.connect.Expression.Literal.StructH\x00R\x06struct\x12\x61\n\x11specialized_array\x18\x19 \x01(\x0b\x32\x32.spark.connect.Expression.Literal.SpecializedArrayH\x00R\x10specializedArray\x12<\n\x04time\x18\x1a \x01(\x0b\x32&.spark.connect.Expression.Literal.TimeH\x00R\x04time\x12\x65\n\x13timestamp_ntz_nanos\x18\x1d \x01(\x0b\x32\x33.spark.connect.Expression.Literal.TimestampNTZNanosH\x00R\x11timestampNtzNanos\x12\x65\n\x13timestamp_ltz_nanos\x18\x1e \x01(\x0b\x32\x33.spark.connect.Expression.Literal.TimestampLTZNanosH\x00R\x11timestampLtzNanos\x12\x34\n\tdata_type\x18\x64 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x08\x64\x61taType\x1au\n\x07\x44\x65\x63imal\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\x12!\n\tprecision\x18\x02 \x01(\x05H\x00R\tprecision\x88\x01\x01\x12\x19\n\x05scale\x18\x03 \x01(\x05H\x01R\x05scale\x88\x01\x01\x42\x0c\n\n_precisionB\x08\n\x06_scale\x1a\x62\n\x10\x43\x61lendarInterval\x12\x16\n\x06months\x18\x01 \x01(\x05R\x06months\x12\x12\n\x04\x64\x61ys\x18\x02 \x01(\x05R\x04\x64\x61ys\x12"\n\x0cmicroseconds\x18\x03 \x01(\x03R\x0cmicroseconds\x1a\x86\x01\n\x05\x41rray\x12>\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\x0b\x65lementType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lements\x1a\xeb\x01\n\x03Map\x12\x36\n\x08key_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\x07keyType\x12:\n\nvalue_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\tvalueType\x12\x35\n\x04keys\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x04keys\x12\x39\n\x06values\x18\x04 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1a\x85\x01\n\x06Struct\x12<\n\x0bstruct_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeB\x02\x18\x01R\nstructType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lements\x1a\xc0\x02\n\x10SpecializedArray\x12,\n\x05\x62ools\x18\x01 \x01(\x0b\x32\x14.spark.connect.BoolsH\x00R\x05\x62ools\x12)\n\x04ints\x18\x02 \x01(\x0b\x32\x13.spark.connect.IntsH\x00R\x04ints\x12,\n\x05longs\x18\x03 \x01(\x0b\x32\x14.spark.connect.LongsH\x00R\x05longs\x12/\n\x06\x66loats\x18\x04 \x01(\x0b\x32\x15.spark.connect.FloatsH\x00R\x06\x66loats\x12\x32\n\x07\x64oubles\x18\x05 \x01(\x0b\x32\x16.spark.connect.DoublesH\x00R\x07\x64oubles\x12\x32\n\x07strings\x18\x06 \x01(\x0b\x32\x16.spark.connect.StringsH\x00R\x07stringsB\x0c\n\nvalue_type\x1aK\n\x04Time\x12\x12\n\x04nano\x18\x01 \x01(\x03R\x04nano\x12!\n\tprecision\x18\x02 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precision\x1a\x95\x01\n\x11TimestampNTZNanos\x12!\n\x0c\x65poch_micros\x18\x01 \x01(\x03R\x0b\x65pochMicros\x12,\n\x12nanos_within_micro\x18\x02 \x01(\x05R\x10nanosWithinMicro\x12!\n\tprecision\x18\x03 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precision\x1a\x95\x01\n\x11TimestampLTZNanos\x12!\n\x0c\x65poch_micros\x18\x01 \x01(\x03R\x0b\x65pochMicros\x12,\n\x12nanos_within_micro\x18\x02 \x01(\x05R\x10nanosWithinMicro\x12!\n\tprecision\x18\x03 \x01(\x05H\x00R\tprecision\x88\x01\x01\x42\x0c\n\n_precisionB\x0e\n\x0cliteral_typeJ\x04\x08\x1b\x10\x1cJ\x04\x08\x1c\x10\x1d\x1a\xba\x01\n\x13UnresolvedAttribute\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x12\x31\n\x12is_metadata_column\x18\x03 \x01(\x08H\x01R\x10isMetadataColumn\x88\x01\x01\x42\n\n\x08_plan_idB\x15\n\x13_is_metadata_column\x1a\x82\x02\n\x12UnresolvedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x1f\n\x0bis_distinct\x18\x03 \x01(\x08R\nisDistinct\x12\x37\n\x18is_user_defined_function\x18\x04 \x01(\x08R\x15isUserDefinedFunction\x12$\n\x0bis_internal\x18\x05 \x01(\x08H\x00R\nisInternal\x88\x01\x01\x42\x0e\n\x0c_is_internal\x1a\x32\n\x10\x45xpressionString\x12\x1e\n\nexpression\x18\x01 \x01(\tR\nexpression\x1a|\n\x0eUnresolvedStar\x12,\n\x0funparsed_target\x18\x01 \x01(\tH\x00R\x0eunparsedTarget\x88\x01\x01\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x01R\x06planId\x88\x01\x01\x42\x12\n\x10_unparsed_targetB\n\n\x08_plan_id\x1aV\n\x0fUnresolvedRegex\x12\x19\n\x08\x63ol_name\x18\x01 \x01(\tR\x07\x63olName\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x42\n\n\x08_plan_id\x1a\x84\x01\n\x16UnresolvedExtractValue\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12\x39\n\nextraction\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nextraction\x1a\xbb\x01\n\x0cUpdateFields\x12\x46\n\x11struct_expression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x10structExpression\x12\x1d\n\nfield_name\x18\x02 \x01(\tR\tfieldName\x12\x44\n\x10value_expression\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0fvalueExpression\x1ax\n\x05\x41lias\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12\x12\n\x04name\x18\x02 \x03(\tR\x04name\x12\x1f\n\x08metadata\x18\x03 \x01(\tH\x00R\x08metadata\x88\x01\x01\x42\x0b\n\t_metadata\x1a\x9e\x01\n\x0eLambdaFunction\x12\x35\n\x08\x66unction\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x08\x66unction\x12U\n\targuments\x18\x02 \x03(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableR\targuments\x1a>\n\x1dUnresolvedNamedLambdaVariable\x12\x1d\n\nname_parts\x18\x01 \x03(\tR\tnamePartsB\x0b\n\texpr_type"A\n\x10\x45xpressionCommon\x12-\n\x06origin\x18\x01 \x01(\x0b\x32\x15.spark.connect.OriginR\x06origin"\x8d\x03\n\x1f\x43ommonInlineUserDefinedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12$\n\rdeterministic\x18\x02 \x01(\x08R\rdeterministic\x12\x37\n\targuments\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x39\n\npython_udf\x18\x04 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\tpythonUdf\x12I\n\x10scalar_scala_udf\x18\x05 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\x0escalarScalaUdf\x12\x33\n\x08java_udf\x18\x06 \x01(\x0b\x32\x16.spark.connect.JavaUDFH\x00R\x07javaUdf\x12\x1f\n\x0bis_distinct\x18\x07 \x01(\x08R\nisDistinctB\n\n\x08\x66unction"\xf1\x01\n\tPythonUDF\x12\x38\n\x0boutput_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1b\n\teval_type\x18\x02 \x01(\x05R\x08\x65valType\x12\x18\n\x07\x63ommand\x18\x03 \x01(\x0cR\x07\x63ommand\x12\x1d\n\npython_ver\x18\x04 \x01(\tR\tpythonVer\x12/\n\x13\x61\x64\x64itional_includes\x18\x05 \x03(\tR\x12\x61\x64\x64itionalIncludes\x12#\n\rbroadcast_ids\x18\x06 \x03(\x03R\x0c\x62roadcastIds"\xd6\x01\n\x0eScalarScalaUDF\x12\x18\n\x07payload\x18\x01 \x01(\x0cR\x07payload\x12\x37\n\ninputTypes\x18\x02 \x03(\x0b\x32\x17.spark.connect.DataTypeR\ninputTypes\x12\x37\n\noutputType\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1a\n\x08nullable\x18\x04 \x01(\x08R\x08nullable\x12\x1c\n\taggregate\x18\x05 \x01(\x08R\taggregate"\x95\x01\n\x07JavaUDF\x12\x1d\n\nclass_name\x18\x01 \x01(\tR\tclassName\x12=\n\x0boutput_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\noutputType\x88\x01\x01\x12\x1c\n\taggregate\x18\x03 \x01(\x08R\taggregateB\x0e\n\x0c_output_type"c\n\x18TypedAggregateExpression\x12G\n\x10scalar_scala_udf\x18\x01 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFR\x0escalarScalaUdf"l\n\x0c\x43\x61llFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"\\\n\x17NamedArgumentExpression\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\x80\x04\n\x0bMergeAction\x12\x46\n\x0b\x61\x63tion_type\x18\x01 \x01(\x0e\x32%.spark.connect.MergeAction.ActionTypeR\nactionType\x12<\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\tcondition\x88\x01\x01\x12G\n\x0b\x61ssignments\x18\x03 \x03(\x0b\x32%.spark.connect.MergeAction.AssignmentR\x0b\x61ssignments\x1aj\n\nAssignment\x12+\n\x03key\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\xa7\x01\n\nActionType\x12\x17\n\x13\x41\x43TION_TYPE_INVALID\x10\x00\x12\x16\n\x12\x41\x43TION_TYPE_DELETE\x10\x01\x12\x16\n\x12\x41\x43TION_TYPE_INSERT\x10\x02\x12\x1b\n\x17\x41\x43TION_TYPE_INSERT_STAR\x10\x03\x12\x16\n\x12\x41\x43TION_TYPE_UPDATE\x10\x04\x12\x1b\n\x17\x41\x43TION_TYPE_UPDATE_STAR\x10\x05\x42\x0c\n\n_condition"\xc5\x05\n\x12SubqueryExpression\x12\x17\n\x07plan_id\x18\x01 \x01(\x03R\x06planId\x12S\n\rsubquery_type\x18\x02 \x01(\x0e\x32..spark.connect.SubqueryExpression.SubqueryTypeR\x0csubqueryType\x12\x62\n\x11table_arg_options\x18\x03 \x01(\x0b\x32\x31.spark.connect.SubqueryExpression.TableArgOptionsH\x00R\x0ftableArgOptions\x88\x01\x01\x12G\n\x12in_subquery_values\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x10inSubqueryValues\x1a\xea\x01\n\x0fTableArgOptions\x12@\n\x0epartition_spec\x18\x01 \x03(\x0b\x32\x19.spark.connect.ExpressionR\rpartitionSpec\x12\x42\n\norder_spec\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\torderSpec\x12\x37\n\x15with_single_partition\x18\x03 \x01(\x08H\x00R\x13withSinglePartition\x88\x01\x01\x42\x18\n\x16_with_single_partition"\x90\x01\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x12\x1b\n\x17SUBQUERY_TYPE_TABLE_ARG\x10\x03\x12\x14\n\x10SUBQUERY_TYPE_IN\x10\x04\x42\x14\n\x12_table_arg_optionsB6\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -135,27 +135,27 @@ _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_start = 7899 _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_end = 8296 _globals["_PYTHONUDF"]._serialized_start = 8299 - _globals["_PYTHONUDF"]._serialized_end = 8503 - _globals["_SCALARSCALAUDF"]._serialized_start = 8506 - _globals["_SCALARSCALAUDF"]._serialized_end = 8720 - _globals["_JAVAUDF"]._serialized_start = 8723 - _globals["_JAVAUDF"]._serialized_end = 8872 - _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_start = 8874 - _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_end = 8973 - _globals["_CALLFUNCTION"]._serialized_start = 8975 - _globals["_CALLFUNCTION"]._serialized_end = 9083 - _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_start = 9085 - _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_end = 9177 - _globals["_MERGEACTION"]._serialized_start = 9180 - _globals["_MERGEACTION"]._serialized_end = 9692 - _globals["_MERGEACTION_ASSIGNMENT"]._serialized_start = 9402 - _globals["_MERGEACTION_ASSIGNMENT"]._serialized_end = 9508 - _globals["_MERGEACTION_ACTIONTYPE"]._serialized_start = 9511 - _globals["_MERGEACTION_ACTIONTYPE"]._serialized_end = 9678 - _globals["_SUBQUERYEXPRESSION"]._serialized_start = 9695 - _globals["_SUBQUERYEXPRESSION"]._serialized_end = 10404 - _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_start = 10001 - _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_end = 10235 - _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_start = 10238 - _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_end = 10382 + _globals["_PYTHONUDF"]._serialized_end = 8540 + _globals["_SCALARSCALAUDF"]._serialized_start = 8543 + _globals["_SCALARSCALAUDF"]._serialized_end = 8757 + _globals["_JAVAUDF"]._serialized_start = 8760 + _globals["_JAVAUDF"]._serialized_end = 8909 + _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_start = 8911 + _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_end = 9010 + _globals["_CALLFUNCTION"]._serialized_start = 9012 + _globals["_CALLFUNCTION"]._serialized_end = 9120 + _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_start = 9122 + _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_end = 9214 + _globals["_MERGEACTION"]._serialized_start = 9217 + _globals["_MERGEACTION"]._serialized_end = 9729 + _globals["_MERGEACTION_ASSIGNMENT"]._serialized_start = 9439 + _globals["_MERGEACTION_ASSIGNMENT"]._serialized_end = 9545 + _globals["_MERGEACTION_ACTIONTYPE"]._serialized_start = 9548 + _globals["_MERGEACTION_ACTIONTYPE"]._serialized_end = 9715 + _globals["_SUBQUERYEXPRESSION"]._serialized_start = 9732 + _globals["_SUBQUERYEXPRESSION"]._serialized_end = 10441 + _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_start = 10038 + _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_end = 10272 + _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_start = 10275 + _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_end = 10419 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.pyi b/python/pyspark/sql/connect/proto/expressions_pb2.pyi index c613ade2f43f0..1dad6ff10a814 100644 --- a/python/pyspark/sql/connect/proto/expressions_pb2.pyi +++ b/python/pyspark/sql/connect/proto/expressions_pb2.pyi @@ -1826,6 +1826,7 @@ class PythonUDF(google.protobuf.message.Message): COMMAND_FIELD_NUMBER: builtins.int PYTHON_VER_FIELD_NUMBER: builtins.int ADDITIONAL_INCLUDES_FIELD_NUMBER: builtins.int + BROADCAST_IDS_FIELD_NUMBER: builtins.int @property def output_type(self) -> pyspark.sql.connect.proto.types_pb2.DataType: """(Required) Output type of the Python UDF""" @@ -1840,6 +1841,15 @@ class PythonUDF(google.protobuf.message.Message): self, ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """(Optional) Additional includes for the Python UDF.""" + @property + def broadcast_ids( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: + """(Optional) (SPARK-51705) Server-assigned ids of broadcast variables referenced by this UDF. + Resolved by SparkConnectPlanner against the per-session SessionHolder broadcasts registry + into SimplePythonFunction.broadcastVars. The ids are the driver-side Broadcast ids returned + by CreateBroadcastResult (which the executor/worker also key on), not opaque handles. + """ def __init__( self, *, @@ -1848,6 +1858,7 @@ class PythonUDF(google.protobuf.message.Message): command: builtins.bytes = ..., python_ver: builtins.str = ..., additional_includes: collections.abc.Iterable[builtins.str] | None = ..., + broadcast_ids: collections.abc.Iterable[builtins.int] | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["output_type", b"output_type"] @@ -1857,6 +1868,8 @@ class PythonUDF(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "additional_includes", b"additional_includes", + "broadcast_ids", + b"broadcast_ids", "command", b"command", "eval_type", diff --git a/python/pyspark/sql/connect/session.py b/python/pyspark/sql/connect/session.py index 99da7308a1a00..af698304ad106 100644 --- a/python/pyspark/sql/connect/session.py +++ b/python/pyspark/sql/connect/session.py @@ -37,6 +37,7 @@ overload, Iterable, Mapping, + TypeVar, TYPE_CHECKING, ClassVar, ) @@ -103,6 +104,7 @@ if TYPE_CHECKING: import pyspark.sql.connect.proto as pb2 + from pyspark.core.broadcast import Broadcast from pyspark.sql.connect._typing import OptionalPrimitiveType from pyspark.sql.connect.catalog import Catalog from pyspark.sql.connect.udf import UDFRegistration @@ -112,6 +114,9 @@ from pyspark.sql.connect.datasource import DataSourceRegistration +T = TypeVar("T") + + class SparkSession: # The active SparkSession for the current thread _active_session: ClassVar[threading.local] = threading.local() @@ -866,6 +871,20 @@ def range( range.__doc__ = PySparkSession.range.__doc__ + def broadcast(self, value: "T") -> "Broadcast[T]": + # (SPARK-51705) Create a broadcast variable over Spark Connect. The value is cloudpickled + # and uploaded through the existing cache artifact channel; the server materializes a + # PythonBroadcast on the driver SparkContext and returns a driver-side broadcast id. + from pyspark.serializers import CloudPickleSerializer + from pyspark.sql.connect.broadcast import ConnectBroadcast + + blob = CloudPickleSerializer().dumps(value) + artifact_hash = self._client.cache_artifact(blob) + broadcast_id = self._client._create_broadcast(artifact_hash, len(blob)) + return ConnectBroadcast(self, broadcast_id, value) + + broadcast.__doc__ = PySparkSession.broadcast.__doc__ + @functools.cached_property def catalog(self) -> "Catalog": from pyspark.sql.connect.catalog import Catalog diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py index 1ac0baad15e59..3b2bb187ee4dc 100644 --- a/python/pyspark/sql/pandas/serializers.py +++ b/python/pyspark/sql/pandas/serializers.py @@ -19,7 +19,7 @@ Serializers for PyArrow and pandas conversions. See `pyspark.serializers` for more details. """ -from typing import IO, TYPE_CHECKING, Iterable, Iterator, List, Optional, Tuple +from typing import IO, TYPE_CHECKING, Iterable, Iterator, List, Tuple from pyspark.errors import PySparkRuntimeError, PySparkValueError from pyspark.serializers import ( @@ -28,34 +28,11 @@ write_int, UTF8Deserializer, ) -from pyspark.sql.conversion import ( - ArrowBatchTransformer, - PandasToArrowConversion, -) -from pyspark.sql.types import ( - DataType, - StructType, - StructField, -) if TYPE_CHECKING: - import pandas as pd import pyarrow as pa -def _normalize_packed(packed): - """ - Normalize UDF output to a uniform tuple-of-tuples form. - - Iterator UDFs yield a single (series, spark_type) tuple directly, - while batched UDFs return a tuple of tuples ((s1, t1), (s2, t2), ...). - This function normalizes both forms to a tuple of tuples. - """ - if len(packed) == 2 and isinstance(packed[1], DataType): - return (packed,) - return tuple(packed) - - class SpecialLengths: END_OF_DATA_SECTION = -1 PYTHON_EXCEPTION_THROWN = -2 @@ -212,104 +189,3 @@ def load_stream( errorClass="INVALID_NUMBER_OF_DATAFRAMES_IN_GROUP", messageParameters={"dataframes_in_group": str(dataframes_in_group)}, ) - - -class ArrowStreamPandasSerializer(ArrowStreamSerializer): - """ - Serializes pandas.Series as Arrow data with Arrow streaming format. - - Parameters - ---------- - timezone : str - A timezone to respect when handling timestamp values - safecheck : bool - If True, conversion from Arrow to Pandas checks for overflow/truncation - int_to_decimal_coercion_enabled : bool - If True, applies additional coercions in Python before converting to Arrow. - This has performance penalties. - prefers_large_types : bool - If True, prefer large Arrow types (e.g., large_string instead of string). - struct_in_pandas : str, optional - How to represent struct in pandas ("dict", "row", etc.). Default is "dict". - ndarray_as_list : bool, optional - Whether to convert ndarray as list. Default is False. - prefer_int_ext_dtype : bool, optional - Whether to convert integers to Pandas ExtensionDType. Default is False. - df_for_struct : bool, optional - If True, convert struct columns to DataFrame instead of Series. Default is False. - """ - - def __init__( - self, - *, - timezone, - safecheck, - int_to_decimal_coercion_enabled: bool = False, - prefers_large_types: bool = False, - struct_in_pandas: str = "dict", - ndarray_as_list: bool = False, - prefer_int_ext_dtype: bool = False, - df_for_struct: bool = False, - input_type: Optional["StructType"] = None, - arrow_cast: bool = False, - ): - super().__init__() - self._timezone = timezone - self._safecheck = safecheck - self._int_to_decimal_coercion_enabled = int_to_decimal_coercion_enabled - self._prefers_large_types = prefers_large_types - self._struct_in_pandas = struct_in_pandas - self._ndarray_as_list = ndarray_as_list - self._prefer_int_ext_dtype = prefer_int_ext_dtype - self._df_for_struct = df_for_struct - if input_type is not None: - assert isinstance(input_type, StructType) - self._input_type = input_type - self._arrow_cast = arrow_cast - - def dump_stream(self, iterator, stream): - """ - Make ArrowRecordBatches from Pandas Series and serialize. - Each element in iterator is: - - For batched UDFs: tuple of (series, spark_type) tuples: ((s1, t1), (s2, t2), ...) - - For iterator UDFs: single (series, spark_type) tuple directly - """ - - def create_batch( - series_tuples: Tuple[Tuple["pd.Series", DataType], ...], - ) -> "pa.RecordBatch": - series_data = [s for s, _ in series_tuples] - types = [t for _, t in series_tuples] - schema = StructType([StructField(f"_{i}", t) for i, t in enumerate(types)]) - return PandasToArrowConversion.convert( - series_data, - schema, - timezone=self._timezone, - safecheck=self._safecheck, - prefers_large_types=self._prefers_large_types, - int_to_decimal_coercion_enabled=self._int_to_decimal_coercion_enabled, - ) - - super().dump_stream( - (create_batch(_normalize_packed(packed)) for packed in iterator), stream - ) - - def load_stream(self, stream): - """ - Deserialize ArrowRecordBatches to an Arrow table and return as a list of pandas.Series. - """ - yield from map( - lambda batch: ArrowBatchTransformer.to_pandas( - batch, - timezone=self._timezone, - schema=self._input_type, - struct_in_pandas=self._struct_in_pandas, - ndarray_as_list=self._ndarray_as_list, - prefer_int_ext_dtype=self._prefer_int_ext_dtype, - df_for_struct=self._df_for_struct, - ), - super().load_stream(stream), - ) - - def __repr__(self): - return "ArrowStreamPandasSerializer" diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index c36260b9d13ea..cc2b877cb7518 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -75,6 +75,7 @@ from py4j.java_gateway import JavaClass, JavaObject, JVMView import pyarrow as pa from pyspark.core.context import SparkContext + from pyspark.core.broadcast import Broadcast from pyspark.core.rdd import RDD from pyspark.sql._typing import AtomicValue, RowLike, OptionalPrimitiveType from pyspark.sql.catalog import Catalog @@ -1052,6 +1053,40 @@ def range( return DataFrame(jdf, self) + def broadcast(self, value: "T") -> "Broadcast[T]": + """ + Broadcast a read-only variable to the cluster, returning a + :class:`Broadcast ` object for reading it in distributed + functions. The variable will be sent to each executor only once. + + This is an alias for :meth:`SparkContext.broadcast`, provided so that broadcast variables + can be created from a :class:`SparkSession` in both classic PySpark and Spark Connect + (SPARK-51705). + + .. versionadded:: 4.3.0 + + .. versionchanged:: 4.3.0 + Supports Spark Connect. + + Parameters + ---------- + value : T + value to broadcast to the Spark nodes + + Returns + ------- + :class:`Broadcast ` + :class:`Broadcast ` object, a read-only variable cached + on each machine + + Examples + -------- + >>> bc = spark.broadcast({1: 10001, 2: 10002}) + >>> bc.value + {1: 10001, 2: 10002} + """ + return self._sc.broadcast(value) + def _inferSchemaFromList( self, data: Iterable[Any], names: Optional[List[str]] = None ) -> StructType: diff --git a/python/pyspark/sql/tests/connect/test_connect_broadcast.py b/python/pyspark/sql/tests/connect/test_connect_broadcast.py new file mode 100644 index 0000000000000..960fc9b8afa55 --- /dev/null +++ b/python/pyspark/sql/tests/connect/test_connect_broadcast.py @@ -0,0 +1,106 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +(SPARK-51705) End-to-end tests for ``SparkSession.broadcast()`` over Spark Connect. + +These tests require a running Spark Connect server (they build a real session via +:class:`ReusedConnectTestCase`), so they are gated on ``should_test_connect``. +""" + +import unittest + +from pyspark.errors import PySparkException +from pyspark.testing.connectutils import ( + should_test_connect, + ReusedConnectTestCase, + connect_requirement_message, +) + +if should_test_connect: + from pyspark.sql.connect.broadcast import ConnectBroadcast + from pyspark.sql.functions import udf + from pyspark.sql.types import StringType + + +@unittest.skipIf(not should_test_connect, connect_requirement_message) +class SparkConnectBroadcastTests(ReusedConnectTestCase): + def test_broadcast_returns_connect_broadcast(self): + # spark.broadcast(value) returns a ConnectBroadcast that keeps the value locally for + # driver-side reads. + value = {"a": 1, "b": 2} + bcast = self.spark.broadcast(value) + self.assertIsInstance(bcast, ConnectBroadcast) + self.assertEqual(bcast.value, value) + + def test_broadcast_in_udf(self): + # The exact case that fails today with BROADCAST_VARIABLE_NOT_LOADED on Serverless: + # a dict broadcast referenced from a Python UDF over a DataFrame. + mapping = {i: i * 10 for i in range(1000)} + bcast = self.spark.broadcast(mapping) + + @udf(returnType=StringType()) + def lookup(key): + return str(bcast.value.get(key, -1)) + + df = self.spark.range(0, 100) + rows = df.select(lookup(df.id).alias("v")).collect() + expected = [str(mapping.get(i, -1)) for i in range(100)] + self.assertEqual([r.v for r in rows], expected) + + def test_broadcast_portability(self): + # A UDF referencing a broadcast must produce byte-identical results regardless of how the + # broadcast handle was created; here we just assert repeatable correctness across a large + # broadcast value. + mapping = {i: f"v{i}" for i in range(10_000)} + bcast = self.spark.broadcast(mapping) + + @udf(returnType=StringType()) + def lookup(key): + return bcast.value.get(key, "") + + df = self.spark.range(0, 5000) + rows = df.select(lookup(df.id).alias("v")).collect() + self.assertEqual([r.v for r in rows], [mapping[i] for i in range(5000)]) + + def test_unpersist_and_destroy(self): + # unpersist()/destroy() route to the server; they must not raise. After destroy() the + # broadcast is removed from the session registry (verified server-side / via close sweep). + bcast = self.spark.broadcast(list(range(100))) + bcast.unpersist() + bcast.unpersist(blocking=True) + bcast.destroy() + + def test_broadcast_not_found(self): + # Referencing a broadcast id that was destroyed (or never created on this session) must + # fail loudly with BROADCAST_NOT_FOUND rather than silently emitting an empty broadcast + # list (which would surface as BROADCAST_VARIABLE_NOT_LOADED on the worker). + bcast = self.spark.broadcast({"x": 1}) + bcast.destroy() + + @udf(returnType=StringType()) + def lookup(key): + return str(bcast.value.get("x", -1)) + + df = self.spark.range(0, 10) + with self.assertRaises(PySparkException): + df.select(lookup(df.id).alias("v")).collect() + + +if __name__ == "__main__": + from pyspark.testing import main + + main() diff --git a/python/pyspark/sql/tests/connect/test_parity_udf.py b/python/pyspark/sql/tests/connect/test_parity_udf.py index 33ec11adcf747..87c6da2ea48cf 100644 --- a/python/pyspark/sql/tests/connect/test_parity_udf.py +++ b/python/pyspark/sql/tests/connect/test_parity_udf.py @@ -44,7 +44,11 @@ def test_udf_with_input_file_name_for_hadooprdd(self): def test_same_accumulator_in_udfs(self): super().test_same_accumulator_in_udfs() - @unittest.skip("Spark Connect does not support broadcast but the test depends on it.") + @unittest.skip( + "The test uses SparkContext.broadcast, which Spark Connect does not expose. " + "SparkSession.broadcast over Connect (SPARK-51705) is covered by " + "pyspark.sql.tests.connect.test_connect_broadcast." + ) def test_broadcast_in_udf(self): super().test_broadcast_in_udf() diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala index e40a6e6df88b5..3fb20dcd6420d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala @@ -95,7 +95,7 @@ class V2ExpressionBuilder(e: Expression, isPredicate: Boolean = false) extends L expr: Expression, isPredicate: Boolean = false): Option[V2Expression] = expr match { case literal: Literal => Some(translateLiteral(literal)) case _ if expr.contextIndependentFoldable - && SQLConf.get.getConf(SQLConf.DATA_SOURCE_V2_EXPR_FOLDING) => + && SQLConf.get.getConfByKeyStrict[Boolean]("spark.sql.optimizer.datasourceV2ExprFolding") => // If the expression is context independent foldable, we can convert it to a literal. // This is useful for increasing the coverage of V2 expressions. val constantExpr = ConstantFolding.constantFolding(expr) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index a5d64ddd314ed..a9ea3f9f1c26c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -1920,7 +1920,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE def catalogPluginClassNotImplementedError(name: String, pluginClassName: String): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2214", + errorClass = "CANNOT_LOAD_CATALOG.NOT_A_CATALOG_PLUGIN", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), @@ -1932,7 +1932,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE pluginClassName: String, e: Exception): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2215", + errorClass = "CANNOT_LOAD_CATALOG.PLUGIN_CLASS_NOT_FOUND", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), @@ -1944,7 +1944,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE pluginClassName: String, e: Exception): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2216", + errorClass = "CANNOT_LOAD_CATALOG.CONSTRUCTOR_NOT_FOUND", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), @@ -1956,7 +1956,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE pluginClassName: String, e: Exception): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2217", + errorClass = "CANNOT_LOAD_CATALOG.CONSTRUCTOR_NOT_ACCESSIBLE", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), @@ -1968,7 +1968,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE pluginClassName: String, e: Exception): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2218", + errorClass = "CANNOT_LOAD_CATALOG.ABSTRACT_CLASS", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), @@ -1980,7 +1980,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE pluginClassName: String, e: Exception): Throwable = { new SparkException( - errorClass = "_LEGACY_ERROR_TEMP_2219", + errorClass = "CANNOT_LOAD_CATALOG.CONSTRUCTOR_FAILURE", messageParameters = Map( "name" -> name, "pluginClassName" -> pluginClassName), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 77c69494a15a5..97594802e258d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -35,6 +35,8 @@ import org.apache.hadoop.mapreduce.OutputCommitter import org.slf4j.event.Level import org.apache.spark.{ErrorMessageFormat, SparkConf, SparkContext, SparkException, TaskContext} +import org.apache.spark.config.ConfigRegistry +import org.apache.spark.config.protobuf.Mutability import org.apache.spark.internal.Logging import org.apache.spark.internal.config.{ConfigBindingPolicy, _} import org.apache.spark.internal.io.FileCommitProtocol @@ -75,6 +77,9 @@ object SQLConf { private def register(entry: ConfigEntry[_]): Unit = sqlConfEntriesUpdateLock.synchronized { require(!sqlConfEntries.containsKey(entry.key), s"Duplicate SQLConfigEntry. ${entry.key} has been registered") + require(!ConfigRegistry.containsConfig(entry.key), + s"${entry.key} is defined in a .textproto config file. " + + s"Use buildConfFromConfigFile instead of buildConf.") val updatedMap = new java.util.HashMap[String, ConfigEntry[_]](sqlConfEntries) updatedMap.put(entry.key, entry) sqlConfEntries = updatedMap @@ -88,11 +93,28 @@ object SQLConf { } private[internal] def getConfigEntry(key: String): ConfigEntry[_] = { - sqlConfEntries.get(key) + Option(sqlConfEntries.get(key)).getOrElse(ConfigEntry.findProtoDefinedEntry(key)) } + // TODO: once all configs are migrated to textproto, this can be replaced by + // ConfigEntry.listAllEntries() and callers can filter by config properties. private[sql] def getConfigEntries(): util.Collection[ConfigEntry[_]] = { - sqlConfEntries.values() + // Lazy concatenating view - no intermediate allocation + new util.AbstractCollection[ConfigEntry[_]]() { + private lazy val protoConfigs = ConfigEntry.listAllProtoDefinedConfigs() + + override def iterator(): util.Iterator[ConfigEntry[_]] = { + val first = sqlConfEntries.values().iterator() + val second = protoConfigs.iterator() + new util.Iterator[ConfigEntry[_]]() { + override def hasNext: Boolean = first.hasNext || second.hasNext + override def next(): ConfigEntry[_] = + if (first.hasNext) first.next() else second.next() + } + } + + override def size(): Int = sqlConfEntries.size() + protoConfigs.size() + } } private[internal] def containsConfigEntry(entry: ConfigEntry[_]): Boolean = { @@ -100,7 +122,7 @@ object SQLConf { } private[sql] def containsConfigKey(key: String): Boolean = { - sqlConfEntries.containsKey(key) + sqlConfEntries.containsKey(key) || ConfigRegistry.containsConfig(key) } def registerStaticConfigKey(key: String): Unit = staticConfKeysUpdateLock.synchronized { @@ -109,10 +131,27 @@ object SQLConf { staticConfKeys = updated } - def isStaticConfigKey(key: String): Boolean = staticConfKeys.contains(key) + def isStaticConfigKey(key: String): Boolean = { + staticConfKeys.contains(key) || + Option(ConfigRegistry.getConfig(key)).exists(_.getMutability == Mutability.MUTABILITY_STATIC) + } def buildConf(key: String): ConfigBuilder = ConfigBuilder(key).onCreate(register) + def buildConfFromConfigFile[T](key: String): ProtoBackedConfigEntry[T] = { + Option(ConfigEntry.findProtoBackedEntry(key)).map(_.asInstanceOf[ProtoBackedConfigEntry[T]]) + .getOrElse { + if (ConfigRegistry.containsConfig(key)) { + // The key is defined in a .textproto file but has no default value, so it is backed by + // a ProtoBackedOptionalConfigEntry. Such configs must be accessed via getConfByKeyStrict. + throw SparkException.internalError( + s"Config entry $key has no default value; use getConfByKeyStrict to read it.") + } else { + throw SparkException.internalError(s"Config entry $key not found in ConfigRegistry") + } + } + } + def buildStaticConf(key: String): ConfigBuilder = { ConfigBuilder(key).onCreate { entry => SQLConf.registerStaticConfigKey(entry.key) @@ -129,7 +168,7 @@ object SQLConf { private[sql] def mergeNonStaticSQLConfigs( sqlConf: SQLConf, configs: Map[String, String]): Unit = { - for ((k, v) <- configs if !staticConfKeys.contains(k)) { + for ((k, v) <- configs if !isStaticConfigKey(k)) { sqlConf.setConfString(k, v) } } @@ -570,12 +609,8 @@ object SQLConf { .stringConf .createOptional - val OPTIMIZER_MAX_ITERATIONS = buildConf("spark.sql.optimizer.maxIterations") - .internal() - .doc("The max number of iterations the optimizer runs.") - .version("2.0.0") - .intConf - .createWithDefault(100) + val OPTIMIZER_MAX_ITERATIONS = + buildConfFromConfigFile[Int]("spark.sql.optimizer.maxIterations") val OPTIMIZER_INSET_CONVERSION_THRESHOLD = buildConf("spark.sql.optimizer.inSetConversionThreshold") @@ -1001,13 +1036,9 @@ object SQLConf { .bytesConf(ByteUnit.BYTE) .createWithDefaultString("10MB") - val SHUFFLE_HASH_JOIN_FACTOR = buildConf("spark.sql.shuffledHashJoinFactor") - .doc("The shuffle hash join can be selected if the data size of small" + - " side multiplied by this factor is still smaller than the large side.") - .version("3.3.0") - .intConf - .checkValue(_ >= 1, "The shuffle hash join factor cannot be negative.") - .createWithDefault(3) + val SHUFFLE_HASH_JOIN_FACTOR = + buildConfFromConfigFile[Int]("spark.sql.shuffledHashJoinFactor") + .checkValue(_ >= 1, "The shuffle hash join factor must be at least 1.") val LIMIT_INITIAL_NUM_PARTITIONS = buildConf("spark.sql.limit.initialNumPartitions") .internal() @@ -2217,15 +2248,6 @@ object SQLConf { .booleanConf .createWithDefault(false) - val DATA_SOURCE_V2_EXPR_FOLDING = - buildConf("spark.sql.optimizer.datasourceV2ExprFolding") - .internal() - .version("4.1.0") - .doc("When this config is set to true, do safe constant folding for the " + - "expressions before translation and pushdown.") - .booleanConf - .createWithDefault(true) - // This is used to set the default data source val DEFAULT_DATA_SOURCE_NAME = buildConf("spark.sql.sources.default") .doc("The default data source to use in input/output.") @@ -6675,6 +6697,19 @@ object SQLConf { .booleanConf .createWithDefault(true) + val PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS = + buildConf("spark.sql.variant.pushVariantIntoScan.pullOutExtractions") + .internal() + .doc("When true, extend variant field pushdown to extractions used inside aggregate " + + "functions, join conditions, sort keys, and projections above joins (including through " + + "chained joins), so that only the requested fields are read from the scan for these " + + "cases too instead of the whole variant column. Has no effect unless " + + "spark.sql.variant.pushVariantIntoScan is also true.") + .version("4.3.0") + .withBindingPolicy(ConfigBindingPolicy.SESSION) + .booleanConf + .createWithDefault(true) + val PUSH_VARIANT_INTO_SCAN_DEFER_CAST_ERROR = buildConf("spark.sql.variant.pushVariantIntoScan.deferCastError") .internal() @@ -9475,6 +9510,14 @@ class SQLConf extends Serializable with Logging with SqlApiConf { } } + def getConfByKeyStrict[T](key: String): T = { + Option(ConfigEntry.findProtoDefinedEntry(key)) + .map(_.asInstanceOf[ConfigEntry[T]].readFrom(reader)) + .getOrElse { + throw SparkException.internalError(s"Config entry $key not found in ConfigRegistry") + } + } + private var definedConfsLoaded = false /** * Init [[StaticSQLConf]] and [[org.apache.spark.sql.hive.HiveUtils]] so that all the defined diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala index 658ab5f5312d5..72e36250b068a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala @@ -33,7 +33,7 @@ import org.apache.spark.util.Utils */ object StaticSQLConf { - import SQLConf.buildStaticConf + import SQLConf.{buildConfFromConfigFile, buildStaticConf} val WAREHOUSE_PATH = buildStaticConf("spark.sql.warehouse.dir") .doc("The default location for managed databases and tables.") @@ -198,11 +198,7 @@ object StaticSQLConf { .createOptional val UI_RETAINED_EXECUTIONS = - buildStaticConf("spark.sql.ui.retainedExecutions") - .doc("Number of executions to retain in the Spark UI.") - .version("1.5.0") - .intConf - .createWithDefault(1000) + buildConfFromConfigFile[Int]("spark.sql.ui.retainedExecutions") val SHUFFLE_EXCHANGE_MAX_THREAD_THRESHOLD = buildStaticConf("spark.sql.shuffleExchange.maxThreadThreshold") diff --git a/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java b/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java index c7e8d7b0f7f30..655cb55852a7a 100644 --- a/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java +++ b/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java @@ -97,7 +97,7 @@ public void testLoadMissingClass() { SparkException exc = Assertions.assertThrows(SparkException.class, () -> Catalogs.load("missing", conf)); - Assertions.assertTrue(exc.getMessage().contains("Cannot find catalog plugin class"), + Assertions.assertEquals("CANNOT_LOAD_CATALOG.PLUGIN_CLASS_NOT_FOUND", exc.getCondition(), "Should complain that the class is not found"); Assertions.assertTrue(exc.getMessage().contains("missing"), "Should identify the catalog by name"); @@ -127,7 +127,7 @@ public void testLoadNonCatalogPlugin() { SparkException exc = Assertions.assertThrows(SparkException.class, () -> Catalogs.load("invalid", conf)); - Assertions.assertTrue(exc.getMessage().contains("does not implement CatalogPlugin"), + Assertions.assertEquals("CANNOT_LOAD_CATALOG.NOT_A_CATALOG_PLUGIN", exc.getCondition(), "Should complain that class does not implement CatalogPlugin"); Assertions.assertTrue(exc.getMessage().contains("invalid"), "Should identify the catalog by name"); @@ -144,8 +144,7 @@ public void testLoadConstructorFailureCatalogPlugin() { SparkException exc = Assertions.assertThrows(SparkException.class, () -> Catalogs.load("invalid", conf)); - Assertions.assertTrue( - exc.getMessage().contains("Failed during instantiating constructor for catalog"), + Assertions.assertEquals("CANNOT_LOAD_CATALOG.CONSTRUCTOR_FAILURE", exc.getCondition(), "Should identify the constructor error"); Assertions.assertTrue(exc.getCause().getMessage().contains("Expected failure"), "Should have expected error message"); @@ -160,14 +159,47 @@ public void testLoadAccessErrorCatalogPlugin() { SparkException exc = Assertions.assertThrows(SparkException.class, () -> Catalogs.load("invalid", conf)); - Assertions.assertTrue( - exc.getMessage().contains("Failed to call public no-arg constructor for catalog"), + Assertions.assertEquals("CANNOT_LOAD_CATALOG.CONSTRUCTOR_NOT_ACCESSIBLE", exc.getCondition(), "Should complain that no public constructor is provided"); Assertions.assertTrue(exc.getMessage().contains("invalid"), "Should identify the catalog by name"); Assertions.assertTrue(exc.getMessage().contains(invalidClassName), "Should identify the class"); } + + @Test + public void testLoadNoNoArgConstructorCatalogPlugin() { + SQLConf conf = new SQLConf(); + String invalidClassName = NoNoArgConstructorCatalogPlugin.class.getCanonicalName(); + conf.setConfString("spark.sql.catalog.invalid", invalidClassName); + + SparkException exc = Assertions.assertThrows(SparkException.class, + () -> Catalogs.load("invalid", conf)); + + Assertions.assertEquals("CANNOT_LOAD_CATALOG.CONSTRUCTOR_NOT_FOUND", exc.getCondition(), + "Should complain that no public no-arg constructor is found"); + Assertions.assertTrue(exc.getMessage().contains("invalid"), + "Should identify the catalog by name"); + Assertions.assertTrue(exc.getMessage().contains(invalidClassName), + "Should identify the class"); + } + + @Test + public void testLoadAbstractCatalogPlugin() { + SQLConf conf = new SQLConf(); + String invalidClassName = AbstractCatalogPlugin.class.getCanonicalName(); + conf.setConfString("spark.sql.catalog.invalid", invalidClassName); + + SparkException exc = Assertions.assertThrows(SparkException.class, + () -> Catalogs.load("invalid", conf)); + + Assertions.assertEquals("CANNOT_LOAD_CATALOG.ABSTRACT_CLASS", exc.getCondition(), + "Should complain that the class is abstract"); + Assertions.assertTrue(exc.getMessage().contains("invalid"), + "Should identify the catalog by name"); + Assertions.assertTrue(exc.getMessage().contains(invalidClassName), + "Should identify the class"); + } } class TestCatalogPlugin implements CatalogPlugin { @@ -218,6 +250,25 @@ public String name() { } } +class NoNoArgConstructorCatalogPlugin implements CatalogPlugin { // no public no-arg constructor + NoNoArgConstructorCatalogPlugin(String arg) { + } + + @Override + public void initialize(String name, CaseInsensitiveStringMap options) { + } + + @Override + public String name() { + return null; + } +} + +abstract class AbstractCatalogPlugin implements CatalogPlugin { // abstract, cannot be instantiated + AbstractCatalogPlugin() { + } +} + class InvalidCatalogPlugin { // doesn't implement CatalogPlugin public void initialize(CaseInsensitiveStringMap options) { } diff --git a/sql/connect/common/src/main/protobuf/spark/connect/base.proto b/sql/connect/common/src/main/protobuf/spark/connect/base.proto index c7247129f1907..14dd889cb944d 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/base.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/base.proto @@ -429,6 +429,9 @@ message ExecutePlanResponse { // register its result with the server. PipelineQueryFunctionExecutionSignal pipeline_query_function_execution_signal = 23; + // (SPARK-51705) Server-assigned handle for a CreateBroadcastCommand. + CreateBroadcastResult create_broadcast_result = 24; + // Support arbitrary result objects. google.protobuf.Any extension = 999; } @@ -1176,6 +1179,14 @@ message CheckpointCommandResult { CachedRemoteRelation relation = 1; } +// (SPARK-51705) Server-assigned broadcast handle (mirrors CheckpointCommandResult). +message CreateBroadcastResult { + // (Required) The driver-side broadcast id. The client embeds this id in the pickled + // Broadcast closure (Broadcast._from_id), which the executor/worker resolves against its + // _broadcastRegistry keyed by the same id. + int64 broadcast_id = 1; +} + message CloneSessionRequest { // (Required) // diff --git a/sql/connect/common/src/main/protobuf/spark/connect/commands.proto b/sql/connect/common/src/main/protobuf/spark/connect/commands.proto index dcf5aff2366f7..394673d50b2ea 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/commands.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/commands.proto @@ -53,6 +53,8 @@ message Command { MlCommand ml_command = 17; ExecuteExternalCommand execute_external_command = 18; PipelineCommand pipeline_command = 19; + CreateBroadcastCommand create_broadcast_command = 20; + UnpersistBroadcastCommand unpersist_broadcast_command = 21; // This field is used to mark extensions to the protocol. When plugins generate arbitrary // Commands they can add them here. During the planning the correct resolution is done. @@ -61,6 +63,27 @@ message Command { } } +// (SPARK-51705) Create a broadcast variable from an already-uploaded cache/ artifact. +// The client uploads cloudpickle(value) through the existing cache artifact channel and then +// sends this command with the returned hash. The server materializes a PythonBroadcast on the +// live driver SparkContext and returns a CreateBroadcastResult with the driver-side broadcast id. +message CreateBroadcastCommand { + // (Required) sha256 hash returned by client.cache_artifact(cloudpickle(value)). + string artifact_hash = 1; + // (Optional) Uncompressed byte size, used to enforce the BROADCAST_VALUE_TOO_LARGE quota. + int64 size_bytes = 2; +} + +// (SPARK-51705) Release a broadcast variable created over Spark Connect. +message UnpersistBroadcastCommand { + // (Required) The driver-side broadcast id returned by CreateBroadcastResult. + int64 broadcast_id = 1; + // (Optional) Whether to block until unpersisting has completed. + bool blocking = 2; + // (Optional) Whether to destroy (not just unpersist) all data and metadata of the broadcast. + bool destroy = 3; +} + // A SQL Command is used to trigger the eager evaluation of SQL commands in Spark. // // When the SQL provide as part of the message is a command it will be immediately evaluated diff --git a/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto b/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto index 18f8f294f0c02..8631d6e6def87 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto @@ -475,6 +475,11 @@ message PythonUDF { string python_ver = 4; // (Optional) Additional includes for the Python UDF. repeated string additional_includes = 5; + // (Optional) (SPARK-51705) Server-assigned ids of broadcast variables referenced by this UDF. + // Resolved by SparkConnectPlanner against the per-session SessionHolder broadcasts registry + // into SimplePythonFunction.broadcastVars. The ids are the driver-side Broadcast ids returned + // by CreateBroadcastResult (which the executor/worker also key on), not opaque handles. + repeated int64 broadcast_ids = 6; } message ScalarScalaUDF { diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/InvalidInputErrors.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/InvalidInputErrors.scala index 604ad38606970..6e3dec3f8df1d 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/InvalidInputErrors.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/InvalidInputErrors.scala @@ -256,4 +256,8 @@ object InvalidInputErrors { InvalidPlanInput( "CONNECT_INVALID_PLAN.CANNOT_FIND_CACHED_LOCAL_RELATION", Map("hash" -> hash)) + + // (SPARK-51705) A Python UDF referenced a broadcast id that is not registered on this session. + def broadcastNotFound(id: Long): InvalidPlanInput = + InvalidPlanInput("CONNECT_INVALID_PLAN.BROADCAST_NOT_FOUND", Map("id" -> id.toString)) } diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala index 841ce26402cf0..1a9d6c5844a88 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.connect.planner +import java.io.{File, FileOutputStream} +import java.nio.file.Files import java.util.{HashMap, Properties, UUID} import scala.collection.mutable @@ -31,7 +33,8 @@ import io.grpc.stub.StreamObserver import org.apache.spark.{SparkClassNotFoundException, SparkEnv, SparkException} import org.apache.spark.annotation.{DeveloperApi, Since} -import org.apache.spark.api.python.{PythonEvalType, SimplePythonFunction} +import org.apache.spark.api.python.{PythonBroadcast, PythonEvalType, SimplePythonFunction} +import org.apache.spark.broadcast.Broadcast import org.apache.spark.connect.proto import org.apache.spark.connect.proto.{CheckpointCommand, CreateResourceProfileCommand, ExecutePlanResponse, SqlCommand, StreamingForeachFunction, StreamingQueryCommand, StreamingQueryCommandResult, StreamingQueryInstanceId, StreamingQueryManagerCommand, StreamingQueryManagerCommandResult, WriteStreamOperationStart, WriteStreamOperationStartResult} import org.apache.spark.connect.proto.ExecutePlanResponse.SqlCommandResult @@ -2227,12 +2230,33 @@ class SparkConnectPlanner( pythonIncludes = (fun.getAdditionalIncludesList.asScala.toSeq ++ sessionHolder.artifactManager.getPythonIncludes).asJava, pythonVer = fun.getPythonVer, - // Empty broadcast variables - broadcastVars = Lists.newArrayList(), + // (SPARK-51705) Resolve broadcast variables referenced by this UDF from the per-session + // registry. Empty list when the UDF references no broadcasts. + broadcastVars = resolveBroadcasts(fun.getBroadcastIdsList), // Accumulator if available accumulator = sessionHolder.pythonAccumulator.orNull) } + /** + * (SPARK-51705) Resolve the broadcast ids referenced by a Python UDF into the driver-side + * Broadcast[PythonBroadcast] handles from the per-session SessionHolder registry. An id that is + * unknown (never created on this session, or belonging to another session) fails loudly with + * BROADCAST_NOT_FOUND rather than silently emitting an empty list -- which would otherwise + * surface as BROADCAST_VARIABLE_NOT_LOADED on the Python worker. + */ + private def resolveBroadcasts(broadcastIds: java.util.List[java.lang.Long]) + : java.util.List[Broadcast[PythonBroadcast]] = { + val resolved = new java.util.ArrayList[Broadcast[PythonBroadcast]](broadcastIds.size()) + broadcastIds.forEach { boxedId => + val id = boxedId.longValue() + val bcast = sessionHolder + .getBroadcast(id) + .getOrElse(throw InvalidInputErrors.broadcastNotFound(id)) + resolved.add(bcast) + } + resolved + } + /** * Translates a LambdaFunction from proto to the Catalyst expression. */ @@ -2932,6 +2956,10 @@ class SparkConnectPlanner( handlePipelineCommand(command.getPipelineCommand, responseObserver) case proto.Command.CommandTypeCase.EXECUTE_EXTERNAL_COMMAND => handleExecuteExternalCommand(command.getExecuteExternalCommand, responseObserver) + case proto.Command.CommandTypeCase.CREATE_BROADCAST_COMMAND => + handleCreateBroadcastCommand(command.getCreateBroadcastCommand, responseObserver) + case proto.Command.CommandTypeCase.UNPERSIST_BROADCAST_COMMAND => + handleUnpersistBroadcastCommand(command.getUnpersistBroadcastCommand) case other => throw InvalidInputErrors.invalidOneOfField(other, command.getDescriptorForType) @@ -4012,6 +4040,81 @@ class SparkConnectPlanner( executeHolder.eventsManager.postFinished() } + /** + * (SPARK-51705) Materialize a broadcast variable from an already-uploaded cache/ + * artifact and register it on this session. + * + * The client uploaded cloudpickle(value) through the existing artifact cache channel; here we + * read the block back (decrypting transparently under spark.io.encryption), stage it to a temp + * file under the local dir (PythonBroadcast needs a path, and the cache block may be memory + * only), wrap it in a PythonBroadcast, and broadcast it on the live driver SparkContext -- the + * same object pythonAccumulator uses. The returned broadcast id is the driver-side + * Broadcast.id, which the client embeds into the pickled UDF closure and which the worker keys + * on. + */ + private def handleCreateBroadcastCommand( + command: proto.CreateBroadcastCommand, + responseObserver: StreamObserver[proto.ExecutePlanResponse]): Unit = { + val hash = command.getArtifactHash + val blockManager = session.sparkContext.env.blockManager + val blockId = sessionHolder.artifactManager.getCachedBlockId(hash).getOrElse { + throw InvalidInputErrors.cannotFindCachedLocalRelation(hash) + } + // Stage the (possibly memory-only, possibly encrypted) cache block to a temp file that + // PythonBroadcast can read. Mirrors the block-read idiom in transformCachedLocalRelation. + val bytes = blockManager + .getLocalBytes(blockId) + .getOrElse { + throw InvalidInputErrors.notFoundCachedLocalRelation(blockId.hash, blockId.sessionUUID) + } + val dir = new File(Utils.getLocalDir(session.sparkContext.conf)) + val file = Files.createTempFile(dir.toPath, "broadcast", "").toFile + try { + val in = bytes.toInputStream() + val out = new FileOutputStream(file) + Utils.tryWithSafeFinally { + Utils.copyStream(in, out) + } { + out.close() + in.close() + } + } finally { + blockManager.releaseLock(blockId) + } + + val pythonBroadcast = new PythonBroadcast(file.getAbsolutePath) + val bcast = session.sparkContext.broadcast(pythonBroadcast) + val broadcastId = sessionHolder.registerBroadcast(bcast) + logInfo(log"Created broadcast with id ${MDC(LogKeys.BROADCAST_ID, broadcastId)}") + + executeHolder.eventsManager.postFinished() + responseObserver.onNext( + proto.ExecutePlanResponse + .newBuilder() + .setSessionId(sessionId) + .setServerSideSessionId(sessionHolder.serverSessionId) + .setCreateBroadcastResult( + proto.CreateBroadcastResult + .newBuilder() + .setBroadcastId(broadcastId) + .build()) + .build()) + } + + /** + * (SPARK-51705) Release a broadcast variable created over Spark Connect. Unknown ids are a + * no-op (idempotent unpersist), consistent with the client possibly retrying. + */ + private def handleUnpersistBroadcastCommand(command: proto.UnpersistBroadcastCommand): Unit = { + val broadcastId = command.getBroadcastId + if (command.getDestroy) { + sessionHolder.removeBroadcast(broadcastId).foreach(_.destroy(command.getBlocking)) + } else { + sessionHolder.getBroadcast(broadcastId).foreach(_.unpersist(command.getBlocking)) + } + executeHolder.eventsManager.postFinished() + } + private def transformMergeIntoTableCommand(cmd: proto.MergeIntoTableCommand)( tracker: QueryPlanningTracker): LogicalPlan = { def transformActions(actions: java.util.List[proto.Expression]): Seq[MergeAction] = diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala index 2276230545e67..070a511fdb670 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala @@ -24,12 +24,15 @@ import scala.collection.mutable import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.util.Try +import scala.util.control.NonFatal import com.google.common.base.Ticker import com.google.common.cache.{Cache, CacheBuilder} import org.apache.spark.{SparkEnv, SparkException, SparkSQLException} +import org.apache.spark.api.python.PythonBroadcast import org.apache.spark.api.python.PythonFunction.PythonAccumulator +import org.apache.spark.broadcast.Broadcast import org.apache.spark.connect.proto import org.apache.spark.internal.{Logging, LogKeys} import org.apache.spark.sql.DataFrame @@ -425,6 +428,21 @@ case class SessionHolder(userId: String, sessionId: String, session: SparkSessio // Clean up ML cache (only if ML models were created) mlCache.close() + // (SPARK-51705) Destroy all broadcast variables created over Spark Connect, freeing the + // driver + executor blocks. Best-effort: a failure to destroy one must not abort the sweep. + broadcasts.forEach { (id, bcast) => + try { + bcast.destroy() + } catch { + case NonFatal(e) => + logWarning( + log"Failed to destroy broadcast ${MDC(LogKeys.BROADCAST_ID, id)} " + + log"while closing session ${MDC(LogKeys.SESSION_ID, sessionId)}", + e) + } + } + broadcasts.clear() + session.cleanupPythonWorkerLogs() eventManager.postClosed() @@ -586,6 +604,29 @@ case class SessionHolder(userId: String, sessionId: String, session: SparkSessio private[connect] val pythonAccumulator: Option[PythonAccumulator] = Try(session.sparkContext.collectionAccumulator[Array[Byte]]).toOption + /** + * (SPARK-51705) Registry of broadcast variables created over Spark Connect for this session, + * keyed by the driver-side broadcast id (Broadcast.id). This id is what the client embeds in + * the pickled UDF closure (via Broadcast._from_id) and what the executor/worker keys on, so it + * is used both as the client handle (CreateBroadcastResult.broadcast_id / + * PythonUDF.broadcast_ids) and as the registry key. Membership in this per-session map enforces + * isolation: SparkConnectPlanner.resolveBroadcasts rejects ids that are absent + * (BROADCAST_NOT_FOUND). Sibling to `pythonAccumulator`. + */ + private[connect] val broadcasts: ConcurrentMap[Long, Broadcast[PythonBroadcast]] = + new ConcurrentHashMap() + + private[connect] def registerBroadcast(bcast: Broadcast[PythonBroadcast]): Long = { + broadcasts.put(bcast.id, bcast) + bcast.id + } + + private[connect] def getBroadcast(id: Long): Option[Broadcast[PythonBroadcast]] = + Option(broadcasts.get(id)) + + private[connect] def removeBroadcast(id: Long): Option[Broadcast[PythonBroadcast]] = + Option(broadcasts.remove(id)) + /** * Transform a relation into a logical plan, using the plan cache if enabled. The plan cache is * enable only if `spark.connect.session.planCache.maxSize` is greater than zero AND diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectSessionHolderSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectSessionHolderSuite.scala index cff5f345d2573..e63bebe0a8752 100644 --- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectSessionHolderSuite.scala +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectSessionHolderSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.connect.service import java.nio.charset.StandardCharsets import java.nio.file.Files +import java.util.UUID import java.util.concurrent.{TimeoutException, TimeUnit} import scala.collection.mutable @@ -31,7 +32,7 @@ import com.google.common.collect.Lists import org.scalatest.time.SpanSugar._ import org.apache.spark.SparkEnv -import org.apache.spark.api.python.SimplePythonFunction +import org.apache.spark.api.python.{PythonBroadcast, SimplePythonFunction} import org.apache.spark.connect.proto import org.apache.spark.sql.IntegratedUDFTestUtils import org.apache.spark.sql.connect.{PythonTestDepsChecker, SparkConnectTestUtils} @@ -106,6 +107,48 @@ class SparkConnectSessionHolderSuite extends SharedSparkSession { } } + test("SPARK-51705: broadcast registry put, get and remove") { + val sessionHolder = SparkConnectTestUtils.createDummySessionHolder(spark) + + val pb1 = new PythonBroadcast(Files.createTempFile("broadcast", "").toString) + val pb2 = new PythonBroadcast(Files.createTempFile("broadcast", "").toString) + val bcast1 = spark.sparkContext.broadcast(pb1) + val bcast2 = spark.sparkContext.broadcast(pb2) + + // The registry is keyed by the driver-side broadcast id, which is the client handle. + val id1 = sessionHolder.registerBroadcast(bcast1) + val id2 = sessionHolder.registerBroadcast(bcast2) + assert(id1 == bcast1.id) + assert(id2 == bcast2.id) + assert(sessionHolder.getBroadcast(id1).contains(bcast1)) + assert(sessionHolder.getBroadcast(id2).contains(bcast2)) + + // Unknown id is None -- resolveBroadcasts turns this into BROADCAST_NOT_FOUND. + assert(sessionHolder.getBroadcast(Long.MaxValue).isEmpty) + + // Remove one, the other survives. + assert(sessionHolder.removeBroadcast(id1).contains(bcast1)) + assert(sessionHolder.getBroadcast(id1).isEmpty) + assert(sessionHolder.getBroadcast(id2).contains(bcast2)) + } + + test("SPARK-51705: close() sweeps registered broadcasts") { + // Use an unregistered holder marked Started (not createDummySessionHolder) so that close() + // runs the full cleanup -- for a still-Pending session in tests close() returns early -- and + // so the closed holder is not left behind in the global session manager store. + val sessionHolder = + SessionHolder(userId = "testUser", sessionId = UUID.randomUUID().toString, session = spark) + sessionHolder.eventManager.status_(SessionStatus.Started) + + val pb = new PythonBroadcast(Files.createTempFile("broadcast", "").toString) + val bcast = spark.sparkContext.broadcast(pb) + val id = sessionHolder.registerBroadcast(bcast) + assert(sessionHolder.getBroadcast(id).contains(bcast)) + + sessionHolder.close() + assert(sessionHolder.getBroadcast(id).isEmpty) + } + private def streamingForeachBatchFunction(pysparkPythonPath: String): Array[Byte] = { var binaryFunc: Array[Byte] = null withTempPath { path => diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala index 54158d5bb4a94..0994220afe0c1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.optimizer._ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.connector.catalog.CatalogManager -import org.apache.spark.sql.execution.datasources.{MarkSingleTaskExecution, PruneFileSourcePartitions, PushVariantIntoScan, SchemaPruning, V1Writes} +import org.apache.spark.sql.execution.datasources.{MarkSingleTaskExecution, PruneFileSourcePartitions, PullOutVariantExtractions, PushVariantIntoScan, SchemaPruning, V1Writes} import org.apache.spark.sql.execution.datasources.v2.{GroupBasedRowLevelOperationScanPlanning, OptimizeMetadataOnlyDeleteFromTable, V2ScanPartitioningAndOrdering, V2ScanRelationPushDown, V2Writes} import org.apache.spark.sql.execution.dynamicpruning.{CleanupDynamicPruningFilters, PartitionPruning, RowLevelOperationRuntimeGroupFiltering} import org.apache.spark.sql.execution.python.{ExtractGroupingPythonUDFFromAggregate, ExtractPythonUDFFromAggregate, ExtractPythonUDFs, ExtractPythonUDTFs} @@ -37,6 +37,10 @@ class SparkOptimizer( override def earlyScanPushDownRules: Seq[Rule[LogicalPlan]] = // TODO: move SchemaPruning into catalyst Seq( + // Hoist variant extractions out of operators that variant-into-scan pushdown cannot see + // through (aggregate function arguments, etc.) into a Project above the scan, so the + // extractions below become visible to V2ScanRelationPushDown and PushVariantIntoScan. + PullOutVariantExtractions, SchemaPruning, GroupBasedRowLevelOperationScanPlanning, V1Writes, diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PullOutVariantExtractions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PullOutVariantExtractions.scala new file mode 100644 index 0000000000000..6695e24ad6186 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PullOutVariantExtractions.scala @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources + +import scala.collection.mutable + +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeSet, Cast, Expression, + NamedExpression, SortOrder} +import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression +import org.apache.spark.sql.catalyst.expressions.variant.VariantGet +import org.apache.spark.sql.catalyst.plans.LeftExistence +import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Join, LogicalPlan, Project, Sort} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.catalyst.trees.TreePattern.{AGGREGATE, JOIN, SORT} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.VariantType + +/** + * Hoists variant extractions out of operators that variant-into-scan pushdown cannot see through, + * into a [[Project]] directly below that operator, so the extraction becomes visible to the + * pushdown rules ([[PushVariantIntoScan]] for v1 and + * [[org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown]] for v2). The rule + * applies uniformly to all three table read paths: DS1 (e.g. `spark.read.parquet`), DS2 (e.g. + * `spark.read.parquet` with `spark.sql.sources.useV1SourceList` cleared), and Spark native + * catalog tables (managed or external tables created with `CREATE TABLE ... USING PARQUET`). In + * every case the downstream pushdown rule sees the hoisted extraction in the + * `Project`/`Filter` chain that [[org.apache.spark.sql.catalyst.planning.PhysicalOperation]] + * collapses above the scan; an extraction embedded in an `Aggregate`/`Join`/`Sort` is invisible + * to them and the whole variant column is read raw (or, when the column is also read raw for + * pass-through, shredded with a wasteful full-variant slot). + * + * This complements the rules that already relocate variant extractions into such a `Project`: + * `PhysicalOperation` (Project/Filter), `PullOutGroupingExpressions` (GROUP BY / DISTINCT keys), + * and `ExtractWindowExpressions` (window partition/order keys and window function arguments). The + * residual, un-relocated extractions live in aggregate function arguments, join conditions, and + * sort orders. This rule handles those three: + * + * - Aggregate: `Aggregate([name], [name, max(variant_get(v, '$.p'))], child)` becomes + * `Aggregate([name], [name, max(_ve0)], Project([name, variant_get(v, '$.p') AS _ve0], child))`. + * The Aggregate defines its own output, so the raw `v` is simply not passed through. + * + * - Sort / Join: these operators pass their child's columns through, so a naive hoist would keep + * a bare `v` alongside the hoisted extraction and the pushdown would still request the full + * variant. To avoid that we match the `Project` sitting above the barrier -- its `references` + * are exactly the columns still live above the barrier -- and drop a variant column that is no + * longer referenced once its extraction has been hoisted. A variant genuinely needed raw above + * the barrier stays in `references` and is preserved (keeping its full-variant slot). + * + * A `Join`, and a `Sort` directly under a narrowing `Project`, are handled with that Project's + * `references` telling us which columns remain live: a variant used only via the hoisted + * extraction is dropped (no redundant full-variant slot). This is the common `Sort` shape when + * only a subset of columns is selected (e.g. `SELECT name ... ORDER BY variant_get(v, ...)`), + * where the analyzer adds a `Project` above the `Sort` to project away the order-only columns. + * + * A bare/root `Sort` -- one with no narrowing `Project` above it, as in + * `SELECT * FROM t ORDER BY variant_get(v, '$.b')` (all order columns are already selected, so + * the analyzer adds no narrowing `Project`) -- is also handled, by a second `transformUp` pass + * (see `apply`). Its output must equal its child's output exactly, so ALL child columns stay + * live: a variant present in the output stays raw AND gains an extra `_ve` slot, producing a + * multi-slot shredded struct with a full-variant slot -- the same shape as a `Sort`-under- + * `Project` whose `v` is also selected. `Sort`s under a `GlobalLimit`/`LocalLimit` (a + * `SELECT * ... ORDER BY ... LIMIT n`) are covered by the same second pass. A `Sort`/`Join` in + * any other shape -- under a `Filter`/`Aggregate`/another non-`Project` operator that is not a + * limit wrapper -- is left unchanged: the extraction stays in place, the variant is read raw, + * and the plan is correct (just not shredded). When the `Sort` itself sits over a `Join`, the + * hoisted order-key aliases are additionally pushed *through* the join to the scan side (see the + * through-a-`Join` note below), not merely parked in a `Project` above it. + * + * Pushing extractions *through* a `Join`: hoisting an extraction into a `Project`/child directly + * above a `Join` is not enough on its own, because `PhysicalOperation` (which both pushdown rules + * rely on) collapses only a contiguous `Project`/`Filter` chain and stops at the `Join`. A variant + * column `v` that reaches the barrier only via a hoisted `variant_get` would still flow up through + * the join as a bare attribute and be read raw at the scan. So once an extraction is hoisted above + * a `Join` (by the `Aggregate` or `Sort` case, or by the projection/condition handling of the + * `Project`-over-`Join` case), `pushSideAliases` pushes the resulting `_ve` aliases *down* through + * the join tree, routing each to the side whose output owns the referenced attribute, until it + * lands in a `Project` directly above a non-join child (the scan side). This descends any depth of + * chained joins in a single pass, so a variant fact-table column joined to several dimensions + * shreds to just the requested typed fields. Correctness is per join type: `Inner`/`Cross` and the + * outer joins (`LeftOuter`/`RightOuter`/`FullOuter`) push to either side -- the outer nullable side + * is value-preserving because `variant_get(NULL) = NULL` and null-padding commutes with the + * extraction. `LeftSemi`/`LeftAnti`/`ExistenceJoin` push only left-side aliases (their right side + * is not in the join output); a right-side alias on such a join is not pushed and the parent keeps + * a `Project` referencing it above the join. A variant still needed raw above the join stays live + * and keeps its full-variant slot, exactly as in the non-join cases. + * + * Every rewrite is a plain alias/`Project` introduction and is semantics-preserving; the pushdown + * shred/rewrite machinery is unchanged. The rule is gated on + * [[SQLConf.PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS]] (and is a no-op unless + * [[SQLConf.PUSH_VARIANT_INTO_SCAN]] is also enabled) and only fires when a hoistable extraction is + * present, so non-variant plans are untouched. + * + * Cast-error surface: relocating a strict extraction (`variant_get(..., failOnError = true)` or a + * strict `Cast`) below a `Join` means it is evaluated at the scan on rows the join later eliminates + * -- so a cast failure can surface for a row the un-hoisted plan would never have cast (here the + * eliminating rows come from the *other* joined table). This is the same pre-existing trade-off as + * [[PushVariantIntoScan]] pushing casts below a `Filter`, not a new error class: in both, the + * strict cast runs before the operator that would have discarded the failing row. This rule only + * relocates the extraction into a `Project`; [[PushVariantIntoScan]] still does the scan-level + * materialization and, when [[SQLConf.PUSH_VARIANT_INTO_SCAN_DEFER_CAST_ERROR]] is set (default + * false), wraps the cast with a per-row cast-error companion slot so the error is only raised when + * the original expression consumes the failing row. That deferral is provenance-agnostic -- it acts + * on the relocated extraction regardless of how it reached the `Project` -- so enabling the flag + * suppresses the join-eliminated-row error exactly as it does the filter-eliminated-row case. With + * the flag off (the default), the strict cast raises immediately on any failing scanned row, as + * documented for below-`Filter` pushdown. + */ +object PullOutVariantExtractions extends Rule[LogicalPlan] { + + override def apply(plan: LogicalPlan): LogicalPlan = { + // Hoisting is only useful when variant-into-scan pushdown is enabled to consume the relocated + // extractions, and is independently gated so it can be turned off on its own. + if (!SQLConf.get.getConf(SQLConf.PUSH_VARIANT_INTO_SCAN) || + !SQLConf.get.getConf(SQLConf.PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS)) { + plan + } else { + // First pass: handle operators where a narrowing Project above the barrier tells us which + // columns remain live. `transformUp` so inner operators are rewritten before their parents. + // Sort/Join are matched together with the Project above them (see class doc). Prune to + // subtrees that actually contain one of the matched operators so plans with none (and the + // Once-batch idempotence re-run) skip the traversal. + val afterProjectCases = plan.transformUpWithPruning( + _.containsAnyPattern(AGGREGATE, SORT, JOIN)) { + case agg: Aggregate => rewriteAggregate(agg) + case p @ Project(projectList, s: Sort) => rewriteSortUnderProject(p, projectList, s) + case p @ Project(projectList, j: Join) => rewriteJoinUnderProject(p, projectList, j) + } + // Second pass: handle a Sort NOT under a narrowing Project -- a bare/root Sort (as in + // `SELECT * ORDER BY variant_get(...)`), or one under a GlobalLimit/LocalLimit. This must be + // a separate pass, not another arm in the first `transformUp`: that traversal is bottom-up, + // so a bare-Sort arm would fire on the inner Sort of a `Project(_, Sort)` shape before the + // parent Project is visited, keeping v raw and pre-empting `rewriteSortUnderProject`. After + // the first pass, any Sort that was under a Project has its order rewritten to non-hoistable + // `_ve` attribute references, so `rewriteBareSort` is a no-op on it (idempotent). Prune to + // Sort-bearing subtrees so Sort-free plans skip this pass entirely. + afterProjectCases.transformUpWithPruning(_.containsPattern(SORT)) { + case s: Sort => rewriteBareSort(s) + } + } + } + + // A variant column path: an `Attribute` (possibly through `GetStructField`) typed `VariantType`. + // Used by `isHoistable` to verify the extraction's source is a scan column, not a computed + // expression. Non-column variants (e.g. `variant_get(parse_json(x), ...)`) are excluded. + private def isVariantColumnPath(e: Expression): Boolean = + e.dataType.isInstanceOf[VariantType] && StructPath.unapply(e).isDefined + + private def isHoistable(e: Expression): Boolean = e match { + // Exclude the two-arg form variant_get(v, path) whose targetType is VariantType: it produces a + // variant-typed alias that the pushdown drops (see SPARK-57499), so hoisting adds a Project for + // no benefit. It would also break Once-batch idempotence: the hoisted `_ve0` is a VariantType + // attribute, so a wrapping `cast(_ve0 as string)` (e.g. from `v:a::string`, which desugars to + // `cast(variant_get(v, '$.a') as string)`) would match the Cast branch below on the next pass. + case g: VariantGet => + !g.targetType.isInstanceOf[VariantType] && g.path.foldable && isVariantColumnPath(g.child) + // A variant-to-variant `Cast` (e.g. `cast(v as variant)`) is a whole-variant read: its sole + // requested field targets `VariantType`, which the pushdown drops (see SPARK-57499) because a + // lone variant-typed slot saves no I/O and is mishandled by the reader. Hoisting it would add a + // Project + alias for no shredding benefit and leave the column raw anyway, so exclude it. + // A hoistable `Cast` yields a non-variant-typed alias, so it is never re-hoisted (idempotent). + case c: Cast => !c.dataType.isInstanceOf[VariantType] && isVariantColumnPath(c.child) + case _ => false + } + + /** + * Collects hoisted extractions as aliases, de-duplicated by canonical form so a repeated + * extraction maps to a single output slot. + */ + private class ExtractionHoister { + private val extracted = mutable.LinkedHashMap.empty[Expression, Alias] + + def aliases: Seq[NamedExpression] = extracted.values.toSeq + + def isEmpty: Boolean = extracted.isEmpty + + /** Returns (creating if needed) the alias attribute for a single hoistable extraction. */ + def aliasFor(ex: Expression): Attribute = + extracted.getOrElseUpdate(ex.canonicalized, Alias(ex, s"_ve${extracted.size}")()).toAttribute + + /** Replaces every hoistable extraction in `e` with a reference to its (new) alias. */ + def hoist(e: Expression): Expression = e.transformDown { + case ex if isHoistable(ex) => aliasFor(ex) + } + } + + // Recursively pushes hoisted extraction aliases down through a join tree until each lands in a + // `Project` directly above a non-join child (the scan side, or the `Filter`/`Project` chain above + // it that `PhysicalOperation` collapses). Hoisting an extraction into a `Project` above a `Join` + // is not sufficient on its own, because `PhysicalOperation` stops at the `Join`: the variant + // would still be read raw at the scan. This descends any depth of chained joins in a single call. + // + // `aliases` are the extraction aliases still to be placed. Each is routed to the join side whose + // output owns its referenced attribute. `needed` is the set of attributes that must remain in the + // pushed-into child's output -- the columns referenced above the push point plus the + // join-condition references accumulated while recursing, so join keys are never dropped. A + // variant consumed solely via a hoisted extraction is thereby not passed through (no redundant + // full-variant slot); one used raw elsewhere stays in `needed` and keeps its full-variant slot. + private def pushSideAliases( + child: LogicalPlan, + aliases: Seq[NamedExpression], + needed: AttributeSet): LogicalPlan = { + if (aliases.isEmpty) { + child + } else { + child match { + case join: Join => + val leftOutput = join.left.outputSet + val rightOutput = join.right.outputSet + // For LeftSemi/LeftAnti/ExistenceJoin (all matched by `LeftExistence`) the right side is + // not in the join output, so a right-side alias could never be referenced above the join. + // Route only left-side aliases there; leave any right-side alias in place (it will not + // have been produced for such joins in practice). + val rightEligible = join.joinType match { + case LeftExistence(_) => false + case _ => true + } + val (leftAliases, rest) = + aliases.partition(_.references.subsetOf(leftOutput)) + val (rightAliases, unrouted) = + if (rightEligible) rest.partition(_.references.subsetOf(rightOutput)) + else (Seq.empty[NamedExpression], rest) + if (leftAliases.isEmpty && rightAliases.isEmpty) { + // Nothing cleanly routable (e.g. an ExistenceJoin, or an alias not confined to one + // side). Fall back to a Project above the join keeping the live columns. + val keep = join.output.filter(needed.contains) + Project(keep ++ aliases, join) + } else { + val neededHere = + needed ++ join.condition.map(_.references).getOrElse(AttributeSet.empty) + val newLeft = pushSideAliases(join.left, leftAliases, neededHere) + val newRight = pushSideAliases(join.right, rightAliases, neededHere) + val newJoin = join.copy(left = newLeft, right = newRight) + // An alias that could not be routed to a single side stays above the join. + if (unrouted.isEmpty) { + newJoin + } else { + val keep = newJoin.output.filter(needed.contains) + Project(keep ++ unrouted, newJoin) + } + } + // Descend through a pass-through `Project` (as `PhysicalOperation` does) to reach the + // join/scan beneath the intermediate Projects the optimizer leaves between chained joins. + // A `pushable` alias resolves against the grandchild and is pushed there; a `stay` alias + // references a column this Project introduces and cannot be pushed. + case project @ Project(projectList, grandChild) => + val (pushable, stay) = + aliases.partition(_.references.subsetOf(grandChild.outputSet)) + if (pushable.isEmpty) { + val keep = project.output.filter(needed.contains) + Project(keep ++ aliases, project) + } else { + // Prune to the still-live columns before re-adding the pushed alias references, else a + // bare variant consumed only via a now-pushed extraction would stay live and be read + // raw. Retain any projection a `stay` alias references so it stays resolvable. + val stayRefs = AttributeSet(stay.flatMap(_.references)) + val keptProjections = projectList.filter { e => + needed.contains(e.toAttribute) || stayRefs.contains(e.toAttribute) + } + val newGrandChild = pushSideAliases( + grandChild, pushable, needed ++ AttributeSet(keptProjections.flatMap(_.references))) + Project(keptProjections ++ pushable.map(_.toAttribute) ++ stay, newGrandChild) + } + // Any other operator -- notably a `Filter` above the scan: wrap a `Project` above it. We do + // NOT descend through a `Filter`, as that would force its variant predicate reference into + // `needed` and pass the raw variant through. The Filter and this Project collapse into the + // same `PhysicalOperation` chain, so the hoisted extraction is still seen at the scan. + case other => + val keep = other.output.filter(needed.contains) + Project(keep ++ aliases, other) + } + } + } + + // Routes hoistable extractions found in `projectList` to the owning-side hoister and returns the + // projectList with those extractions replaced by references to their (new) alias attributes. + // Mirrors the join-condition routing in `rewriteJoinUnderProject`: a single variant extraction + // references exactly one attribute, hence one join side. Anything not cleanly on one side is left + // in place. + private def hoistProjectListExtractions( + projectList: Seq[NamedExpression], + leftOutput: AttributeSet, + rightOutput: AttributeSet, + leftHoister: ExtractionHoister, + rightHoister: ExtractionHoister): Seq[NamedExpression] = { + projectList.map { e => + e.transformDown { + case ex if isHoistable(ex) => + if (ex.references.subsetOf(leftOutput)) { + leftHoister.aliasFor(ex) + } else if (ex.references.subsetOf(rightOutput)) { + rightHoister.aliasFor(ex) + } else { + ex + } + }.asInstanceOf[NamedExpression] + } + } + + private def rewriteAggregate(agg: Aggregate): LogicalPlan = { + val hoister = new ExtractionHoister + // Only hoist extractions that sit inside an aggregate function's arguments (or filter). A + // top-level extraction in `aggregateExpressions` is a grouping-key reference (grouping keys are + // already pulled out by `PullOutGroupingExpressions`); hoisting it would leave the `Aggregate` + // referencing a column that is neither grouped nor aggregated. + val newAggExprs = agg.aggregateExpressions.map { + _.transform { + case ae: AggregateExpression => ae.withNewChildren(ae.children.map(hoister.hoist)) + }.asInstanceOf[NamedExpression] + } + + if (hoister.isEmpty) { + agg + } else { + // `referenced` is what the rewritten Aggregate still needs directly (aggregate-function args + // after hoisting, plus grouping keys). A variant consumed solely via a hoisted extraction is + // not in it and so is dropped rather than passed through raw -- see the class doc. + val referenced = AttributeSet( + newAggExprs.flatMap(_.references) ++ agg.groupingExpressions.flatMap(_.references)) + val newChild = agg.child match { + // Fuse into the Project that PullOutGroupingExpressions/PhysicalOperation already placed + // below the Aggregate; the Once-batch earlyScanPushDownRules has no CollapseProject to + // flatten a second stacked Project. Sound only when the fused Project still resolves + // against `grandChild.output` (see `canFuseIntoChildProject`); else keep the nested one. + case Project(projectList, grandChild) + if canFuseIntoChildProject(projectList, referenced, hoister, grandChild) => + val keptProjections = projectList.filter(e => referenced.contains(e.toAttribute)) + val fused = Project(keptProjections ++ hoister.aliases, grandChild) + grandChild match { + // The fused Project sits directly above a Join, which `PhysicalOperation` cannot see + // through -- and this newly created node will not be revisited by the outer + // `transformUp` in this pass. Push the hoisted aliases the rest of the way down through + // the join immediately, reusing the Project-over-Join handling. + case grandChildJoin: Join => + rewriteJoinUnderProject(fused, fused.projectList, grandChildJoin) + case _ => fused + } + // No fusable child Project. Only the still-referenced columns plus the hoisted aliases + // reach the child, so a variant used solely via a hoisted extraction is dropped. + case join: Join => + // The Aggregate's child is a Join. `pushSideAliases` pushes the hoisted aliases down + // through the join tree onto the owning side(s), where they land directly above the scan; + // the Aggregate references the `_ve` aliases, which the join propagates up through its + // output. Any variant still needed raw (in `referenced`) stays live. A `Project` wrapper + // above the join would leave the aliases invisible to the pushdown (`PhysicalOperation` + // stops at the join) and would not be revisited by the outer `transformUp`, so we push + // down instead. + pushSideAliases(join, hoister.aliases, referenced) + case other => + val passthrough = other.output.filter(referenced.contains) + Project(passthrough ++ hoister.aliases, other) + } + agg.copy(aggregateExpressions = newAggExprs, child = newChild) + } + } + + // Fusing the hoisted aliases into an existing child `Project` (collapsing it onto its own child) + // is sound only if everything the fused Project carries still resolves against `grandChild`: + // - the retained projections (those whose output is still referenced), and + // - the hoisted alias expressions. + // If any of these references an attribute the Project introduces itself (not present in + // `grandChild.output`) -- e.g. a nested-variant extraction materialized as an intermediate alias + // -- collapsing would leave that reference unresolved, so we must not fuse. + private def canFuseIntoChildProject( + projectList: Seq[NamedExpression], + referenced: AttributeSet, + hoister: ExtractionHoister, + grandChild: LogicalPlan): Boolean = { + val grandChildOutput = grandChild.outputSet + val keptProjections = projectList.filter(e => referenced.contains(e.toAttribute)) + val carried = keptProjections ++ hoister.aliases + carried.forall(_.references.subsetOf(grandChildOutput)) + } + + private def rewriteSortUnderProject( + project: Project, projectList: Seq[NamedExpression], sort: Sort): LogicalPlan = { + val hoister = new ExtractionHoister + val newOrder = sort.order.map(hoister.hoist(_).asInstanceOf[SortOrder]) + if (hoister.isEmpty) { + project + } else { + // Columns still needed from the Sort's child: what the Project above consumes, plus what the + // rewritten order references directly (the alias attributes are supplied below, not from the + // child). A variant used only by the (now hoisted) order key is thereby dropped. + val needed = + AttributeSet(projectList.flatMap(_.references) ++ newOrder.flatMap(_.references)) + // `pushSideAliases` places the aliases in a Project directly above the child's non-join + // descendant, descending through any Join/Project chain between the Sort and the scan. For a + // flat child (scan / Filter above a scan) this is a single Project directly below the Sort, + // matching the previous behavior; when the Sort sits over a Join the aliases are pushed + // through it so the extraction reaches the scan instead of being read raw. + val newChild = pushSideAliases(sort.child, hoister.aliases, needed) + // Reproduce the original projectList so the alias columns do not leak into the output. + project.copy(child = sort.copy(order = newOrder, child = newChild)) + } + } + + // Handles a `Sort` that is NOT directly under a narrowing `Project` -- a bare/root Sort such as + // `SELECT * FROM t ORDER BY variant_get(v, '$.b')` (all order columns are already selected, so + // the analyzer adds no narrowing Project above the Sort), or a Sort under a + // GlobalLimit/LocalLimit. + // Unlike `rewriteSortUnderProject`, there is no Project above to tell us which columns remain + // live; a bare Sort's output IS its child's output, so ALL child columns must stay live. A + // variant present in the output (SELECT *) therefore stays raw AND gains an extra `_ve` slot, + // yielding a multi-slot shredded struct with a full-variant slot -- the same shape as a + // Sort-under-Project whose `v` is also selected. See the class doc. + private def rewriteBareSort(sort: Sort): LogicalPlan = { + val hoister = new ExtractionHoister + val newOrder = sort.order.map(hoister.hoist(_).asInstanceOf[SortOrder]) + if (hoister.isEmpty) { + sort + } else { + // All child output columns stay live: the bare Sort's output equals its child's output. The + // hoisted `_ve` aliases are supplied to `pushSideAliases` separately (not via `needed`). + val needed = sort.child.outputSet + // `pushSideAliases` places the aliases in a Project directly above the child's non-join + // descendant, descending through any Join/Project chain between the Sort and the scan. + val newChild = pushSideAliases(sort.child, hoister.aliases, needed) + // Wrap in a Project restoring the original output so the `_ve` alias columns do not leak. + // `sort.child.output` (original attributes, in order) is the correct projectList -- NOT + // `newSort.output`, which would include the `_ve` columns `pushSideAliases` added below. + // `sort.copy` preserves `sort.global` and `sort.hint`. + Project(sort.child.output, sort.copy(order = newOrder, child = newChild)) + } + } + + private def rewriteJoinUnderProject( + project: Project, projectList: Seq[NamedExpression], join: Join): LogicalPlan = { + val leftOutput = join.left.outputSet + val rightOutput = join.right.outputSet + val leftHoister = new ExtractionHoister + val rightHoister = new ExtractionHoister + + // A single variant extraction references exactly one attribute, hence one join side; route it + // to that side's hoister. Anything not cleanly on one side is left in place. We hoist from both + // the join condition (its extractions feed the equi-key comparison) and the Project above the + // join (its extractions -- e.g. aggregate arguments hoisted here by `rewriteAggregate`, or a + // user's `SELECT variant_get(...)`), so the pushdown sees them below the join. + val newCondition = join.condition.map(_.transformDown { + case ex if isHoistable(ex) => + if (ex.references.subsetOf(leftOutput)) { + leftHoister.aliasFor(ex) + } else if (ex.references.subsetOf(rightOutput)) { + rightHoister.aliasFor(ex) + } else { + ex + } + }) + val newProjectList = + hoistProjectListExtractions(projectList, leftOutput, rightOutput, leftHoister, rightHoister) + + if (leftHoister.isEmpty && rightHoister.isEmpty) { + project + } else { + // Columns still live above the Join: what the (rewritten) Project consumes and what the + // rewritten condition references directly (alias attributes are supplied by the side Projects + // below). A side's variant used only by a hoisted extraction is dropped rather than passed + // through, so no redundant full-variant slot is requested. + val needed = AttributeSet( + newProjectList.flatMap(_.references) ++ + newCondition.map(_.references).getOrElse(AttributeSet.empty)) + // `pushSideAliases` places the aliases in a Project directly above the side's non-join child + // (recursing through nested joins), so the extractions reach the scan even when the side is + // itself a chain of joins. For a flat side (scan / Filter above a scan) this is a single + // Project, matching the previous behavior. + val newJoin = join.copy( + left = pushSideAliases(join.left, leftHoister.aliases, needed), + right = pushSideAliases(join.right, rightHoister.aliases, needed), + condition = newCondition) + project.copy(projectList = newProjectList, child = newJoin) + } + } +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/AsyncOffsetSeqLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/AsyncOffsetSeqLog.scala index 14fb4620527c0..c2ae9ac0fae6b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/AsyncOffsetSeqLog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/AsyncOffsetSeqLog.scala @@ -83,7 +83,7 @@ class AsyncOffsetSeqLog( * to indicate some write error. */ def addAsync(batchId: Long, metadata: OffsetSeqBase): CompletableFuture[(Long, Boolean)] = { - require(metadata != null, "'null' metadata cannot written to a metadata log") + require(metadata != null, "'null' metadata cannot be written to a metadata log") def issueAsyncWrite(batchId: Long): CompletableFuture[Long] = { lastCommitIssuedTimestampMs.set(clock.getTimeMillis()) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala index b06a90de44c48..3f2271ac2b2c2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala @@ -133,7 +133,7 @@ class HDFSMetadataLog[T <: AnyRef : ClassTag]( * metadata has already been stored, this method will return `false`. */ override def add(batchId: Long, metadata: T): Boolean = { - require(metadata != null, "'null' metadata cannot written to a metadata log") + require(metadata != null, "'null' metadata cannot be written to a metadata log") val res = addNewBatchByStream(batchId) { output => serialize(metadata, output) } if (metadataCacheEnabled && res) batchCache.put(batchId, metadata) res diff --git a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala index 8c9e3eae49816..1ca88ce408c80 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala @@ -67,6 +67,8 @@ class RuntimeConfigSuite extends SparkFunSuite { // SQL configs assert(!conf.isModifiable(GLOBAL_TEMP_DATABASE.key)) assert(conf.isModifiable(CHECKPOINT_LOCATION.key)) + // Proto-backed cluster config is not modifiable + assert(!conf.isModifiable("spark.sql.ui.retainedExecutions")) // Core configs assert(!conf.isModifiable(config.CPUS_PER_TASK.key)) assert(!conf.isModifiable("spark.executor.cores")) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala index 98904e6976074..bcd8ba185d1dd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala @@ -285,7 +285,7 @@ class SupportsCatalogOptionsSuite extends SharedSparkSession with BeforeAndAfter sql(s"create table t1 (id bigint) using $format") } - assert(e.getMessage.contains("Cannot find catalog plugin class")) + assert(e.getMessage.contains("cannot find the plugin class")) assert(e.getMessage.contains("InvalidCatalogClass")) } finally { spark.sessionState.catalogManager.reset() diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScanSuite.scala index 67bee55c22756..d6ed274327a66 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScanSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScanSuite.scala @@ -55,6 +55,63 @@ trait PushVariantIntoScanSuiteBase extends SharedSparkSession { } } + // A single parquet-backed table for `withVariantParquetTables`: the temp view name to expose, its + // column schema, and the row literals to insert. + protected case class VariantTable(view: String, schema: String, inserts: Seq[String]) + + // Like `withVariantParquetData` but sets up several parquet tables at once, each exposed as a + // temp view under the current read path (V1 or V2, per `useV2`). Used by the join tests, which + // need a variant fact table plus dimension tables. Each table is written via V1 parquet to an + // external location, then re-read as a view so the V2 suites exercise the DSv2 scan. + protected def withVariantParquetTables(tables: VariantTable*)(body: => Unit): Unit = { + withTempPath { dir => + val setupNames = tables.map(t => s"temp_setup_${t.view}") + withTable(setupNames: _*) { + tables.zip(setupNames).foreach { case (t, setupName) => + val path = s"${dir.getCanonicalPath}/${t.view}" + sql(s"create table $setupName (${t.schema}) using PARQUET location '$path'") + t.inserts.foreach(values => sql(s"insert into $setupName values $values")) + } + } + val sourceListConf: Seq[(String, String)] = + if (useV2) Seq(SQLConf.USE_V1_SOURCE_LIST.key -> "") else Nil + withSQLConf(sourceListConf: _*) { + tables.foreach { t => + spark.read.parquet(s"${dir.getCanonicalPath}/${t.view}").createOrReplaceTempView(t.view) + } + try body finally tables.foreach(t => spark.catalog.dropTempView(t.view)) + } + } + } + + // Find the scan relation (V1 or V2) whose output contains a column named `columnName`, and return + // that column's data type. Used by multi-scan join tests to inspect the fact table's variant + // column. Fails if zero or more than one matching scan is found. + protected def scanColumnType(plan: LogicalPlan, columnName: String): DataType = { + val matching = plan.collect { + case s: DataSourceV2ScanRelation if s.output.exists(_.name == columnName) => s.output + case l: LogicalRelation if l.output.exists(_.name == columnName) => l.output + } + assert(matching.length == 1, + s"Expected exactly one scan with a column named '$columnName' but found " + + s"${matching.length}:\n$plan") + matching.head.find(_.name == columnName).get.dataType + } + + // Assert that a variant column shredded to a struct and (optionally) that it has no full-variant + // slot -- i.e. the whole blob is not read. + protected def assertShreddedStruct( + dataType: DataType, expectFullVariantSlot: Boolean = false): StructType = { + val struct = dataType match { + case s: StructType => s + case other => fail(s"Expected the variant column shredded to a struct, but got $other") + } + val hasFullVariantSlot = struct.fields.exists(_.dataType.isInstanceOf[VariantType]) + assert(hasFullVariantSlot == expectFullVariantSlot, + s"Expected full-variant slot present=$expectFullVariantSlot, but got $struct") + struct + } + protected def localTimeZone = spark.sessionState.conf.sessionLocalTimeZone // Return a `StructField` with the expected `VariantMetadata`. @@ -73,6 +130,268 @@ trait PushVariantIntoScanSuiteBase extends SharedSparkSession { } } + // Locate the single scan relation in an optimized plan and return its output, regardless of the + // read path (V1 `LogicalRelation` or V2 `DataSourceV2ScanRelation`). Lets a shared test assert + // shredding for both the V1 and V2 pushdown rules. + protected def scanRelationOutput(plan: LogicalPlan): Seq[Attribute] = { + val outputs = plan.collect { + case s: DataSourceV2ScanRelation => s.output + case l: LogicalRelation => l.output + } + assert(outputs.length == 1, + s"Expected exactly one scan relation but found ${outputs.length}:\n$plan") + outputs.head + } + + test("aggregate function argument variant_get is hoisted below the aggregate and shredded") { + // `max(variant_get(v, '$.price'))` lives inside the aggregate function, above the aggregate + // barrier that PhysicalOperation-based pushdown cannot see through. PullOutVariantExtractions + // hoists it into a Project directly below the Aggregate so both the V1 and V2 pushdown rules + // shred `v` to just the `$.price` slot, with no full-variant slot (the raw column is never + // read). Runs under both the V1 and V2 pushdown paths. + withVariantParquetData( + "v variant, v2 variant, name string", + "(parse_json('{\"price\": 10}'), parse_json('{\"junk\": 1}'), 'a')", + "(parse_json('{\"price\": 30}'), parse_json('{\"junk\": 2}'), 'a')", + "(parse_json('{\"price\": 20}'), parse_json('{\"junk\": 3}'), 'b')") { + val query = "select name, max(variant_get(v, '$.price', 'int')) as mx from T group by name" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // a -> 30, b -> 20 + val output = scanRelationOutput(sql(query).queryExecution.optimizedPlan) + val v = output.find(_.name == "v").getOrElse(fail(s"v missing from ${output.map(_.name)}")) + val vStruct = v.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + } + } + + test("sort order variant_get is hoisted below the sort and shredded") { + // The sort key variant_get(v, '$.price') lives in Sort.order. PullOutVariantExtractions hoists + // it into a Project below the Sort; since only `name` is selected (v is not otherwise live + // above the Sort), v shreds to the $.price slot with no full-variant slot. Runs under V1/V2. + withVariantParquetData( + "v variant, name string", + "(parse_json('{\"price\": 3}'), 'x')", + "(parse_json('{\"price\": 1}'), 'z')", + "(parse_json('{\"price\": 2}'), 'y')") { + val query = "select name from T order by variant_get(v, '$.price', 'int')" + val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().map(_.getString(0)).toList + } + assert(expectedOrder == List("z", "y", "x"), s"baseline sanity: $expectedOrder") + assert(sql(query).collect().map(_.getString(0)).toList == expectedOrder, + "ORDER BY variant_get produced the wrong row order") + val output = scanRelationOutput(sql(query).queryExecution.optimizedPlan) + val vStruct = output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + } + } + + test("bare/root sort (SELECT *): order-key variant_get is hoisted and shredded") { + // SELECT * keeps v in the query output, so the sort key variant_get(v, '$.price') lives in a + // bare/root Sort with no narrowing Project above it (the analyzer adds no Project since every + // order column is already selected). PullOutVariantExtractions handles this via rewriteBareSort + // in its second pass. Because v is in the SELECT * output it stays live, so v shreds to a + // multi-slot struct WITH a full-variant slot (the raw blob is still materialized) plus the + // typed $.price slot the sort reads. Runs under V1/V2. + withVariantParquetData( + "v variant, name string", + "(parse_json('{\"price\": 3}'), 'x')", + "(parse_json('{\"price\": 1}'), 'z')", + "(parse_json('{\"price\": 2}'), 'y')") { + val query = "select * from T order by variant_get(v, '$.price', 'int')" + // Order-sensitive: compare the non-variant `name` column sequence. + val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().map(_.getString(1)).toList + } + assert(expectedOrder == List("z", "y", "x"), s"baseline sanity: $expectedOrder") + assert(sql(query).collect().map(_.getString(1)).toList == expectedOrder, + "SELECT * ORDER BY variant_get produced the wrong row order") + val output = scanRelationOutput(sql(query).queryExecution.optimizedPlan) + assertShreddedStruct( + output.find(_.name == "v").get.dataType, expectFullVariantSlot = true) + } + } + + test("bare/root sort (SELECT id, v): order-key variant_get is hoisted and shredded") { + // Explicitly selecting the variant column (SELECT id, v) produces the same bare/root Sort shape + // as SELECT *: no narrowing Project above the Sort. Exercises rewriteBareSort's second pass; + // v stays live (it is selected) so it shreds with a full-variant slot plus the $.b slot. + withVariantParquetData( + "id int, v variant", + "(1, parse_json('{\"b\": 3}'))", + "(2, parse_json('{\"b\": 1}'))", + "(3, parse_json('{\"b\": 2}'))") { + val query = "select id, v from T order by variant_get(v, '$.b', 'int')" + val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().map(_.getInt(0)).toList + } + assert(expectedOrder == List(2, 3, 1), s"baseline sanity: $expectedOrder") + assert(sql(query).collect().map(_.getInt(0)).toList == expectedOrder, + "SELECT id, v ORDER BY variant_get produced the wrong row order") + val output = scanRelationOutput(sql(query).queryExecution.optimizedPlan) + assertShreddedStruct( + output.find(_.name == "v").get.dataType, expectFullVariantSlot = true) + } + } + + test("bare sort over a join (SELECT *): order-key extraction pushed through the join") { + // A bare/root Sort whose child is a Join, with all join columns selected (no narrowing Project + // above the Sort). The hoisted order-key alias must be pushed DOWN through the join to the ss + // scan side. `data` is in the query output, so it shreds with a full-variant slot plus the + // $.qty slot. Runs under V1/V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')", "(2, 'y')"))) { + val query = + "select ss.k, ss.data, d.name from SS ss join DIM d on ss.k = d.k " + + "order by variant_get(ss.data, '$.qty', 'double')" + val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().map(_.getInt(0)).toList + } + assert(sql(query).collect().map(_.getInt(0)).toList == expectedOrder, + s"SELECT with join columns ORDER BY variant_get produced the wrong row order: " + + s"got ${sql(query).collect().map(_.getInt(0)).toList}, expected $expectedOrder") + assertShreddedStruct( + scanColumnType(sql(query).queryExecution.optimizedPlan, "data"), + expectFullVariantSlot = true) + } + } + + test("cast(v as variant) in aggregate function arg is not hoisted") { + // The aggregate argument is `cast(v as variant)`: a whole-variant read whose sole requested + // field targets VariantType. PullOutVariantExtractions must NOT hoist it (isHoistable excludes + // VariantType-target casts) -- hoisting would add a Project + `_ve` alias for no shredding + // benefit (the pushdown drops a lone VariantType slot anyway, see SPARK-57499). Assert the rule + // introduces no hoist alias, so the plan matches what the pull-out rule being off would give. + // Plan-shape only (no execution): a lone-VariantType shred is a separate, pre-existing concern. + // Runs under both the V1 and V2 pushdown paths. + withVariantParquetData( + "v variant, name string", + "(parse_json('{\"price\": 10}'), 'a')", + "(parse_json('{\"price\": 20}'), 'b')") { + val query = "select name, count(cast(v as variant)) as c from T group by name" + val plan = sql(query).queryExecution.optimizedPlan + val hoistAlias = plan.exists(_.expressions.exists(_.exists { + case a: Alias => a.name.startsWith("_ve") + case _ => false + })) + assert(!hoistAlias, + s"Expected no `_ve` hoist alias for a VariantType-target cast, but plan was:\n$plan") + } + } + + test("two-arg variant_get (returns VariantType) in aggregate function arg is not hoisted") { + // variant_get(v, '$.price') with no target type returns VariantType. isHoistable must NOT hoist + // it: the pushdown drops a lone VariantType requested field (SPARK-57499), so hoisting + // produces a `_ve` alias for no shredding benefit. It would also re-match on the second + // idempotence pass, breaking the Once-batch check. Assert no `_ve` alias is introduced. + withVariantParquetData( + "v variant, name string", + "(parse_json('{\"price\": 10}'), 'a')", + "(parse_json('{\"price\": 20}'), 'b')") { + val query = "select name, count(variant_get(v, '$.price')) as c from T group by name" + val plan = sql(query).queryExecution.optimizedPlan + val hoistAlias = plan.exists(_.expressions.exists(_.exists { + case a: Alias => a.name.startsWith("_ve") + case _ => false + })) + assert(!hoistAlias, + s"Expected no `_ve` hoist alias for a two-arg variant_get (VariantType result), " + + s"but plan was:\n$plan") + } + } + + test("listagg(DISTINCT v:a::string) WITHIN GROUP (ORDER BY v:a::string) does not break " + + "Once-batch idempotence") { + // Regression test for the idempotence bug: `v:a::string` desugars to + // `cast(variant_get(v, '$.a') as string)`, where the inner two-arg variant_get returns + // VariantType. Excluding that two-arg form from hoisting (isHoistable's VariantGet branch) + // leaves the whole `cast(variant_get(...))` untouched -- so nothing is hoisted here and there + // is no VariantType `_ve` alias for the Cast branch to re-hoist on the next pass. Without the + // exclusion the VariantGet would hoist to `_ve0: VariantType`, then `cast(_ve0 as string)` + // would re-match, producing a different plan and crashing with + // "Once strategy's idempotence is broken". + withVariantParquetData( + "v variant", + "(parse_json('{\"a\": \"x\"}'))", + "(parse_json('{\"a\": \"y\"}'))", + "(parse_json('{\"a\": \"x\"}'))") { + val query = + "select listagg(distinct variant_get(v, '$.a', 'string'), ',') " + + "within group (order by variant_get(v, '$.a', 'string')) from T" + // Must execute without an idempotence exception and return the expected deduplicated result. + checkAnswer(sql(query), Seq(org.apache.spark.sql.Row("x,y"))) + } + } + + test("sort key hoisted while a different projected extraction stays above the sort: correct") { + // The projected extraction variant_get(v, '$.a') sits in the select list (above the Sort, + // lifted by PhysicalOperation), while the sort key variant_get(v, '$.b') is hoisted below the + // Sort. Because the projectList still references v raw, the hoist must NOT drop v -- it stays + // live above the Sort so the projected extraction can be computed. This guards the liveness + // computation in rewriteSortUnderProject: results and row order must both be correct. Runs + // under both the V1 and V2 pushdown paths. + withVariantParquetData( + "v variant, name string", + "(parse_json('{\"a\": 100, \"b\": 3}'), 'x')", + "(parse_json('{\"a\": 200, \"b\": 1}'), 'z')", + "(parse_json('{\"a\": 300, \"b\": 2}'), 'y')") { + val query = + "select name, variant_get(v, '$.a', 'int') as a " + + "from T order by variant_get(v, '$.b', 'int')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().toSeq + } + // Baseline sanity: ordered by $.b ascending -> z(1), y(2), x(3) with their $.a values. + assert(expected.map(r => (r.getString(0), r.getInt(1))) == + Seq(("z", 200), ("y", 300), ("x", 100)), s"baseline sanity: $expected") + // Order-sensitive comparison: a mis-hoist that dropped v or mis-shredded would change the + // projected $.a value or the row order, which an order-insensitive checkAnswer would miss. + assert(sql(query).collect().toSeq.map(r => (r.getString(0), r.getInt(1))) == + expected.map(r => (r.getString(0), r.getInt(1))), + "projected extraction / order incorrect after hoisting the sort key") + } + } + + test("group by + aggregate hoist fuses into one Project, no stacked Project") { + // GROUP BY variant_get(v, '$.k') is relocated below the Aggregate by PullOutGroupingExpressions + // (creating a Project), and the aggregated variant_get(v, '$.price') is hoisted by + // PullOutVariantExtractions. The hoist must FUSE its aliases into that existing Project rather + // than stack a second one -- the earlyScanPushDownRules batch runs Once with no CollapseProject + // to flatten it afterward. Assert no two directly-stacked Projects survive in the optimized + // plan. Runs under both the V1 and V2 pushdown paths. + withVariantParquetData( + "v variant", + "(parse_json('{\"k\": \"a\", \"price\": 10}'))", + "(parse_json('{\"k\": \"a\", \"price\": 30}'))", + "(parse_json('{\"k\": \"b\", \"price\": 20}'))") { + val query = "select variant_get(v, '$.k', 'string') as k, " + + "max(variant_get(v, '$.price', 'int')) as mx " + + "from T group by variant_get(v, '$.k', 'string')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // a -> 30, b -> 20 + val plan = sql(query).queryExecution.optimizedPlan + val stacked = plan.exists { + case Project(_, _: Project) => true + case _ => false + } + assert(!stacked, + s"Expected no directly-stacked Project nodes after hoist fusion, but got:\n$plan") + } + } + // Returns true iff `t` or any of its causes is an INVALID_VARIANT_CAST error. The failure may // surface directly or be wrapped in a task failure. protected def hasCastCondition(t: Throwable): Boolean = t match { @@ -275,6 +594,317 @@ trait PushVariantIntoScanSuiteBase extends SharedSparkSession { } } } + + // A variant fact table `SS(k int, data variant)` used by the push-through-join tests below. + private def factTable(view: String = "SS"): VariantTable = VariantTable( + view, "k int, data variant", + Seq( + "(1, parse_json('{\"qty\": 10, \"id\": \"a\"}'))", + "(1, parse_json('{\"qty\": 30, \"id\": \"a\"}'))", + "(2, parse_json('{\"qty\": 20, \"id\": \"b\"}'))")) + + test("aggregate over a single inner join: variant hoisted through the join and shredded") { + // AVG(variant_get(ss.data, '$.qty')) sits in an Aggregate whose child is a Join. The extraction + // is hoisted, then pushed down through the join onto the ss side so it lands directly above the + // ss scan. `data` is used ONLY via extractions (aggregate arg + GROUP BY key), so it shreds + // with no full-variant slot -- the whole blob is never read. Runs under both V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')", "(2, 'y')"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "avg(variant_get(ss.data, '$.qty', 'double')) as avg_qty " + + "from SS ss join DIM d on ss.k = d.k " + + "group by variant_get(ss.data, '$.id', 'string')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // a -> 20.0, b -> 20.0 + assertShreddedStruct(scanColumnType(sql(query).queryExecution.optimizedPlan, "data")) + } + } + + test("aggregate over chained joins: variant pushed through all join levels and shredded") { + // The ss variant is joined to three dimensions. The aggregate-argument extractions must be + // pushed down through EVERY join level to reach the ss scan. This guards the recursion in + // pushSideAliases: a single non-recursive hoist would leave the extraction above the outermost + // join and `data` would still be read raw. Runs under both V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("D1", "k int, a string", Seq("(1, 'p')", "(2, 'q')")), + VariantTable("D2", "k int, b string", Seq("(1, 'r')", "(2, 's')")), + VariantTable("D3", "k int, c string", Seq("(1, 't')", "(2, 'u')"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "max(variant_get(ss.data, '$.qty', 'double')) as max_qty " + + "from SS ss " + + "join D1 on ss.k = D1.k join D2 on ss.k = D2.k join D3 on ss.k = D3.k " + + "group by variant_get(ss.data, '$.id', 'string')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // a -> 30.0, b -> 20.0 + assertShreddedStruct(scanColumnType(sql(query).queryExecution.optimizedPlan, "data")) + } + } + + test("variant used raw and extracted across a join: keeps a full-variant slot") { + // ss.data is both selected raw (needs the whole blob) and extracted. After the extraction is + // pushed to the ss scan side, `data` stays live (still referenced raw), so it shreds to the + // typed slot PLUS a full-variant slot -- the raw value is preserved. Runs under both V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')", "(2, 'y')"))) { + val query = + "select ss.data as raw, variant_get(ss.data, '$.qty', 'double') as qty " + + "from SS ss join DIM d on ss.k = d.k" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) + val struct = + assertShreddedStruct( + scanColumnType(sql(query).queryExecution.optimizedPlan, "data"), + expectFullVariantSlot = true) + // One typed slot ($.qty) plus the full-variant slot. + assert(struct.fields.length == 2, s"Expected two slots, but got $struct") + } + } + + test("aggregate over a left outer join: extraction on the preserved side is shredded") { + // ss is the preserved (left) side of a LEFT JOIN. Its variant extraction pushes onto the left + // side; unmatched rows (null-padded on the right) do not affect the ss-side extraction. Assert + // no full-variant slot and correct results including the unmatched row. Runs under V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "max(variant_get(ss.data, '$.qty', 'double')) as max_qty " + + "from SS ss left join DIM d on ss.k = d.k " + + "group by variant_get(ss.data, '$.id', 'string')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // a -> 30.0 (k=1 matched), b -> 20.0 (k=2 unmatched) + assertShreddedStruct(scanColumnType(sql(query).queryExecution.optimizedPlan, "data")) + } + } + + test("pull-out flag off reads the whole variant across a join") { + // With PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS disabled, the aggregate-argument extractions + // are not hoisted or pushed through the join. `ss.data` flows up through the join as a bare + // attribute, so the scan reads the whole blob -- either left raw as VariantType, or shredded to + // a single full-variant slot (struct<0:variant>). Either way there is no shredding benefit. + // Guards the config gate for the new join push-through path. Runs under both V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')", "(2, 'y')"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "avg(variant_get(ss.data, '$.qty', 'double')) as avg_qty " + + "from SS ss join DIM d on ss.k = d.k " + + "group by variant_get(ss.data, '$.id', 'string')" + withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS.key -> "false") { + val dataType = scanColumnType(sql(query).queryExecution.optimizedPlan, "data") + val readsWholeBlob = dataType match { + case _: VariantType => true + case s: StructType => s.fields.exists(_.dataType.isInstanceOf[VariantType]) + case _ => false + } + assert(readsWholeBlob, + s"Expected data read as a whole blob with the pull-out flag off, but got $dataType") + } + } + } + + test("aggregate over a left semi join: left-side extraction pushed and shredded") { + // A LEFT SEMI join's output is only the left (ss) columns. The ss-side aggregate extraction is + // pushed onto the left side and shreds with no full-variant slot; the semi join's existence + // semantics are unchanged. Runs under both V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int", Seq("(1)"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "max(variant_get(ss.data, '$.qty', 'double')) as max_qty " + + "from SS ss left semi join DIM d on ss.k = d.k " + + "group by variant_get(ss.data, '$.id', 'string')" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // only k=1 rows survive -> a -> 30.0 + assertShreddedStruct(scanColumnType(sql(query).queryExecution.optimizedPlan, "data")) + } + } + + test("left outer join: extraction on the NULLABLE (right) side is pushed and value-preserving") { + // The extraction reads the DIM (right) variant, which is the NULLABLE side of a LEFT OUTER + // join: unmatched left rows are null-padded on the right. Pushing the extraction below the join + // computes variant_get BEFORE null-padding; the join then null-pads the `_ve` alias for + // unmatched rows. This must equal the baseline, where variant_get(NULL) is evaluated ABOVE the + // join for those rows -- i.e. null-padding commutes with the extraction (variant_get(NULL) = + // NULL). Asserting correctness here is the real point; also assert DIM.data shreds. The `ss` + // (left) side has its own unrelated extraction so both scans are exercised. Runs under V1/V2. + withVariantParquetTables( + factTable(), + VariantTable( + "DIM", "k int, data variant", + // Only k=1 matches SS's k in {1, 2}; SS rows with k=2 are unmatched -> right side null. + Seq("(1, parse_json('{\"label\": \"L1\"}'))"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id, " + + "variant_get(d.data, '$.label', 'string') as label " + + "from SS ss left join DIM d on ss.k = d.k " + + "order by id, label" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + // Order-sensitive comparison so a wrong null for unmatched rows cannot be masked. + assert(sql(query).collect().toSeq.map(r => (r.get(0), r.get(1))) == + expected.toSeq.map(r => (r.get(0), r.get(1))), + "Nullable-side extraction over a left outer join produced wrong values") + // Both SS.data and DIM.data are extracted only via variant_get; each shreds with no + // full-variant slot on its own scan (both scans expose a `data` column, so assert per scan). + val plan = sql(query).queryExecution.optimizedPlan + val dataTypes = plan.collect { + case s: DataSourceV2ScanRelation => s.output + case l: LogicalRelation => l.output + }.flatMap(_.filter(_.name == "data")).map(_.dataType) + assert(dataTypes.length == 2, s"Expected two `data` scan columns, but got:\n$plan") + dataTypes.foreach(dt => assertShreddedStruct(dt)) + } + } + + test("full outer join: extractions on both (nullable) sides are pushed and value-preserving") { + // In a FULL OUTER join both sides are nullable: unmatched rows are null-padded on the opposite + // side. Extractions on each side push onto that side, computed before null-padding; the join + // null-pads the resulting `_ve` aliases for the unmatched rows. Result must match the baseline + // that evaluates variant_get above the join (variant_get(NULL) = NULL). Runs under V1 and V2. + withVariantParquetTables( + VariantTable( + "L", "k int, data variant", + Seq("(1, parse_json('{\"lv\": \"a\"}'))", "(2, parse_json('{\"lv\": \"b\"}'))")), + VariantTable( + "R", "k int, data variant", + Seq("(2, parse_json('{\"rv\": \"y\"}'))", "(3, parse_json('{\"rv\": \"z\"}'))"))) { + val query = + "select variant_get(l.data, '$.lv', 'string') as lv, " + + "variant_get(r.data, '$.rv', 'string') as rv " + + "from L l full outer join R r on l.k = r.k " + + "order by lv, rv" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + assert(sql(query).collect().toSeq.map(r => (r.get(0), r.get(1))) == + expected.toSeq.map(r => (r.get(0), r.get(1))), + "Both-nullable-side extractions over a full outer join produced wrong values") + // Both L.data and R.data are extracted only via variant_get, so each shreds with no + // full-variant slot on its own scan. Both scans expose a column named `data`, so assert on + // every `data` scan output rather than via the single-match scanColumnType helper. + val plan = sql(query).queryExecution.optimizedPlan + val dataTypes = plan.collect { + case s: DataSourceV2ScanRelation => s.output + case l: LogicalRelation => l.output + }.flatMap(_.filter(_.name == "data")).map(_.dataType) + assert(dataTypes.length == 2, s"Expected two `data` scan columns, but got:\n$plan") + dataTypes.foreach(dt => assertShreddedStruct(dt)) + } + } + + test("strict cast pushed through a join: cast-error deferral covers join-eliminated rows") { + // SS has a row whose data is a non-castable type; that row is eliminated by the join (its k + // has no match in DIM). The aggregate-argument extraction `cast(ss.data as int)` is hoisted + // onto the SS side and pushed through the join to the scan. + // - With deferral OFF: the strict cast is evaluated at the scan on the eliminated row and + // raises INVALID_VARIANT_CAST (the same trade-off as below-Filter pushdown). + // - With deferral ON: the cast-error companion slot carries the error instead; the join + // eliminates the row before its error is consumed; the query returns correct values. + // Data: SS has k=1 (string data, bad for cast) and k=2 (int data, good); DIM has k=2 only. + // Result: only the k=2 row survives the join and is included in the aggregate. + withVariantParquetTables( + VariantTable( + "SS", "k int, data variant", + Seq("(1, parse_json('\"hello\"'))", "(2, parse_json('42'))")), + VariantTable( + "DIM", "k int, name string", + Seq("(2, 'match')"))) { + val query = "select avg(cast(ss.data as int)) as avg_val from SS ss join DIM d on ss.k = d.k" + + // Without deferral, the strict cast pushed into the scan raises on the k=1 row (malformed) + // even though that row is eliminated by the join condition ss.k = d.k (no k=1 in DIM). + withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN_DEFER_CAST_ERROR.key -> "false") { + val ex = intercept[Exception](sql(query).collect()) + assert(hasCastCondition(ex), s"Expected INVALID_VARIANT_CAST, got $ex") + } + + // With deferral, the cast error is deferred and not raised because the failing row is + // eliminated before its error is consumed. The query returns the correct average. + withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN_DEFER_CAST_ERROR.key -> "true") { + val rows = sql(query).collect() + assert(rows.length == 1, s"Expected 1 row, got ${rows.length}") + val avgVal = rows(0).getDouble(0) + assert(avgVal == 42.0, s"Expected avg=42.0 (only k=2 row survives), got $avgVal") + } + } + } + + test("left semi join with a right-side variant_get in the condition: right side shredded") { + // The join condition extracts from the RIGHT (DIM) variant. For a LEFT SEMI join the right side + // is not in the join output, but the condition is still evaluated over both children, so the + // right-side extraction is hoisted onto the right side and its `_ve` alias feeds the condition. + // Exercises the right-side routing in rewriteJoinUnderProject for a LeftExistence join. Assert + // correct existence semantics and that DIM.data shreds with no full-variant slot. V1 and V2. + withVariantParquetTables( + factTable(), + VariantTable( + "DIM", "dk int, data variant", + Seq("(1, parse_json('{\"jk\": 1}'))", "(2, parse_json('{\"jk\": 9}'))"))) { + val query = + "select variant_get(ss.data, '$.id', 'string') as id " + + "from SS ss left semi join DIM d " + + "on ss.k = variant_get(d.data, '$.jk', 'int') " + + "order by id" + val expected = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expected) // ss.k in {1,2}; DIM jk in {1,9}; only k=1 rows survive + // DIM.data (right side) is used only via the condition extraction, so it shreds to the $.jk + // slot with no full-variant slot. Both SS and DIM expose a `data` column; assert both scans + // shred (the SS-side `$.id` extraction and the DIM-side `$.jk` condition extraction). + val plan = sql(query).queryExecution.optimizedPlan + val dataTypes = plan.collect { + case s: DataSourceV2ScanRelation => s.output + case l: LogicalRelation => l.output + }.flatMap(_.filter(_.name == "data")).map(_.dataType) + assert(dataTypes.length == 2, s"Expected two `data` scan columns, but got:\n$plan") + dataTypes.foreach(dt => assertShreddedStruct(dt)) + } + } + + test("sort over a join: order-key extraction is pushed through the join and shredded") { + // ORDER BY variant_get(ss.data, ...) sits in a Sort whose child is a Join (via the intermediate + // Project the optimizer leaves above the join). The hoisted order-key alias must be pushed DOWN + // through the join onto the ss side to reach the ss scan; parking it in a Project above the + // join would leave `data` read raw (PhysicalOperation stops at the join). Only ss.k is + // selected, so `data` is used only via the order key and shreds with no full-variant slot. + // Runs under V1/V2. + withVariantParquetTables( + factTable(), + VariantTable("DIM", "k int, name string", Seq("(1, 'x')", "(2, 'y')"))) { + val query = + "select ss.k from SS ss join DIM d on ss.k = d.k " + + "order by variant_get(ss.data, '$.qty', 'double')" + // Order-sensitive: compare the exact row sequence against the pushdown-disabled baseline. + val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect().map(_.getInt(0)).toList + } + assert(sql(query).collect().map(_.getInt(0)).toList == expectedOrder, + s"ORDER BY variant_get over a join produced the wrong row order: got " + + s"${sql(query).collect().map(_.getInt(0)).toList}, expected $expectedOrder") + assertShreddedStruct(scanColumnType(sql(query).queryExecution.optimizedPlan, "data")) + } + } } // V1 DataSource tests with parameterized reader type @@ -651,6 +1281,72 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } } + test(s"V2 test - nested variant extraction in aggregate function arg is hoisted and shredded " + + s"($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (vs struct, name string) " + + s"using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(named_struct('v1', parse_json('{\"price\": 10}'), 'i', 1), 'a'), " + + "(named_struct('v1', parse_json('{\"price\": 30}'), 'i', 2), 'a'), " + + "(named_struct('v1', parse_json('{\"price\": 20}'), 'i', 3), 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // The extraction is on a variant nested in a struct (vs.v1). Its child is a GetStructField + // chain rooted at an attribute, which isVariantColumnPath accepts, so it is hoisted and the + // nested variant shreds to the $.price slot. + val query = + "select name, max(variant_get(vs.v1, '$.price', 'int')) as mx from T_V2 group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) // a -> 30, b -> 20 + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vsStruct = scanRelation.output.find(_.name == "vs").get.dataType match { + case s: StructType => s + case other => fail(s"Expected vs to stay a struct, but got $other") + } + val v1Struct = vsStruct.fields.find(_.name == "v1").get.dataType match { + case s: StructType => s + case other => fail(s"Expected nested v1 shredded to struct, but got $other") + } + assert(!v1Struct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot in nested v1, but got $v1Struct") + } + } + } + + test(s"V2 test - cast(variant) in aggregate function arg is hoisted and shredded ($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant, name string) using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('\"10\"'), 'a'), (parse_json('\"30\"'), 'a'), (parse_json('\"20\"'), 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // The aggregate argument is `cast(v as string)`, not variant_get. isHoistable's Cast branch + // (matching collectRequestedFields) makes it eligible, so v shreds to the cast slot. + val query = "select name, max(cast(v as string)) as mx from T_V2 group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vStruct = scanRelation.output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + } + } + } + test(s"V2 test - no pushdown when struct is used ($readerName)") { withTempPath { dir => val path = dir.getCanonicalPath @@ -1031,7 +1727,7 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } } - test(s"V2 test - order by variant_get: correct ordering, variant read raw, sibling pruned " + + test(s"V2 test - order by variant_get: hoisted and shredded, correct ordering, sibling pruned " + s"($readerName)") { withTempPath { dir => val path = dir.getCanonicalPath @@ -1045,11 +1741,11 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { spark.read.parquet(path).createOrReplaceTempView("T_V2") - // The sort key variant_get(v, '$.price') lives in Sort.order, above the scan window, so v - // is lifted in only as a bare reference -> a whole-variant request -> v stays raw and the - // sort evaluates on the real variant. Compare ORDER-SENSITIVELY: a shredded full-variant - // slot would collapse to a boolean placeholder and silently mis-order, which the - // order-insensitive checkAnswer would not catch. + // The sort key variant_get(v, '$.price') lives in Sort.order. PullOutVariantExtractions + // hoists it into a Project below the Sort, and since v is not otherwise referenced above + // the Sort (only `name` is selected), the raw v is dropped and v shreds to the $.price slot + // (no full-variant slot). Compare ORDER-SENSITIVELY: a placeholder slot would silently + // mis-order, which the order-insensitive checkAnswer would not catch. val query = "select name from T_V2 order by variant_get(v, '$.price', 'int')" val expectedOrder = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { sql(query).collect().map(_.getString(0)).toList @@ -1061,15 +1757,18 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) assert(scanRelation.output.map(_.name).toSet == Set("v", "name"), s"Expected scan output {v, name} but got ${scanRelation.output.map(_.name)}") - val vAttr = scanRelation.output.find(_.name == "v").get - assert(vAttr.dataType.isInstanceOf[VariantType], - s"Expected v left as raw VariantType, but got ${vAttr.dataType}") + val vStruct = scanRelation.output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") } } } - test(s"V2 test - aggregate max(variant_get) with no local filter: no codegen crash, variant " + - s"read raw ($readerName)") { + test(s"V2 test - aggregate max(variant_get) with no local filter: hoisted and shredded " + + s"($readerName)") { withTempPath { dir => val path = dir.getCanonicalPath withTable("temp_v1") { @@ -1082,11 +1781,10 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { spark.read.parquet(path).createOrReplaceTempView("T_V2") - // variant_get(v, '$.price') is inside an aggregate function, above the aggregate barrier, - // with no local filter/projection on v, so v is lifted in only as a bare reference -> a - // whole-variant request. A whole-variant read is kept raw: shredding it to a lone - // full-variant slot would collapse to a boolean placeholder, and max(variant_get(, - // ...)) would fail to codegen. Keeping v raw yields correct aggregates with a valid plan. + // variant_get(v, '$.price') is inside an aggregate function, above the aggregate barrier. + // PullOutVariantExtractions hoists it into a Project directly below the Aggregate, so the + // pushdown shreds v to just the $.price slot: no full-variant slot (so no boolean + // placeholder to break codegen), and the raw column is never read. val query = "select name, max(variant_get(v, '$.price', 'int')) as mx from T_V2 group by name" val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { @@ -1095,15 +1793,152 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant checkAnswer(sql(query), expectedRows) // a -> 30, b -> 20 val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) val vAttr = scanRelation.output.find(_.name == "v").get - assert(vAttr.dataType.isInstanceOf[VariantType], - s"Expected v left as raw VariantType, but got ${vAttr.dataType}") + val vStruct = vAttr.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") assert(!scanRelation.output.exists(_.name == "v2"), s"Expected v2 pruned but got ${scanRelation.output.map(_.name)}") } } } - test(s"V2 test - join on variant_get key: no crash, variant read raw, sibling pruned " + + test(s"V2 test - aggregate over the whole variant is left raw, no hoist ($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant, name string) using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('{\"price\": 10}'), 'a'), (parse_json('{\"price\": 20}'), 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // count(v) references the whole variant, not a field extraction. PullOutVariantExtractions + // has nothing to hoist, and the pushdown keeps v raw (shredding a sole full-variant request + // saves no I/O). + val query = "select name, count(v) as c from T_V2 group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vAttr = scanRelation.output.find(_.name == "v").get + assert(vAttr.dataType.isInstanceOf[VariantType], + s"Expected v left raw, but got ${vAttr.dataType}") + } + } + } + + test(s"V2 test - GROUP BY key and aggregate function argument both shredded, dedup " + + s"($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant) using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('{\"k\": \"a\", \"price\": 10}')), " + + "(parse_json('{\"k\": \"a\", \"price\": 30}')), " + + "(parse_json('{\"k\": \"b\", \"price\": 20}'))") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // $.k is the GROUP BY key (relocated below the Aggregate by PullOutGroupingExpressions); + // $.price appears twice inside aggregate functions (relocated by the pull-out rule). + // The two $.price extractions must dedup to a single struct slot, and there must be no + // full-variant slot. + val query = "select variant_get(v, '$.k', 'string') as k, " + + "max(variant_get(v, '$.price', 'int')) as mx, " + + "min(variant_get(v, '$.price', 'int')) as mn " + + "from T_V2 group by variant_get(v, '$.k', 'string')" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) // a -> (30, 10), b -> (20, 20) + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vStruct = scanRelation.output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + // Exactly two data slots: one for $.k, one shared for the two $.price extractions. + assert(vStruct.fields.length == 2, + s"Expected 2 shredded fields ($$.k and a deduped $$.price), but got $vStruct") + } + } + } + + test(s"V2 test - aggregate function arg with a non-literal path is not hoisted ($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant, p string, name string) " + + s"using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('{\"price\": 10}'), '$.price', 'a'), " + + "(parse_json('{\"price\": 20}'), '$.price', 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // The extraction path is a column (non-foldable), which the pushdown cannot handle, so the + // rule must not hoist it and v is read raw. Guards the `path.foldable` predicate. + val query = "select name, max(variant_get(v, p, 'int')) as mx from T_V2 group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vAttr = scanRelation.output.find(_.name == "v").get + assert(vAttr.dataType.isInstanceOf[VariantType], + s"Expected v left raw for a non-literal path, but got ${vAttr.dataType}") + } + } + } + + test(s"V2 test - join selecting the whole variant keeps v raw ($readerName)") { + withTempPath { dir => + val itemsPath = dir.getCanonicalPath + "/items" + val countriesPath = dir.getCanonicalPath + "/countries" + withTable("temp_items", "temp_countries") { + sql(s"create table temp_items (v variant, name string) using PARQUET location '$itemsPath'") + sql("insert into temp_items values " + + "(parse_json('{\"country_code\": \"CN\"}'), 'widget'), " + + "(parse_json('{\"country_code\": \"JP\"}'), 'gadget')") + sql(s"create table temp_countries (code string, country_name string) " + + s"using PARQUET location '$countriesPath'") + sql("insert into temp_countries values ('CN', 'China'), ('JP', 'Japan')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(itemsPath).createOrReplaceTempView("ITEMS_V2") + spark.read.parquet(countriesPath).createOrReplaceTempView("COUNTRIES_V2") + // i.v is selected raw (needed above the Join) AND used in the join key. The hoist must NOT + // drop it: v shreds with a full-variant slot plus the $.country_code slot. Guards the join + // liveness computation. + val query = + "select i.v, c.country_name from ITEMS_V2 i join COUNTRIES_V2 c " + + "on variant_get(i.v, '$.country_code', 'string') = c.code" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scans = sql(query).queryExecution.optimizedPlan.collect { + case s: DataSourceV2ScanRelation => s + } + val itemsScan = scans.find(_.output.exists(_.name == "v")).getOrElse( + fail(s"Could not find the items scan in:\n${sql(query).queryExecution.optimizedPlan}")) + val vStruct = itemsScan.output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected a full-variant slot because i.v is selected raw, but got $vStruct") + } + } + } + + test(s"V2 test - join on variant_get key: hoisted and shredded, sibling pruned " + s"($readerName)") { withTempPath { dir => val itemsPath = dir.getCanonicalPath + "/items" @@ -1127,21 +1962,125 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { sql(query).collect() } - // A variant_get join key produces a valid plan (no attribute-binding failure) and - // correct results; the key is read raw and the extraction is evaluated above the scan. checkAnswer(sql(query), expectedRows) val scans = sql(query).queryExecution.optimizedPlan.collect { case s: DataSourceV2ScanRelation => s } val itemsScan = scans.find(_.output.exists(_.name == "v")).getOrElse( fail(s"Could not find the items scan in:\n${sql(query).queryExecution.optimizedPlan}")) - // v is the join key (referenced only via variant_get in the join condition): it must be - // left as a raw variant, not shredded. v2 is unreferenced and must be pruned. + // v is the join key (referenced only via variant_get in the join condition). + // PullOutVariantExtractions hoists the extraction onto the items side, dropping the raw v + // so it shreds to just the $.country_code slot (no full-variant slot). v2 is pruned. assert(itemsScan.output.map(_.name).toSet == Set("v", "name"), s"Expected items scan output {v, name} but got ${itemsScan.output.map(_.name)}") + val vStruct = itemsScan.output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + } + } + } + + test(s"V2 test - self join on variant_get key: both sides shredded ($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant, name string) using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('{\"id\": 1}'), 'a'), (parse_json('{\"id\": 2}'), 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // Both sides' join keys are variant_get(_, '$.id'); each is hoisted onto its own side and + // both variant columns shred to the $.id slot with no full-variant slot. + val query = "select a.name, b.name from T_V2 a join T_V2 b " + + "on variant_get(a.v, '$.id', 'int') = variant_get(b.v, '$.id', 'int')" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scans = sql(query).queryExecution.optimizedPlan.collect { + case s: DataSourceV2ScanRelation => s + } + assert(scans.length == 2, s"Expected two scans but got ${scans.length}") + scans.foreach { s => + val vStruct = s.output.find(_.name == "v").get.dataType match { + case st: StructType => st + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") + } + } + } + } + + test(s"V2 test - join on variant_get key with no narrowing Project: left raw, correct " + + s"($readerName)") { + withTempPath { dir => + val itemsPath = dir.getCanonicalPath + "/items" + val countriesPath = dir.getCanonicalPath + "/countries" + withTable("temp_items", "temp_countries") { + sql(s"create table temp_items (v variant, name string) using PARQUET location '$itemsPath'") + sql("insert into temp_items values " + + "(parse_json('{\"country_code\": \"CN\"}'), 'widget'), " + + "(parse_json('{\"country_code\": \"JP\"}'), 'gadget')") + sql(s"create table temp_countries (code string, country_name string) " + + s"using PARQUET location '$countriesPath'") + sql("insert into temp_countries values ('CN', 'China'), ('JP', 'Japan')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + spark.read.parquet(itemsPath).createOrReplaceTempView("ITEMS_V2") + spark.read.parquet(countriesPath).createOrReplaceTempView("COUNTRIES_V2") + // `select *` leaves the Join without a narrowing Project directly above it, so + // PullOutVariantExtractions does not hoist the join-key extraction (see class doc: hoisting + // requires a Project above the barrier to know the live-above set). The plan must still be + // correct with v read raw -- this locks in the documented fallback behavior. + val query = + "select * from ITEMS_V2 i join COUNTRIES_V2 c " + + "on variant_get(i.v, '$.country_code', 'string') = c.code" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scans = sql(query).queryExecution.optimizedPlan.collect { + case s: DataSourceV2ScanRelation => s + } + val itemsScan = scans.find(_.output.exists(_.name == "v")).getOrElse( + fail(s"Could not find the items scan in:\n${sql(query).queryExecution.optimizedPlan}")) val vAttr = itemsScan.output.find(_.name == "v").get assert(vAttr.dataType.isInstanceOf[VariantType], - s"Expected v left as raw VariantType, but got ${vAttr.dataType}") + s"Expected v left raw with no narrowing Project above the join, " + + s"but got ${vAttr.dataType}") + } + } + } + + test(s"V2 test - pull-out flag off leaves aggregate function arg raw ($readerName)") { + withTempPath { dir => + val path = dir.getCanonicalPath + withTable("temp_v1") { + sql(s"create table temp_v1 (v variant, name string) using PARQUET location '$path'") + sql("insert into temp_v1 values " + + "(parse_json('{\"price\": 10}'), 'a'), (parse_json('{\"price\": 20}'), 'b')") + } + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "", + SQLConf.PUSH_VARIANT_INTO_SCAN_PULL_OUT_EXTRACTIONS.key -> "false") { + spark.read.parquet(path).createOrReplaceTempView("T_V2") + // With the pull-out rule disabled, the aggregate function argument is not hoisted and v is + // read raw (pushdown still enabled). Confirms the flag gates the rule. + val query = + "select name, max(variant_get(v, '$.price', 'int')) as mx from T_V2 group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) + val scanRelation = findScanRelation(sql(query).queryExecution.optimizedPlan) + val vAttr = scanRelation.output.find(_.name == "v").get + assert(vAttr.dataType.isInstanceOf[VariantType], + s"Expected v left raw with pull-out disabled, but got ${vAttr.dataType}") } } } @@ -1177,8 +2116,8 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } } - test(s"V2 test - join on variant_get key through an aliasing projection: no crash, no " + - s"mis-shred ($readerName)") { + test(s"V2 test - join on variant_get key through an aliasing projection: hoisted and shredded " + + s"($readerName)") { withTempPath { dir => val itemsPath = dir.getCanonicalPath + "/items" val countriesPath = dir.getCanonicalPath + "/countries" @@ -1196,9 +2135,9 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant spark.read.parquet(itemsPath).createOrReplaceTempView("ITEMS_V2") spark.read.parquet(countriesPath).createOrReplaceTempView("COUNTRIES_V2") // The variant column is aliased (v AS vw) in a subquery before the join condition reads - // it via variant_get. Whether the optimizer inlines the alias or keeps it, the query must - // optimize without crashing, return correct results, and not over-shred (v2 pruned, the - // variant join key not turned into a struct that yields wrong data). + // it via variant_get. The optimizer inlines the alias, PullOutVariantExtractions hoists the + // extraction onto the items side, and the join key shreds to the $.country_code slot with + // correct results (checkAnswer) and no wrong data. v2 is pruned. val query = "select c.country_name from " + "(select v as vw, name as nm from ITEMS_V2) i join COUNTRIES_V2 c " + @@ -1217,10 +2156,14 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant // v2 is unreferenced and must be pruned. assert(!itemsScan.output.exists(_.name == "v2"), s"Expected v2 pruned but items scan was ${itemsScan.output.map(_.name)}") - // The variant join key must not be shredded: it is read as a raw variant. - val vAttr = itemsScan.output.find(a => a.name == "v" || a.name == "vw").get - assert(vAttr.dataType.isInstanceOf[VariantType], - s"Expected the variant join key left raw, but got ${vAttr.dataType}") + // The variant join key is hoisted and shredded to the $.country_code slot, no full-variant. + val keyAttr = itemsScan.output.find(a => a.name == "v" || a.name == "vw").get + val vStruct = keyAttr.dataType match { + case s: StructType => s + case other => fail(s"Expected the variant join key shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot, but got $vStruct") } } } @@ -1692,6 +2635,36 @@ abstract class PushVariantIntoScanV2SuiteBase extends QueryTest with PushVariant } } + test(s"Spark native managed table: aggregate hoist shreds without full-variant slot " + + s"($readerName)") { + // A Spark native (managed catalog) Parquet table created with CREATE TABLE ... USING PARQUET + // resolves to a V1 `LogicalRelation` regardless of USE_V1_SOURCE_LIST -- that config only + // affects the `spark.read`/DataSource resolution of external file reads, not managed catalog + // tables. PullOutVariantExtractions must still hoist the aggregate-function extraction just + // like it does for external-file reads, and the pushdown must shred v to the $.price slot + // with no full-variant slot. `scanRelationOutput` handles whichever scan node the plan uses. + withTable("T_native") { + sql("create table T_native (v variant, name string) using parquet") + sql("insert into T_native values " + + "(parse_json('{\"price\": 10}'), 'a'), " + + "(parse_json('{\"price\": 30}'), 'a'), " + + "(parse_json('{\"price\": 20}'), 'b')") + val query = + "select name, max(variant_get(v, '$.price', 'int')) as mx from T_native group by name" + val expectedRows = withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "false") { + sql(query).collect() + } + checkAnswer(sql(query), expectedRows) // a -> 30, b -> 20 + val output = scanRelationOutput(sql(query).queryExecution.optimizedPlan) + val vStruct = output.find(_.name == "v").get.dataType match { + case s: StructType => s + case other => fail(s"Expected v shredded to struct, but got $other") + } + assert(!vStruct.fields.exists(_.dataType.isInstanceOf[VariantType]), + s"Expected no full-variant slot for a native managed table, but got $vStruct") + } + } + test(s"V2 No push down for JSON ($readerName)") { withTempPath { dir => val path = dir.getCanonicalPath diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala index 0301c1d0f5baa..4af5a32515349 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala @@ -1026,7 +1026,7 @@ class DataSourceV2StrategySuite extends SharedSparkSession { val expr = Abs(Literal(-5), failOnError = true) checkV2Conversion(expr, LiteralValue(5, IntegerType)) - withSQLConf(SQLConf.DATA_SOURCE_V2_EXPR_FOLDING.key -> "false") { + withSQLConf("spark.sql.optimizer.datasourceV2ExprFolding" -> "false") { // when spark.sql.optimizer.datasourceV2ExprFolding = false // expression will be converted to V2 expressions, but not folded checkV2Conversion(expr, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala index 94d860174cece..f29d91ba7bdf4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path import org.apache.logging.log4j.Level import org.apache.spark.{SPARK_DOC_ROOT, SparkIllegalArgumentException, SparkNoSuchElementException} +import org.apache.spark.internal.config.{ConfigBuilder, ConfigEntry} import org.apache.spark.network.util.ByteUnit import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.catalyst.parser.ParseException @@ -183,6 +184,88 @@ class SQLConfSuite extends SharedSparkSession { } } + test("proto-backed config - OPTIMIZER_MAX_ITERATIONS") { + // Verify that the config is loaded from the prototext file with correct properties + val config = SQLConf.OPTIMIZER_MAX_ITERATIONS + assert(config.key === "spark.sql.optimizer.maxIterations") + assert(config.defaultValueString === "100") + assert(config.doc === "The max number of iterations the optimizer runs.") + assert(config.version === "2.0.0") + // This config is internal (not public) + assert(!config.isPublic) + + // Verify the config works correctly at runtime + sqlConf.clear() + assert(sqlConf.getConf(config) === 100) + sql(s"set ${config.key}=50") + assert(sqlConf.getConf(config) === 50) + sqlConf.clear() + } + + test("proto-backed config cannot be replaced by an ordinary config entry") { + val key = SQLConf.OPTIMIZER_MAX_ITERATIONS.key + val original = ConfigEntry.findEntry(key) + + intercept[IllegalArgumentException] { + ConfigBuilder(key).intConf.createWithDefault(1) + } + assert(ConfigEntry.findEntry(key) eq original) + } + + test("proto-backed config with checkValue - SHUFFLE_HASH_JOIN_FACTOR") { + val config = SQLConf.SHUFFLE_HASH_JOIN_FACTOR + assert(config.key === "spark.sql.shuffledHashJoinFactor") + assert(config.defaultValueString === "3") + assert(config.version === "3.3.0") + assert(config.isPublic) + + // Verify the config works correctly at runtime + sqlConf.clear() + assert(sqlConf.getConf(config) === 3) + sql(s"set ${config.key}=5") + assert(sqlConf.getConf(config) === 5) + + // Verify the checkValue validation works - value must be >= 1 + checkError( + exception = intercept[SparkIllegalArgumentException] { + spark.conf.set(config.key, 0) + }, + condition = "INVALID_CONF_VALUE.REQUIREMENT", + parameters = Map( + "confName" -> config.key, + "confValue" -> "0", + "confRequirement" -> "The shuffle hash join factor must be at least 1.")) + + sqlConf.clear() + } + + test("getConfByKeyStrict - read proto-backed config by key") { + sqlConf.clear() + + // Test reading default value + assert(sqlConf.getConfByKeyStrict[Int]("spark.sql.optimizer.maxIterations") === 100) + assert(sqlConf.getConfByKeyStrict[Boolean]( + "spark.sql.optimizer.datasourceV2ExprFolding") === true) + + // Test reading configured value + sql("set spark.sql.optimizer.maxIterations=50") + assert(sqlConf.getConfByKeyStrict[Int]("spark.sql.optimizer.maxIterations") === 50) + + // Test that non-existent key throws error + val e = intercept[Exception] { + sqlConf.getConfByKeyStrict[Int]("spark.nonexistent.config") + } + assert(e.getMessage.contains("not found in ConfigRegistry")) + + // Test that type mismatch throws ClassCastException + // spark.sql.optimizer.maxIterations is an INT config, reading as Boolean should fail + intercept[ClassCastException] { + sqlConf.getConfByKeyStrict[Boolean]("spark.sql.optimizer.maxIterations") + } + + sqlConf.clear() + } + test("reset - user-defined conf") { sqlConf.clear() val userDefinedConf = "x.y.z.reset" diff --git a/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions b/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions index 75a905223e67a..37598f184c021 100644 --- a/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions +++ b/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions @@ -786,7 +786,6 @@ spark.sql.optimizer.avoidCollapseUDFWithExpensiveExpr spark.sql.optimizer.avoidDoubleFilterEval spark.sql.optimizer.canChangeCachedPlanOutputPartitioning spark.sql.optimizer.collapseProjectAlwaysInline -spark.sql.optimizer.datasourceV2ExprFolding spark.sql.optimizer.datasourceV2JoinPushdown spark.sql.optimizer.decorrelateExistsSubqueryLegacyIncorrectCountHandling.enabled spark.sql.optimizer.decorrelateInnerQuery.enabled @@ -811,7 +810,6 @@ spark.sql.optimizer.expression.nestedPruning.enabled spark.sql.optimizer.expressionProjectionCandidateLimit spark.sql.optimizer.inSetConversionThreshold spark.sql.optimizer.inSetSwitchThreshold -spark.sql.optimizer.maxIterations spark.sql.optimizer.metadataOnly spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources spark.sql.optimizer.nestedSchemaPruning.enabled @@ -947,7 +945,6 @@ spark.sql.shuffle.partitions spark.sql.shuffleDependency.fileCleanup.enabled spark.sql.shuffleDependency.skipMigration.enabled spark.sql.shuffleExchange.maxThreadThreshold -spark.sql.shuffledHashJoinFactor spark.sql.sort.enableRadixSort spark.sql.sortMergeJoinExec.buffer.in.memory.threshold spark.sql.sortMergeJoinExec.buffer.spill.size.threshold @@ -1103,7 +1100,6 @@ spark.sql.transposeMaxValues spark.sql.truncateTable.ignorePermissionAcl.enabled spark.sql.tvf.allowMultipleTableArguments.enabled spark.sql.ui.explainMode -spark.sql.ui.retainedExecutions spark.sql.unionOutputPartitioning spark.sql.useCommonExprIdForAlias spark.sql.variable.substitute diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala index 157d2d5788b6d..e41f223322bcb 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala @@ -21,6 +21,7 @@ import org.apache.spark.SparkException import org.apache.spark.sql.{functions => F} import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.{caseInsensitiveResolution, caseSensitiveResolution} import org.apache.spark.sql.catalyst.expressions.{CreateMap, If, Literal, RaiseError} import org.apache.spark.sql.catalyst.util.QuotingUtils import org.apache.spark.sql.classic.{DataFrame, ExpressionUtils} @@ -922,28 +923,12 @@ case class Scd2BatchProcessor( * the eligible user-data columns (those not in [[ChangeArgs.keys]] or the framework * reserved set) filtered through [[ChangeArgs.trackHistorySelection]]. */ - private def computeTrackedHistoryColumns(df: DataFrame): Seq[String] = { - val conf = df.sparkSession.sessionState.conf - val resolver = conf.resolver - - val keyColNames = changeArgs.keys.map(_.name) - val reservedColNames = Scd2BatchProcessor.reservedFrameworkColNames - - val eligibleSchema = StructType(df.schema.fields.filterNot { field => - reservedColNames.exists(resolver(_, field.name)) || - keyColNames.exists(resolver(_, field.name)) - }) - - ColumnSelection - .applyToSchema( - schemaName = "trackHistorySelection", - schema = eligibleSchema, - columnSelection = changeArgs.trackHistorySelection, - caseSensitive = conf.caseSensitiveAnalysis - ) - .fieldNames - .toImmutableArraySeq - } + private def computeTrackedHistoryColumns(df: DataFrame): Seq[String] = + Scd2BatchProcessor.computeTrackedHistoryColumns( + schema = df.schema, + changeArgs = changeArgs, + caseSensitive = df.sparkSession.sessionState.conf.caseSensitiveAnalysis + ) /** * Tag each post-reconciliation row with [[Scd2BatchProcessor.shouldRouteToAuxTableColName]]: @@ -1396,20 +1381,58 @@ object Scd2BatchProcessor { private[pipelines] val endAtColName: String = "__END_AT" /** - * Column names reserved by AutoCDC that will be projected onto the microbatch and - * eventually persisted in the target table. If the user's source dataframe contains any of - * these columns, SCD2 reconciliation will fail. + * Column names reserved by AutoCDC that are projected onto the microbatch and persisted in the + * target table. A source dataframe must not contain any of them. * - * TODO(SPARK-57251): validate at [[AutoCdcMergeFlow]] construction time that the source - * schema and column selection do not collide with these reserved names, so we fail fast - * with a user-actionable error instead of silently overwriting them at preprocess time. + * [[startAtColName]] and [[endAtColName]] do NOT carry the reserved + * [[AutoCdcReservedNames.prefix]], so a source-column collision with them is not caught by the + * prefix-based guard; [[org.apache.spark.sql.pipelines.graph.AutoCdcMergeFlow]] validates the + * source schema against the non-prefixed names in this set at construction time, failing fast + * instead of silently overwriting them at preprocess time. */ - private val reservedFrameworkColNames: Set[String] = Set( + private[pipelines] val reservedFrameworkColNames: Set[String] = Set( startAtColName, endAtColName, AutoCdcReservedNames.cdcMetadataColName ) + /** + * Resolve [[ChangeArgs.trackHistorySelection]] against `schema` and return the field names of + * the history-tracking columns: the eligible user-data columns (those that are neither + * [[ChangeArgs.keys]] nor framework reserved columns) filtered through the selection. + * + * This is the single source of truth for which columns define an SCD2 run. It is called both + * per-microbatch (against the reconciled dataframe's schema) and at + * [[org.apache.spark.sql.pipelines.graph.AutoCdcMergeFlow]] construction time (against the + * user-selected source schema), so an unresolvable or ineligible selection fails fast with a + * user-actionable [[org.apache.spark.sql.AnalysisException]] instead of surfacing mid-stream. + * + * Throws `AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA` if the selection references a column that is not + * an eligible history-tracking column in `schema` (i.e. absent, or a key/framework column). + */ + private[pipelines] def computeTrackedHistoryColumns( + schema: StructType, + changeArgs: ChangeArgs, + caseSensitive: Boolean): Seq[String] = { + val resolver = if (caseSensitive) caseSensitiveResolution else caseInsensitiveResolution + val keyColNames = changeArgs.keys.map(_.name) + + val eligibleSchema = StructType(schema.fields.filterNot { field => + reservedFrameworkColNames.exists(resolver(_, field.name)) || + keyColNames.exists(resolver(_, field.name)) + }) + + ColumnSelection + .applyToSchema( + schemaName = "trackHistorySelection", + schema = eligibleSchema, + columnSelection = changeArgs.trackHistorySelection, + caseSensitive = caseSensitive + ) + .fieldNames + .toImmutableArraySeq + } + /** * Name of temporary column projected onto microbatch to compute the min sequencing value per * key within the microbatch. diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DataflowGraphTransformer.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DataflowGraphTransformer.scala index 2523c0ae5502a..f52d5572054d2 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DataflowGraphTransformer.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DataflowGraphTransformer.scala @@ -22,10 +22,10 @@ import java.util.concurrent.{ ConcurrentLinkedDeque, ConcurrentLinkedQueue, ExecutionException, + ExecutorCompletionService, Future } -import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ import scala.util.control.NoStackTrace @@ -134,159 +134,168 @@ class DataflowGraphTransformer(graph: DataflowGraph) extends AutoCloseable { val failedFlowsQueue = new ConcurrentLinkedQueue[ResolutionFailedFlow]() val failedDependentFlows = new ConcurrentHashMap[TableIdentifier, Seq[ResolutionFailedFlow]]() - var futures = ArrayBuffer[Future[Unit]]() + val completionService = new ExecutorCompletionService[Unit](executor) + var outstanding = 0 val toBeResolvedFlows = new ConcurrentLinkedDeque[Flow]() toBeResolvedFlows.addAll(flows.asJava) - while (futures.nonEmpty || toBeResolvedFlows.peekFirst() != null) { - val (done, notDone) = futures.partition(_.isDone) - // Explicitly call future.get() to propagate exceptions one by one if any + // Waits on a finished resolution task and propagates its exception, if any. + def reap(finished: Future[Unit]): Unit = { try { - done.foreach(_.get()) + finished.get() } catch { case exn: ExecutionException => // Computation threw the exception that is the cause of exn throw exn.getCause } - futures = notDone - val flowOpt = { - // We only schedule [[batchSize]] number of flows in parallel. - if (futures.size < batchSize) { - Option(toBeResolvedFlows.pollFirst()) - } else { - None - } + outstanding -= 1 + } + + while (outstanding > 0 || toBeResolvedFlows.peekFirst() != null) { + // Reap every resolution task that has already finished, without blocking. + var finished = completionService.poll() + while (finished != null) { + reap(finished) + finished = completionService.poll() } - flowOpt.foreach { flow => - futures.append( - executor.submit( - () => + // We only schedule [[batchSize]] number of flows in parallel. + if (outstanding < batchSize && toBeResolvedFlows.peekFirst() != null) { + val flow = toBeResolvedFlows.pollFirst() + outstanding += 1 + completionService.submit( + () => + try { try { - try { - // Note: Flow don't need their inputs passed, so for now we send empty Seq. - val result = transformer(flow, Seq.empty) - require( - result.forall(_.isInstanceOf[ResolvedFlow]), - "transformer must return a Seq[Flow]" - ) + // Note: Flow don't need their inputs passed, so for now we send empty Seq. + val result = transformer(flow, Seq.empty) + require( + result.forall(_.isInstanceOf[ResolvedFlow]), + "transformer must return a Seq[Flow]" + ) - val transformedFlows = result.map(_.asInstanceOf[ResolvedFlow]) - resolvedFlowsMap.put(flow.identifier, transformedFlows) - resolvedFlows.addAll(transformedFlows.asJava) - } catch { - case e: TransformNodeRetryableException => - val datasetIdentifier = e.datasetIdentifier - failedDependentFlows.compute( - datasetIdentifier, - (_, flows) => { - // Don't add the input flow back but the failed flow object - // back which has relevant failure information. - val failedFlow = e.failedNode - if (flows == null) { - Seq(failedFlow) - } else { - flows :+ failedFlow - } + val transformedFlows = result.map(_.asInstanceOf[ResolvedFlow]) + resolvedFlowsMap.put(flow.identifier, transformedFlows) + resolvedFlows.addAll(transformedFlows.asJava) + } catch { + case e: TransformNodeRetryableException => + val datasetIdentifier = e.datasetIdentifier + failedDependentFlows.compute( + datasetIdentifier, + (_, flows) => { + // Don't add the input flow back but the failed flow object + // back which has relevant failure information. + val failedFlow = e.failedNode + if (flows == null) { + Seq(failedFlow) + } else { + flows :+ failedFlow } - ) - // Between the time the flow started and finished resolving, perhaps the - // dependent dataset was resolved - resolvedFlowDestinationsMap.computeIfPresent( - datasetIdentifier, - (_, resolved) => { - if (resolved) { - // Check if the dataset that the flow is dependent on has been resolved - // and if so, remove all dependent flows from the failedDependentFlows and - // add them to the toBeResolvedFlows queue for retry. - failedDependentFlows.computeIfPresent( - datasetIdentifier, - (_, toRetryFlows) => { - toRetryFlows.foreach(toBeResolvedFlows.addFirst(_)) - null - } - ) - } - resolved + } + ) + // Between the time the flow started and finished resolving, perhaps the + // dependent dataset was resolved + resolvedFlowDestinationsMap.computeIfPresent( + datasetIdentifier, + (_, resolved) => { + if (resolved) { + // Check if the dataset that the flow is dependent on has been resolved + // and if so, remove all dependent flows from the failedDependentFlows and + // add them to the toBeResolvedFlows queue for retry. + failedDependentFlows.computeIfPresent( + datasetIdentifier, + (_, toRetryFlows) => { + toRetryFlows.foreach(toBeResolvedFlows.addFirst(_)) + null + } + ) } + resolved + } + ) + case other: Throwable => throw other + } + // If all flows to this particular destination are resolved, move to the destination + // node transformer + if (flowsTo(flow.destinationIdentifier).forall({ flowToDestination => + resolvedFlowsMap.containsKey(flowToDestination.identifier) + })) { + // If multiple flows completed in parallel, ensure we resolve the destination only + // once by electing a leader via computeIfAbsent + var isCurrentThreadLeader = false + resolvedFlowDestinationsMap.computeIfAbsent(flow.destinationIdentifier, _ => { + isCurrentThreadLeader = true + // Set initial value as false as flow destination is not resolved yet. + false + }) + if (isCurrentThreadLeader) { + if (tableMap.contains(flow.destinationIdentifier)) { + val transformed = + transformer( + tableMap(flow.destinationIdentifier), + flowsTo(flow.destinationIdentifier) + ) + resolvedTables.addAll( + transformed.collect { case t: Table => t }.asJava ) - case other: Throwable => throw other - } - // If all flows to this particular destination are resolved, move to the destination - // node transformer - if (flowsTo(flow.destinationIdentifier).forall({ flowToDestination => - resolvedFlowsMap.containsKey(flowToDestination.identifier) - })) { - // If multiple flows completed in parallel, ensure we resolve the destination only - // once by electing a leader via computeIfAbsent - var isCurrentThreadLeader = false - resolvedFlowDestinationsMap.computeIfAbsent(flow.destinationIdentifier, _ => { - isCurrentThreadLeader = true - // Set initial value as false as flow destination is not resolved yet. - false - }) - if (isCurrentThreadLeader) { - if (tableMap.contains(flow.destinationIdentifier)) { + resolvedFlows.addAll( + transformed.collect { case f: ResolvedFlow => f }.asJava + ) + } else if (viewMap.contains(flow.destinationIdentifier)) { + resolvedViews.addAll { val transformed = transformer( - tableMap(flow.destinationIdentifier), + viewMap(flow.destinationIdentifier), flowsTo(flow.destinationIdentifier) ) - resolvedTables.addAll( - transformed.collect { case t: Table => t }.asJava - ) - resolvedFlows.addAll( - transformed.collect { case f: ResolvedFlow => f }.asJava - ) - } else if (viewMap.contains(flow.destinationIdentifier)) { - resolvedViews.addAll { - val transformed = - transformer( - viewMap(flow.destinationIdentifier), - flowsTo(flow.destinationIdentifier) - ) - transformed.map(_.asInstanceOf[View]).asJava - } - } else if (sinkMap.contains(flow.destinationIdentifier)) { - resolvedSinks.addAll { - val transformed = - transformer( - sinkMap(flow.destinationIdentifier), flowsTo(flow.destinationIdentifier) - ) - require( - transformed.forall(_.isInstanceOf[Sink]), - "transformer must return a Seq[Sink]" + transformed.map(_.asInstanceOf[View]).asJava + } + } else if (sinkMap.contains(flow.destinationIdentifier)) { + resolvedSinks.addAll { + val transformed = + transformer( + sinkMap(flow.destinationIdentifier), flowsTo(flow.destinationIdentifier) ) - transformed.map(_.asInstanceOf[Sink]).asJava - } - } else { - throw new IllegalArgumentException( - s"Unsupported destination ${flow.destinationIdentifier.unquotedString}" + - s" in flow: ${flow.displayName} at transformDownNodes" + require( + transformed.forall(_.isInstanceOf[Sink]), + "transformer must return a Seq[Sink]" ) + transformed.map(_.asInstanceOf[Sink]).asJava } - // Set flow destination as resolved now. - resolvedFlowDestinationsMap.computeIfPresent( - flow.destinationIdentifier, - (_, _) => { - // If there are any other node failures dependent on this destination, retry - // them - failedDependentFlows.computeIfPresent( - flow.destinationIdentifier, - (_, toRetryFlows) => { - toRetryFlows.foreach(toBeResolvedFlows.addFirst(_)) - null - } - ) - true - } + } else { + throw new IllegalArgumentException( + s"Unsupported destination ${flow.destinationIdentifier.unquotedString}" + + s" in flow: ${flow.displayName} at transformDownNodes" ) } + // Set flow destination as resolved now. + resolvedFlowDestinationsMap.computeIfPresent( + flow.destinationIdentifier, + (_, _) => { + // If there are any other node failures dependent on this destination, retry + // them + failedDependentFlows.computeIfPresent( + flow.destinationIdentifier, + (_, toRetryFlows) => { + toRetryFlows.foreach(toBeResolvedFlows.addFirst(_)) + null + } + ) + true + } + ) } - } catch { - case ex: TransformNodeFailedException => failedFlowsQueue.add(ex.failedNode) } - ) + } catch { + case ex: TransformNodeFailedException => failedFlowsQueue.add(ex.failedNode) + } ) + } else if (outstanding > 0) { + // Nothing could be scheduled (slots full, or the queue is drained) but tasks are still + // running: block until the next finishes instead of busy-spinning on Future.isDone. The + // outstanding > 0 guard is required, not redundant: the poll() drain above can take + // outstanding to 0 with an empty queue, and then there is nothing to wait for - the loop + // should just exit rather than block forever in take(). + reap(completionService.take()) } } diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala index 31444f0c15f3d..bb996d527cdce 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala @@ -253,6 +253,7 @@ class AutoCdcMergeFlow( val funcResult: FlowFunctionResult ) extends ResolvedFlow { requireReservedPrefixAbsentInSourceColumns() + requireReservedFrameworkColumnsAbsentInSourceColumns() def changeArgs: ChangeArgs = flow.changeArgs @@ -267,6 +268,9 @@ class AutoCdcMergeFlow( // AutoCDC flows require all key columns to be present in the user-selected source schema, // so that they survive into the target table where SCD reconciliation needs them. requireKeysPresentInSelectedSchema(selectedSchema) + // SCD2 flows may specify history-tracking columns; validate they resolve to eligible columns + // of the selected schema at construction time, rather than failing mid-stream on first batch. + requireTrackHistoryColumnsResolvableInSelectedSchema(selectedSchema) selectedSchema } @@ -397,6 +401,45 @@ class AutoCdcMergeFlow( } } + /** + * Reject a source column that collides with an SCD2 reserved framework column not covered by + * [[requireReservedPrefixAbsentInSourceColumns]]: the prefix guard only rejects + * [[AutoCdcReservedNames.prefix]] names, but SCD2 also persists the non-prefixed + * [[Scd2BatchProcessor.startAtColName]] and [[Scd2BatchProcessor.endAtColName]]. Runs before + * [[schema]] is forced so the collision fails fast rather than being silently overwritten + * during preprocessing. No-op for SCD1, which has no such columns. + */ + private def requireReservedFrameworkColumnsAbsentInSourceColumns(): Unit = { + val resolver = spark.sessionState.conf.resolver + val reservedPrefix = AutoCdcReservedNames.prefix + + // Only the non-prefixed reserved names need checking here; prefixed ones are already rejected + // by requireReservedPrefixAbsentInSourceColumns. + val reservedNames: Set[String] = changeArgs.storedAsScdType match { + case ScdType.Type2 => + Scd2BatchProcessor.reservedFrameworkColNames.filterNot(_.startsWith(reservedPrefix)) + case ScdType.Type1 => + Set.empty + } + + df.schema.fieldNames + .find(name => reservedNames.exists(resolver(_, name))) + .foreach { conflictingColumnName => + throw new AnalysisException( + errorClass = "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT", + messageParameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.of( + spark.sessionState.conf.caseSensitiveAnalysis + ), + "columnName" -> conflictingColumnName, + "schemaName" -> "changeDataFeed", + "scdType" -> changeArgs.storedAsScdType.label, + "reservedColumnNames" -> reservedNames.toSeq.sorted.mkString(", ") + ) + ) + } + } + /** * Validate all keys specified in changeArgs are actually present in the user-selected schema. */ @@ -417,4 +460,26 @@ class AutoCdcMergeFlow( ) } } + + /** + * Validate that this flow's [[ChangeArgs.trackHistorySelection]] (SCD2 `TRACK HISTORY ON ...`) + * resolves against the user-selected source schema at construction time. Without this, an + * unresolvable or ineligible (key/framework) tracking column would only surface when the first + * microbatch runs reconciliation, deep inside the SCD2 batch processor. + * + * Delegates to [[Scd2BatchProcessor.computeTrackedHistoryColumns]] -- the same resolution used at + * runtime -- so the two can never diverge; it throws `AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA` on an + * unresolvable selection. `trackHistorySelection` is `None` for SCD1 (enforced by [[ChangeArgs]]) + * and for SCD2 flows that do not restrict tracking, in which case resolution is a no-op. + */ + private def requireTrackHistoryColumnsResolvableInSelectedSchema( + selectedSchema: StructType): Unit = { + if (changeArgs.trackHistorySelection.isDefined) { + Scd2BatchProcessor.computeTrackedHistoryColumns( + schema = selectedSchema, + changeArgs = changeArgs, + caseSensitive = spark.sessionState.conf.caseSensitiveAnalysis + ) + } + } } diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala index 02871e64aa761..22b64798f598f 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala @@ -418,21 +418,9 @@ class TriggeredGraphExecution( return RunCompletion() } - val executionFailureOpt = failureTracker.iterator - .map { - case (flowIdentifier, failureInfo) => - ( - graphForExecution.flow(flowIdentifier), - failureInfo.lastException, - failureInfo.lastExceptionAction - ) - } - .collectFirst { - case (_, _, GraphExecution.StopFlowExecution(reason)) => - reason.runTerminationReason - } - - executionFailureOpt.getOrElse(UnexpectedRunFailure()) + TriggeredGraphExecution + .chooseRunTerminationReason(failureTracker.iterator) + .getOrElse(UnexpectedRunFailure()) } } @@ -450,6 +438,23 @@ case class TriggeredFailureInfo( object TriggeredGraphExecution { + /** + * Picks the run-termination reason from the flows whose execution was stopped because they + * exhausted their retries. Several flows can stop a run and `failures` comes from an unordered + * map, so the earliest failure is chosen - ties broken by flow name - to keep the reported + * reason stable across otherwise-identical runs. + */ + private[graph] def chooseRunTerminationReason( + failures: Iterator[(TableIdentifier, TriggeredFailureInfo)]): Option[RunTerminationReason] = { + failures + .collect { + case (id, TriggeredFailureInfo(ts, _, _, GraphExecution.StopFlowExecution(r))) => + (ts, id.unquotedString, r.runTerminationReason) + } + .minByOption { case (ts, flowName, _) => (ts, flowName) } + .map { case (_, _, reason) => reason } + } + // All possible states of a data stream for a flow sealed trait StreamState object StreamState { diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala index 98c31a3020ef5..d31b9383354cb 100644 --- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala +++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala @@ -165,13 +165,15 @@ class AutoCdcFlowSuite extends QueryTest with SharedSparkSession { keys: Seq[UnqualifiedColumnName] = Seq(UnqualifiedColumnName("id")), sequencing: Column = F.col("seq"), storedAsScdType: ScdType = ScdType.Type1, - columnSelection: Option[ColumnSelection] = None): AutoCdcMergeFlow = { + columnSelection: Option[ColumnSelection] = None, + trackHistorySelection: Option[ColumnSelection] = None): AutoCdcMergeFlow = { val flow = newAutoCdcFlow( changeArgs = ChangeArgs( keys = keys, sequencing = sequencing, storedAsScdType = storedAsScdType, - columnSelection = columnSelection + columnSelection = columnSelection, + trackHistorySelection = trackHistorySelection ) ) new AutoCdcMergeFlow(flow, successfulFuncResult(sourceDf)) @@ -368,63 +370,42 @@ class AutoCdcFlowSuite extends QueryTest with SharedSparkSession { } } - test("AutoCdcMergeFlow.schema lets a selection exclude a source column that collides with " + - "a non-prefixed framework column, re-adding it from the framework") { - // Two of the three SCD2 framework columns -- __START_AT and __END_AT -- do not carry the - // reserved AutoCDC prefix, so a source change feed may legitimately contain columns with - // those names. The user excludes them via the column selection; they are dropped from the - // user-data portion and the engine's own framework columns are appended in their place, so - // each still appears exactly once in the output with the framework's type (the sequencing - // type), not the source column's type. - // - // The third framework column, _cdc_metadata, DOES carry the reserved prefix, so a source - // that contains it is rejected outright at flow construction -- it can never reach column - // selection and so is deliberately out of scope here. That rejection is covered separately by - // "AutoCdcMergeFlow rejects a source df column whose name equals the reserved CDC metadata - // column". - val session = spark - import session.implicits._ - // Source carries String-typed __START_AT / __END_AT columns alongside the data columns. - val sourceDf = MemoryStream[(Int, String, Option[Long], String, String)] - .toDS() - .toDF( - "id", - "name", - "seq", - Scd2BatchProcessor.startAtColName, - Scd2BatchProcessor.endAtColName) + test("AutoCdcMergeFlow rejects a source column named after a non-prefixed framework column " + + "even when a selection would exclude it") { + // Behavior change introduced by this PR (SPARK-57251): __START_AT / __END_AT do not carry the + // reserved AutoCDC prefix, so before this change a source could legitimately contain columns + // with those names and exclude them via the column selection. The new + // requireReservedFrameworkColumnsAbsentInSourceColumns guard runs against the RAW source + // schema (before column selection is applied), so such a source is now rejected outright at + // flow construction -- an ExcludeColumns selection cannot rescue it. Allowing an explicit + // opt-out (validating post-selection instead) is tracked separately by SPARK-58325. + val sourceDf = sourceDfWithExtraColumns( + Scd2BatchProcessor.startAtColName -> StringType, + Scd2BatchProcessor.endAtColName -> StringType) - val resolvedFlow = newAutoCdcMergeFlow( - sourceDf = sourceDf, - storedAsScdType = ScdType.Type2, - columnSelection = Some( - ColumnSelection.ExcludeColumns( - Seq( - UnqualifiedColumnName(Scd2BatchProcessor.startAtColName), - UnqualifiedColumnName(Scd2BatchProcessor.endAtColName)) + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow( + sourceDf = sourceDf, + storedAsScdType = ScdType.Type2, + columnSelection = Some( + ColumnSelection.ExcludeColumns( + Seq( + UnqualifiedColumnName(Scd2BatchProcessor.startAtColName), + UnqualifiedColumnName(Scd2BatchProcessor.endAtColName)) + ) + ) ) - ) - ) - - // Each framework column appears exactly once (the source copy was excluded, the framework - // copy appended) ... - assert( - resolvedFlow.schema.fieldNames.count(_ == Scd2BatchProcessor.startAtColName) == 1) - assert( - resolvedFlow.schema.fieldNames.count(_ == Scd2BatchProcessor.endAtColName) == 1) - // ... and carries the framework (sequencing) type, not the source String type. - assert(resolvedFlow.schema(Scd2BatchProcessor.startAtColName).dataType == LongType) - assert(resolvedFlow.schema(Scd2BatchProcessor.endAtColName).dataType == LongType) - // Full expected shape: the retained data columns followed by the framework columns. - assert( - resolvedFlow.schema.fieldNames.toSeq == - Seq( - "id", - "name", - "seq", - Scd2BatchProcessor.startAtColName, - Scd2BatchProcessor.endAtColName, - AutoCdcReservedNames.cdcMetadataColName + }, + condition = "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT", + sqlState = "42710", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "columnName" -> Scd2BatchProcessor.startAtColName, + "schemaName" -> "changeDataFeed", + "scdType" -> ScdType.Type2.label, + "reservedColumnNames" -> + Seq(Scd2BatchProcessor.endAtColName, Scd2BatchProcessor.startAtColName).mkString(", ") ) ) } @@ -701,6 +682,122 @@ class AutoCdcFlowSuite extends QueryTest with SharedSparkSession { } } + // =========================================================================================== + // AutoCdcMergeFlow reserved framework-column (non-prefixed) validation tests + // + // SCD2 persists framework columns __START_AT / __END_AT that do NOT carry the reserved + // AutoCDC prefix, so they are not caught by the prefix guard above. These tests lock in that a + // source column colliding with such a name is rejected at construction for SCD2, is allowed + // for SCD1 (which reserves no non-prefixed names), and that the check respects case-sensitivity. + // =========================================================================================== + + /** The SCD2 reserved framework column names that are not covered by the reserved prefix. */ + private val nonPrefixedScd2ReservedNames: Seq[String] = + Scd2BatchProcessor.reservedFrameworkColNames + .filterNot(_.startsWith(AutoCdcReservedNames.prefix)) + .toSeq + .sorted + + test("non-prefixed reserved names exist and are covered by this suite") { + // Guards against a future refactor renaming/removing __START_AT / __END_AT without updating + // the flow-construction validation: if this set ever empties, the tests below silently + // stop exercising anything. + assert( + nonPrefixedScd2ReservedNames == Seq("__END_AT", "__START_AT"), + s"Unexpected non-prefixed SCD2 reserved names: $nonPrefixedScd2ReservedNames" + ) + } + + test( + "an SCD2 flow with a source column colliding with a reserved framework column is rejected " + + "at construction" + ) { + nonPrefixedScd2ReservedNames.foreach { reservedName => + val sourceDf = sourceDfWithExtraColumns(reservedName -> StringType) + + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow(sourceDf, storedAsScdType = ScdType.Type2) + }, + condition = "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT", + sqlState = "42710", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "columnName" -> reservedName, + "schemaName" -> "changeDataFeed", + "scdType" -> ScdType.Type2.label, + "reservedColumnNames" -> nonPrefixedScd2ReservedNames.mkString(", ") + ) + ) + } + } + + test( + "the reserved framework-column check runs before the SCD2-not-supported gate" + ) { + // The reserved-name error is more actionable than AUTOCDC_SCD2_NOT_SUPPORTED, so it must win + // for an SCD2 flow that both is unsupported and carries a colliding source column. This also + // keeps the check meaningful today (before SCD2 is supported) and correct once it lands. + val sourceDf = sourceDfWithExtraColumns(Scd2BatchProcessor.startAtColName -> StringType) + val ex = intercept[AnalysisException] { + newAutoCdcMergeFlow(sourceDf, storedAsScdType = ScdType.Type2) + } + assert(ex.getCondition == "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT") + } + + test( + "an SCD1 flow with a source column matching an SCD2-only reserved name is allowed" + ) { + // SCD1 targets carry no non-prefixed framework columns, so __START_AT / __END_AT are ordinary + // user columns there. Construction succeeds and the column survives into the flow schema. + nonPrefixedScd2ReservedNames.foreach { reservedName => + val sourceDf = sourceDfWithExtraColumns(reservedName -> StringType) + val resolvedFlow = newAutoCdcMergeFlow(sourceDf, storedAsScdType = ScdType.Type1) + assert(resolvedFlow.schema.fieldNames.contains(reservedName)) + } + } + + test( + "an uppercase reserved framework-column name is rejected for SCD2 when caseSensitive=false" + ) { + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { + val conflictingName = Scd2BatchProcessor.startAtColName.toLowerCase(Locale.ROOT) + val sourceDf = sourceDfWithExtraColumns(conflictingName -> StringType) + + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow(sourceDf, storedAsScdType = ScdType.Type2) + }, + condition = "AUTOCDC_RESERVED_COLUMN_NAME_CONFLICT", + sqlState = "42710", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "columnName" -> conflictingName, + "schemaName" -> "changeDataFeed", + "scdType" -> ScdType.Type2.label, + "reservedColumnNames" -> nonPrefixedScd2ReservedNames.mkString(", ") + ) + ) + } + } + + test( + "a differently-cased reserved framework-column name does not trip the reserved check for " + + "SCD2 when caseSensitive=true" + ) { + // Under case-sensitive analysis, a lowercase variant is a distinct identifier and does not + // collide with the reserved (uppercase) framework name, consistent with the prefix guard. + // The reserved-name check therefore does NOT fire and the flow constructs successfully, + // keeping the lowercase column as an ordinary user column in the schema. + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { + val nonConflictingName = Scd2BatchProcessor.startAtColName.toLowerCase(Locale.ROOT) + val sourceDf = sourceDfWithExtraColumns(nonConflictingName -> StringType) + + val resolvedFlow = newAutoCdcMergeFlow(sourceDf, storedAsScdType = ScdType.Type2) + assert(resolvedFlow.schema.fieldNames.contains(nonConflictingName)) + } + } + // =========================================================================================== // AutoCdcMergeFlow keys-presence validation tests (requireKeysPresentInSelectedSchema) // =========================================================================================== @@ -767,4 +864,151 @@ class AutoCdcFlowSuite extends QueryTest with SharedSparkSession { ) ) } + + // =========================================================================================== + // AutoCdcMergeFlow track-history validation tests + // + // SCD2 `TRACK HISTORY ON (...)` populates trackHistorySelection. These tests lock in that an + // unresolvable or ineligible (key / dropped-by-column-selection) tracking column is rejected at + // flow construction rather than deferring to the first microbatch's reconciliation, mirroring + // the keys-presence validator above. A resolvable selection passes the check and the flow + // constructs successfully. + // =========================================================================================== + + test( + "an SCD2 flow tracking a non-existent column is rejected at construction" + ) { + // Eligible tracking columns from the 3-column source (id, name, seq), less the key `id`, are + // {name, seq}. `missing` is absent, so resolution against trackHistorySelection fails. + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("missing"))) + ) + ) + }, + condition = "AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "schemaName" -> "trackHistorySelection", + "missingColumns" -> "missing", + "availableColumns" -> "name, seq" + ) + ) + } + + test( + "an SCD2 flow tracking a key column is rejected at construction (ineligible)" + ) { + // A key is never an eligible history-tracking column, so it is absent from the eligible + // schema {name, seq} and resolution fails -- surfacing the misconfiguration eagerly. + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("id"))) + ) + ) + }, + condition = "AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "schemaName" -> "trackHistorySelection", + "missingColumns" -> "id", + "availableColumns" -> "name, seq" + ) + ) + } + + test( + "an SCD2 flow tracking a column dropped by columnSelection is rejected" + ) { + // `name` exists in the source but is excluded from the selected schema, so it is not an + // eligible tracking column. Eligible columns are then just {seq}. + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + columnSelection = Some( + ColumnSelection.ExcludeColumns(Seq(UnqualifiedColumnName("name"))) + ), + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("name"))) + ) + ) + }, + condition = "AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive, + "schemaName" -> "trackHistorySelection", + "missingColumns" -> "name", + "availableColumns" -> "seq" + ) + ) + } + + test( + "an SCD2 flow with a resolvable track-history selection passes the check" + ) { + // `name` is an eligible tracking column, so the construction-time check passes and the flow + // constructs successfully, resolving its SCD2 schema (no AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA). + val resolvedFlow = newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("name"))) + ) + ) + assert(resolvedFlow.schema.fieldNames.contains(AutoCdcReservedNames.cdcMetadataColName)) + } + + test( + "track-history validation respects case-insensitive analysis" + ) { + // With caseSensitive=false, `NAME` resolves to the eligible `name`, so the check passes and + // the flow constructs successfully. + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { + val resolvedFlow = newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("NAME"))) + ) + ) + assert(resolvedFlow.schema.fieldNames.contains(AutoCdcReservedNames.cdcMetadataColName)) + } + } + + test( + "track-history validation respects case-sensitive analysis" + ) { + // With caseSensitive=true, `NAME` is a distinct identifier from the eligible `name` and does + // not resolve, so the construction-time check rejects it. + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { + checkError( + exception = intercept[AnalysisException] { + newAutoCdcMergeFlow( + sourceDf = threeColumnSourceDf(), + storedAsScdType = ScdType.Type2, + trackHistorySelection = Some( + ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("NAME"))) + ) + ) + }, + condition = "AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA", + parameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.CaseSensitive, + "schemaName" -> "trackHistorySelection", + "missingColumns" -> "NAME", + "availableColumns" -> "name, seq" + ) + ) + } + } } diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/ConnectValidPipelineSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/ConnectValidPipelineSuite.scala index 014e9e1d5a60d..ea6d3202ba868 100644 --- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/ConnectValidPipelineSuite.scala +++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/ConnectValidPipelineSuite.scala @@ -544,6 +544,64 @@ class ConnectValidPipelineSuite extends PipelineTest with SharedSparkSession { ) } + test("resolution terminates and resolves all flows when flow count exceeds parallelism") { + val session = spark + import session.implicits._ + + // DataflowGraphTransformer caps in-flight resolutions at `parallelism` (10). With more + // independent flows than that, the slots fill and the scheduler blocks on a finished task + // (the `take()` branch) once `parallelism` tasks are outstanding - the path the busy-wait + // rewrite changes. Small graphs in the other suites never reach this regime. This asserts the + // outcome only (all flows resolve and the call returns), so it is deterministic and has no + // timing dependence; a regression that deadlocked would hang here until the suite times out. + val numFlows = 25 + class P extends TestGraphRegistrationContext(spark) { + (0 until numFlows).foreach { i => + registerPersistedView(s"v$i", query = dfFlowFunc(Seq(i).toDF("x"))) + } + } + val p = new P().resolveToDataflowGraph() + + assert(p.resolved, "all flows should resolve when their count exceeds parallelism") + (0 until numFlows).foreach { i => + assert( + p.resolvedFlow.contains(fullyQualifiedIdentifier(s"v$i")), + s"flow v$i was not resolved") + } + } + + test("resolution re-queues retryable flows under load when consumers exceed parallelism") { + val session = spark + import session.implicits._ + + // A wide fan-out: many consumers reading from one source view, with the consumer count above + // `parallelism` (10), so slots fill and the loop blocks on take(). The consumers are + // registered (and therefore scheduled) before `src`, so the first batch resolves consumers + // whose `src` input is not yet available: each throws TransformNodeRetryableException and is + // parked as a dependent of `src`; once `src` resolves they are re-queued onto the deque and + // retried, re-driving the loop until every consumer resolves. This deterministically exercises + // the retryable re-queue path together with the blocking branch. Asserts only that everything + // resolves and the call returns - no timing assertions. + val numConsumers = 20 + class P extends TestGraphRegistrationContext(spark) { + (0 until numConsumers).foreach { i => + registerPersistedView(s"c$i", query = sqlFlowFunc(spark, "SELECT x FROM src")) + } + registerPersistedView("src", query = dfFlowFunc(Seq(1, 2, 3).toDF("x"))) + } + val p = new P().resolveToDataflowGraph() + + assert(p.resolved, "source and all consumers should resolve under load") + assert( + p.resolvedFlow.contains(fullyQualifiedIdentifier("src")), + "source flow was not resolved") + (0 until numConsumers).foreach { i => + assert( + p.resolvedFlow.contains(fullyQualifiedIdentifier(s"c$i")), + s"consumer flow c$i was not resolved") + } + } + /** Verifies the [[DataflowGraph]] has the specified [[Flow]] with the specified schema. */ private def verifyFlowSchema( pipeline: DataflowGraph, diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala index 57baf4c2d5b11..db8c368ca89df 100644 --- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala +++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala @@ -1060,4 +1060,53 @@ class TriggeredGraphExecutionSuite extends ExecutionTest with SharedSparkSession assert(warnCount == 2 && errorCount == 1) } + + /** A non-retryable (retries exhausted) failure, i.e. one that stops the run. */ + private def stopFailure(ts: Long, flowName: String, cause: Throwable): TriggeredFailureInfo = { + // currentNumTries > maxAllowedRetries yields a StopFlowExecution. + val action = GraphExecution.determineFlowExecutionActionFromError( + ex = cause, + flowDisplayName = flowName, + currentNumTries = 2, + maxAllowedRetries = 1) + TriggeredFailureInfo(ts, numFailures = 2, lastException = cause, lastExceptionAction = action) + } + + test("chooseRunTerminationReason surfaces the earliest non-retryable failure deterministically") { + val earliestCause = new RuntimeException("earliest") + // A retryable failure with an earlier timestamp must be ignored: it does not stop the run. + val retryable = { + val cause = new RuntimeException("retryable") + val action = GraphExecution.determineFlowExecutionActionFromError( + ex = cause, flowDisplayName = "flow_retry", currentNumTries = 1, maxAllowedRetries = 3) + TableIdentifier("flow_retry") -> TriggeredFailureInfo( + 50, numFailures = 1, lastException = cause, lastExceptionAction = action) + } + val entries = Seq( + TableIdentifier("flow_c") -> stopFailure(300, "flow_c", new RuntimeException("c")), + TableIdentifier("flow_b") -> stopFailure(100, "flow_b", earliestCause), + TableIdentifier("flow_a") -> stopFailure(200, "flow_a", new RuntimeException("a")), + retryable) + + // The earliest non-retryable failure (flow_b @ 100) wins regardless of iteration order. + Seq(entries, entries.reverse).foreach { ordered => + assert( + TriggeredGraphExecution.chooseRunTerminationReason(ordered.iterator) + .contains(QueryExecutionFailure("flow_b", 1, Some(earliestCause)))) + } + } + + test("chooseRunTerminationReason breaks ties between equal timestamps by flow name") { + val aCause = new RuntimeException("a") + val entries = Seq( + TableIdentifier("flow_z") -> stopFailure(100, "flow_z", new RuntimeException("z")), + TableIdentifier("flow_a") -> stopFailure(100, "flow_a", aCause)) + assert( + TriggeredGraphExecution.chooseRunTerminationReason(entries.iterator) + .contains(QueryExecutionFailure("flow_a", 1, Some(aCause)))) + } + + test("chooseRunTerminationReason has no reason when no flow stopped the run") { + assert(TriggeredGraphExecution.chooseRunTerminationReason(Iterator.empty).isEmpty) + } }