byteoak · Tagar · Jul 27, 2026 · Jul 27, 2026 · Jul 27, 2026 · Jul 27, 2026
diff --git a/common/config/README.md b/common/config/README.md
@@ -0,0 +1,122 @@
+# Spark Configuration Module
+
+This module provides a proto-based configuration system for defining Spark configurations. Configurations are defined using Protocol Buffers text format (`.textproto` files) and loaded at runtime by `ConfigRegistry`.
+
+## Directory Structure
+
+```
+common/config/src/main/
+├── java/org/apache/spark/config/
+│   └── ConfigRegistry.java          # Loads and queries configs
+├── protobuf/org/apache/spark/config/
+│   └── config_schema.proto          # Defines the config schema
+└── resources/org/apache/spark/config/
+    ├── cluster_configs/             # CLUSTER scope configs
+    │   └── sql.textproto
+    └── session_configs/             # SESSION scope configs
+        └── sql.textproto
+```
+
+## How to Define a New Config
+
+### Step 1: Choose the Right File
+
+- **CLUSTER scope** (`MUTABILITY_STATIC`): If the config applies to the entire Spark cluster and cannot be changed at runtime, add to a file in `cluster_configs/` directory.
+- **SESSION scope** (`MUTABILITY_DYNAMIC`): If the config applies to a Spark session and can be changed at runtime (e.g., via `SET`), add to a file in `session_configs/` directory.
+
+Currently, SQL configs go in `sql.textproto`. New categories can have their own files.
+
+### Step 2: Add the Config Entry
+
+Add a new `configs` block to the appropriate `.textproto` file:
+
+```protobuf
+configs {
+  key: "spark.sql.myFeature.enabled"
+  value_type: VALUE_TYPE_BOOL
+  default_value: "true"
+  scope: SCOPE_SESSION
+  mutability: MUTABILITY_DYNAMIC
+  visibility: VISIBILITY_PUBLIC
+  binding_policy: BINDING_POLICY_SESSION
+  doc: "When true, enables my new feature."
+  version: "4.0.0"
+}
+```
+
+**Notes**:
+- See `config_schema.proto` for field details and requirements
+- Configs within each file must be ordered alphabetically by key
+- A config that sets `test_default` must also set `default_value` (a test-only override needs a
+  production default to override); this is validated at load time
+- These constraints are validated by tests
+
+### Step 3: Register the Config File (if new)
+
+If you created a new `.textproto` file, add it to `ConfigRegistry.java`:
+
+```java
+private static final String[] DEFAULT_CONFIG_FILES = {
+  "org/apache/spark/config/cluster_configs/sql.textproto",
+  "org/apache/spark/config/session_configs/sql.textproto",
+  "org/apache/spark/config/session_configs/your_new_file.textproto"  // Add here
+};
+```
+
+## Multi-line Documentation
+
+For long documentation strings, use protobuf string concatenation:
+
+```protobuf
+configs {
+  key: "spark.sql.myFeature.threshold"
+  value_type: VALUE_TYPE_INT
+  default_value: "100"
+  scope: SCOPE_SESSION
+  mutability: MUTABILITY_DYNAMIC
+  visibility: VISIBILITY_PUBLIC
+  binding_policy: BINDING_POLICY_NOT_APPLICABLE
+  doc: "This is a long documentation string that spans multiple lines. "
+       "Simply place multiple quoted strings adjacent to each other "
+       "and protobuf will concatenate them automatically."
+  version: "4.0.0"
+}
+```
+
+## Using Configs in Spark
+
+### Reading Config Values
+
+For configs accessed in only one or a few places, use `getConfByKeyStrict` with the config key directly. This will fail at runtime if the key is not defined in a `.textproto` file:
+
+```scala
+def myFunc(conf: SQLConf): Unit = {
+  if (conf.getConfByKeyStrict[Boolean]("spark.sql.myFeature.enabled")) {
+    doSomething()
+  }
+}
+```
+
+For configs accessed in many places, create a `ConfigEntry` in `object SQLConf` (or its friends) to avoid hardcoding keys:
+
+```scala
+object SQLConf {
+  val MY_FEATURE_ENABLED = buildConfFromConfigFile[Boolean]("spark.sql.myFeature.enabled")
+}
+
+def myFunc(conf: SQLConf): Unit = {
+  if (conf.getConf(SQLConf.MY_FEATURE_ENABLED)) {
+    doSomething()
+  }
+}
+```
+
+### Adding Config Value Validation
+
+For configs that need to validate their values, create a `ConfigEntry` and use `checkValue`:
+
+```scala
+val MY_THRESHOLD = buildConfFromConfigFile[Int]("spark.sql.myFeature.threshold")
+  .checkValue(_ > 0, "Threshold must be positive")
+```
+
diff --git a/common/config/pom.xml b/common/config/pom.xml
@@ -0,0 +1,187 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.spark</groupId>
+    <artifactId>spark-parent_2.13</artifactId>
+    <version>5.0.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>spark-config_2.13</artifactId>
+  <packaging>jar</packaging>
+  <name>Spark Project Config</name>
+  <url>https://spark.apache.org/</url>
+  <properties>
+    <sbt.project.name>config</sbt.project.name>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-tags_${scala.binary.version}</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <!--
+      This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude
+      them will yield errors.
+    -->
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-tags_${scala.binary.version}</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
+    <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <!--
+          Override the parent pom's shade configuration completely (rather than
+          merging) so this module bundles protobuf-java relocated into a shared
+          org.sparkproject.protobuf package. This keeps the unshaded protobuf-java
+          off the classpath of every consumer of this low-level module (notably
+          spark-common-utils, and transitively spark-core and the assembly), and
+          removes protobuf-java from this module's published (dependency-reduced)
+          pom. The relocation package is intentionally general (not config-specific)
+          so other low-level modules can share it in the future.
+        -->
+        <configuration combine.self="override">
+          <shadedArtifactAttached>false</shadedArtifactAttached>
+          <shadeTestJar>false</shadeTestJar>
+          <artifactSet>
+            <includes>
+              <include>com.google.protobuf:*</include>
+            </includes>
+          </artifactSet>
+          <relocations>
+            <relocation>
+              <pattern>com.google.protobuf</pattern>
+              <shadedPattern>${spark.shade.packageName}.protobuf</shadedPattern>
+              <includes>
+                <include>com.google.protobuf.**</include>
+              </includes>
+            </relocation>
+          </relocations>
+          <filters>
+            <filter>
+              <artifact>*:*</artifact>
+              <excludes>
+                <exclude>google/protobuf/**</exclude>
+              </excludes>
+            </filter>
+          </filters>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+  <profiles>
+    <profile>
+      <id>default-protoc</id>
+      <activation>
+        <property>
+          <name>!skipDefaultProtoc</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>com.github.os72</groupId>
+            <artifactId>protoc-jar-maven-plugin</artifactId>
+            <version>${protoc-jar-maven-plugin.version}</version>
+            <executions>
+              <execution>
+                <phase>generate-sources</phase>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+                <configuration>
+                  <protocArtifact>com.google.protobuf:protoc:${protobuf.version}</protocArtifact>
+                  <protocVersion>${protobuf.version}</protocVersion>
+                  <inputDirectories>
+                    <include>src/main/protobuf</include>
+                  </inputDirectories>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>user-defined-protoc</id>
+      <properties>
+        <spark.protoc.executable.path>${env.SPARK_PROTOC_EXEC_PATH}</spark.protoc.executable.path>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>com.github.os72</groupId>
+            <artifactId>protoc-jar-maven-plugin</artifactId>
+            <version>${protoc-jar-maven-plugin.version}</version>
+            <executions>
+              <execution>
+                <phase>generate-sources</phase>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+                <configuration>
+                  <protocArtifact>com.google.protobuf:protoc:${protobuf.version}</protocArtifact>
+                  <protocVersion>${protobuf.version}</protocVersion>
+                  <protocCommand>${spark.protoc.executable.path}</protocCommand>
+                  <inputDirectories>
+                    <include>src/main/protobuf</include>
+                  </inputDirectories>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+</project>