From 3c6c09ffa2bbea258f004c54da77e14900720cfb Mon Sep 17 00:00:00 2001 From: Nikita Ryanov Date: Tue, 11 Aug 2026 05:09:53 +0300 Subject: [PATCH] [KYUUBI #3356][SERVER] Add experimental Arrow Flight SQL frontend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Why are the changes needed? Add an experimental Arrow Flight SQL frontend so clients can run SQL and metadata operations through Kyuubi while reusing the existing session/engine lifecycle (Spark, Trino, Hive, JDBC, etc.). The producer is a transport adapter only: it calls BackendService and does not invoke engine APIs directly. Main capabilities: - Enable via `kyuubi.frontend.protocols=FLIGHT_SQL` and configure with `kyuubi.frontend.flight.sql.*` keys (default bind port `10299`). - Statement queries, catalogs/schemas/tables/table-types/type info, Arrow IPC streaming with bounded page fetch, cancellation/cleanup. - Auth: NONE, Basic/LDAP, Kerberos SPNEGO bootstrap with short-lived bearer tokens; TLS via PEM (optional keystore→PEM materialization). - HA: dedicated discovery namespace (`kyuubi.ha.flight.sql.namespace`), advertised Flight endpoint, node-affine tickets (no transparent mid-query failover). - Flight-specific metrics. Out of scope / intentional limits: - Not end-to-end zero-copy past the existing BackendService Thrift/TRowSet Arrow boundary. - Explicit unsupported Flight SQL paths (ingest, Substrait, transactions, full prepared-statement binding) return unimplemented status. ### How was this patch tested? - Unit/integration tests under `kyuubi-server`: - `KyuubiFlightSqlFrontendServiceSuite` (lifecycle / protocol enablement) - `KyuubiFlightSqlQuerySuite` (execute + Arrow stream via Java `FlightSqlClient`; run with Slow tag enabled) - `KyuubiFlightAuthHandlerSuite` - `KyuubiFlightArrowUtilsSuite` - Shared harness: `WithFlightSqlServer` - Config/HA coverage: `KyuubiConfSuite`, `HighAvailabilityConfSuite` - Regenerated `docs/configuration/settings.md` via `dev/gen/gen_all_config_docs.sh` - Spotless formatting on touched modules ### Was this patch authored or co-authored using generative AI tooling? Yes. Assisted-by: Cursor Co-authored-by: Cursor --- docs/configuration/settings.md | 10 +- .../org/apache/kyuubi/config/KyuubiConf.scala | 75 ++- .../kyuubi/config/KyuubiConfSuite.scala | 23 + .../kyuubi/ha/HighAvailabilityConf.scala | 9 + .../ha/client/FlightSqlServiceDiscovery.scala | 37 ++ .../ha/client/HighAvailabilityConfSuite.scala | 7 + .../kyuubi/metrics/MetricsConstants.scala | 16 + kyuubi-server/pom.xml | 11 + .../kyuubi/server/BackendServiceMetric.scala | 49 +- .../KyuubiFlightSqlFrontendService.scala | 163 ++++++ .../apache/kyuubi/server/KyuubiServer.scala | 3 + .../server/flight/FlightResultIterator.scala | 146 ++++++ .../flight/FlightSqlKerberosValidator.scala | 109 ++++ .../flight/KyuubiFlightArrowUtils.scala | 328 ++++++++++++ .../flight/KyuubiFlightAuthHandler.scala | 190 +++++++ .../flight/KyuubiFlightSqlProducer.scala | 495 ++++++++++++++++++ .../server/flight/KyuubiFlightTlsUtils.scala | 146 ++++++ .../apache/kyuubi/WithFlightSqlServer.scala | 33 ++ .../KyuubiFlightSqlFrontendServiceSuite.scala | 86 +++ .../flight/KyuubiFlightArrowUtilsSuite.scala | 141 +++++ .../flight/KyuubiFlightAuthHandlerSuite.scala | 86 +++ .../flight/KyuubiFlightSqlQuerySuite.scala | 60 +++ pom.xml | 10 + 23 files changed, 2216 insertions(+), 17 deletions(-) create mode 100644 kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/FlightSqlServiceDiscovery.scala create mode 100644 kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiFlightSqlFrontendService.scala create mode 100644 kyuubi-server/src/main/scala/org/apache/kyuubi/server/flight/FlightResultIterator.scala create mode 100644 kyuubi-server/src/main/scala/org/apache/kyuubi/server/flight/FlightSqlKerberosValidator.scala create mode 100644 kyuubi-server/src/main/scala/org/apache/kyuubi/server/flight/KyuubiFlightArrowUtils.scala create mode 100644 kyuubi-server/src/main/scala/org/apache/kyuubi/server/flight/KyuubiFlightAuthHandler.scala create mode 100644 kyuubi-server/src/main/scala/org/apache/kyuubi/server/flight/KyuubiFlightSqlProducer.scala create mode 100644 kyuubi-server/src/main/scala/org/apache/kyuubi/server/flight/KyuubiFlightTlsUtils.scala create mode 100644 kyuubi-server/src/test/scala/org/apache/kyuubi/WithFlightSqlServer.scala create mode 100644 kyuubi-server/src/test/scala/org/apache/kyuubi/server/KyuubiFlightSqlFrontendServiceSuite.scala create mode 100644 kyuubi-server/src/test/scala/org/apache/kyuubi/server/flight/KyuubiFlightArrowUtilsSuite.scala create mode 100644 kyuubi-server/src/test/scala/org/apache/kyuubi/server/flight/KyuubiFlightAuthHandlerSuite.scala create mode 100644 kyuubi-server/src/test/scala/org/apache/kyuubi/server/flight/KyuubiFlightSqlQuerySuite.scala diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 020cb0efdd5..4b94b70e28e 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -248,11 +248,18 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.frontend.bind.port | 10009 | (deprecated) Port of the machine on which to run the thrift frontend service via the binary protocol. | int | 1.0.0 | | kyuubi.frontend.connection.url.use.hostname | true | When true, frontend services prefer hostname, otherwise, ip address. Note that, the default value is set to `false` when engine running on Kubernetes to prevent potential network issues. | boolean | 1.5.0 | | kyuubi.frontend.data.agent.operation.timeout | PT2M | Timeout for waiting on data agent engine launch and operation start in the REST frontend. | duration | 1.12.0 | +| kyuubi.frontend.flight.sql.bind.host | <undefined> | Hostname or IP on which to run the Arrow Flight SQL gRPC frontend service. | string | 1.13.0 | +| kyuubi.frontend.flight.sql.bind.port | 10299 | Port on which to run the Arrow Flight SQL gRPC frontend service. | int | 1.13.0 | +| kyuubi.frontend.flight.sql.fetch.max.rows | 1000 | Maximum number of rows requested from Kyuubi for each Arrow Flight SQL result page. | int | 1.13.0 | +| kyuubi.frontend.flight.sql.ssl.cert.file | <undefined> | PEM certificate chain file used by the Arrow Flight SQL frontend when TLS is enabled. | string | 1.13.0 | +| kyuubi.frontend.flight.sql.ssl.enabled | false | Set this to true to enable TLS/SSL encryption on the Arrow Flight SQL gRPC frontend. Arrow Flight 16 requires PEM certificate and private key files, configured by kyuubi.frontend.flight.sql.ssl.cert.file and kyuubi.frontend.flight.sql.ssl.key.file. When those are unset, Kyuubi can materialize temporary PEM files from the shared kyuubi.frontend.ssl.keystore.* settings. | boolean | 1.13.0 | +| kyuubi.frontend.flight.sql.ssl.key.file | <undefined> | PEM private key file used by the Arrow Flight SQL frontend when TLS is enabled. | string | 1.13.0 | +| kyuubi.frontend.flight.sql.token.ttl | PT2H | Lifetime of Arrow Flight SQL bearer tokens issued after Basic or SPNEGO authentication. | duration | 1.13.0 | | kyuubi.frontend.jetty.sendVersion.enabled | true | Whether to send Jetty version in HTTP response. | boolean | 1.9.3 | | kyuubi.frontend.max.message.size | 104857600 | (deprecated) Maximum message size in bytes a Kyuubi server will accept. | int | 1.0.0 | | kyuubi.frontend.max.worker.threads | 999 | (deprecated) Maximum number of threads in the frontend worker thread pool for the thrift frontend service | int | 1.0.0 | | kyuubi.frontend.min.worker.threads | 9 | (deprecated) Minimum number of threads in the frontend worker thread pool for the thrift frontend service | int | 1.0.0 | -| kyuubi.frontend.protocols | THRIFT_BINARY,REST | A comma-separated list for all frontend protocols | seq | 1.4.0 | +| kyuubi.frontend.protocols | THRIFT_BINARY,REST | A comma-separated list for all frontend protocols | seq | 1.4.0 | | kyuubi.frontend.proxy.http.client.ip.header | X-Real-IP | The HTTP header to record the real client IP address. If your server is behind a load balancer or other proxy, the server will see this load balancer or proxy IP address as the client IP address, to get around this common issue, most load balancers or proxies offer the ability to record the real remote IP address in an HTTP header that will be added to the request for other devices to use. Note that, because the header value can be specified to any IP address, so it will not be used for authentication. | string | 1.6.0 | | kyuubi.frontend.rest.bind.host | <undefined> | Hostname or IP of the machine on which to run the REST frontend service. | string | 1.4.0 | | kyuubi.frontend.rest.bind.port | 10099 | Port of the machine on which to run the REST frontend service. | int | 1.4.0 | @@ -317,6 +324,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.ha.etcd.ssl.client.certificate.path | <undefined> | Where the etcd SSL certificate file is stored. | string | 1.6.0 | | kyuubi.ha.etcd.ssl.client.key.path | <undefined> | Where the etcd SSL key file is stored. | string | 1.6.0 | | kyuubi.ha.etcd.ssl.enabled | false | When set to true, will build an SSL secured etcd client. | boolean | 1.6.0 | +| kyuubi.ha.flight.sql.namespace | kyuubi_flight | The root directory for the Arrow Flight SQL frontend service to deploy its instance uri. Must be different from kyuubi.ha.namespace so Thrift/JDBC clients do not discover Flight gRPC endpoints. | string | 1.13.0 | | kyuubi.ha.namespace | kyuubi | The root directory for the service to deploy its instance uri | string | 1.6.0 | | kyuubi.ha.zookeeper.acl.enabled | false | (deprecated) Set to true if the ZooKeeper ensemble is kerberized | boolean | 1.0.0 | | kyuubi.ha.zookeeper.auth.digest | <undefined> | The digest auth string is used for ZooKeeper authentication, like: username:password. | string | 1.3.2 | diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 4239f8b6187..0ec8497bb8b 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -261,6 +261,9 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { } def isRESTEnabled: Boolean = get(FRONTEND_PROTOCOLS).contains(FrontendProtocols.REST.toString) + + def isFlightSqlEnabled: Boolean = + get(FRONTEND_PROTOCOLS).contains(FrontendProtocols.FLIGHT_SQL.toString) } /** @@ -499,7 +502,7 @@ object KyuubiConf { object FrontendProtocols extends Enumeration { type FrontendProtocol = Value - val THRIFT_BINARY, THRIFT_HTTP, REST, TRINO = Value + val THRIFT_BINARY, THRIFT_HTTP, REST, TRINO, FLIGHT_SQL = Value } val FRONTEND_PROTOCOLS: ConfigEntry[Seq[String]] = @@ -512,6 +515,7 @@ object KyuubiConf { "
  • THRIFT_HTTP - HiveServer2 compatible thrift http protocol.
  • " + "
  • REST - Kyuubi defined REST API(experimental).
  • " + "
  • TRINO - Trino compatible http protocol(experimental).
  • " + + "
  • FLIGHT_SQL - Arrow Flight SQL compatible gRPC protocol(experimental).
  • " + "") .version("1.4.0") .stringConf @@ -1373,6 +1377,75 @@ object KyuubiConf { .timeConf .createWithDefaultString("PT5S") + val FRONTEND_FLIGHT_SQL_BIND_HOST: ConfigEntry[Option[String]] = + buildConf("kyuubi.frontend.flight.sql.bind.host") + .doc("Hostname or IP on which to run the Arrow Flight SQL gRPC frontend service.") + .version("1.13.0") + .audience(SERVER) + .immutable + .fallbackConf(FRONTEND_BIND_HOST) + + val FRONTEND_FLIGHT_SQL_BIND_PORT: ConfigEntry[Int] = + buildConf("kyuubi.frontend.flight.sql.bind.port") + .doc("Port on which to run the Arrow Flight SQL gRPC frontend service.") + .version("1.13.0") + .audience(SERVER) + .immutable + .intConf + .checkValue(p => p == 0 || (p > 1024 && p < 65535), "Invalid Port number") + .createWithDefault(10299) + + val FRONTEND_FLIGHT_SQL_SSL_ENABLED: ConfigEntry[Boolean] = + buildConf("kyuubi.frontend.flight.sql.ssl.enabled") + .doc("Set this to true to enable TLS/SSL encryption on the Arrow Flight SQL gRPC frontend. " + + "Arrow Flight 16 requires PEM certificate and private key files, configured by " + + "kyuubi.frontend.flight.sql.ssl.cert.file and kyuubi.frontend.flight.sql.ssl.key.file. " + + "When those are unset, Kyuubi can materialize temporary PEM files from the shared " + + "kyuubi.frontend.ssl.keystore.* settings.") + .version("1.13.0") + .audience(SERVER) + .immutable + .booleanConf + .createWithDefault(false) + + val FRONTEND_FLIGHT_SQL_SSL_CERT_FILE: OptionalConfigEntry[String] = + buildConf("kyuubi.frontend.flight.sql.ssl.cert.file") + .doc("PEM certificate chain file used by the Arrow Flight SQL frontend when TLS is enabled.") + .version("1.13.0") + .audience(SERVER) + .immutable + .stringConf + .createOptional + + val FRONTEND_FLIGHT_SQL_SSL_KEY_FILE: OptionalConfigEntry[String] = + buildConf("kyuubi.frontend.flight.sql.ssl.key.file") + .doc("PEM private key file used by the Arrow Flight SQL frontend when TLS is enabled.") + .version("1.13.0") + .audience(SERVER) + .immutable + .stringConf + .createOptional + + val FRONTEND_FLIGHT_SQL_FETCH_MAX_ROWS: ConfigEntry[Int] = + buildConf("kyuubi.frontend.flight.sql.fetch.max.rows") + .doc("Maximum number of rows requested from Kyuubi for each Arrow Flight SQL result page.") + .version("1.13.0") + .audience(SERVER) + .immutable + .intConf + .checkValue(_ > 0, "must be positive") + .createWithDefault(1000) + + val FRONTEND_FLIGHT_SQL_TOKEN_TTL: ConfigEntry[Long] = + buildConf("kyuubi.frontend.flight.sql.token.ttl") + .doc("Lifetime of Arrow Flight SQL bearer tokens issued after Basic or " + + "SPNEGO authentication.") + .version("1.13.0") + .audience(SERVER) + .immutable + .timeConf + .createWithDefaultString("PT2H") + val KUBERNETES_CONTEXT: OptionalConfigEntry[String] = buildConf("kyuubi.kubernetes.context") .doc("The desired context from your kubernetes config file used to configure the K8s " + diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala index 12025c70a49..616d71f58c2 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala @@ -33,6 +33,29 @@ class KyuubiConfSuite extends KyuubiFunSuite { assert(conf.get(OPERATION_IDLE_TIMEOUT) === Duration.ofHours(3).toMillis) } + test("Flight SQL frontend configuration") { + val conf = KyuubiConf() + assert(conf.get(FRONTEND_FLIGHT_SQL_BIND_PORT) === 10299) + assert(conf.get(FRONTEND_FLIGHT_SQL_FETCH_MAX_ROWS) === 1000) + assert(conf.get(FRONTEND_FLIGHT_SQL_SSL_ENABLED) === false) + assert(conf.get(FRONTEND_FLIGHT_SQL_SSL_CERT_FILE).isEmpty) + assert(conf.get(FRONTEND_FLIGHT_SQL_SSL_KEY_FILE).isEmpty) + assert(conf.isFlightSqlEnabled === false) + + conf.set(FRONTEND_PROTOCOLS, Seq(FrontendProtocols.FLIGHT_SQL.toString)) + assert(conf.isFlightSqlEnabled) + + conf.set(FRONTEND_FLIGHT_SQL_BIND_PORT, 0) + assert(conf.get(FRONTEND_FLIGHT_SQL_BIND_PORT) === 0) + conf.set(FRONTEND_FLIGHT_SQL_BIND_HOST.key, "localhost") + assert(conf.get(FRONTEND_FLIGHT_SQL_BIND_HOST).contains("localhost")) + + assertThrows[IllegalArgumentException]( + conf.set(FRONTEND_FLIGHT_SQL_BIND_PORT, 1024).get(FRONTEND_FLIGHT_SQL_BIND_PORT)) + assertThrows[IllegalArgumentException]( + conf.set(FRONTEND_FLIGHT_SQL_FETCH_MAX_ROWS, 0).get(FRONTEND_FLIGHT_SQL_FETCH_MAX_ROWS)) + } + test("kyuubi conf w/ w/o no sys defaults") { val key = "kyuubi.conf.abc" System.setProperty(key, "xyz") diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/HighAvailabilityConf.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/HighAvailabilityConf.scala index 2ae0791a91c..6f2724f87aa 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/HighAvailabilityConf.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/HighAvailabilityConf.scala @@ -53,6 +53,15 @@ object HighAvailabilityConf { .version("1.6.0") .fallbackConf(HA_ZK_NAMESPACE) + val HA_FLIGHT_SQL_NAMESPACE: ConfigEntry[String] = + buildConf("kyuubi.ha.flight.sql.namespace") + .doc("The root directory for the Arrow Flight SQL frontend service to deploy its " + + "instance uri. Must be different from kyuubi.ha.namespace so Thrift/JDBC clients " + + "do not discover Flight gRPC endpoints.") + .version("1.13.0") + .stringConf + .createWithDefault("kyuubi_flight") + val HA_CLIENT_CLASS: ConfigEntry[String] = buildConf("kyuubi.ha.client.class") .doc("Class name for service discovery client.