From 1baada3e2b5768522d6f0f7c3320ccdc5176df19 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 12 Aug 2026 17:01:48 -0700 Subject: [PATCH] [SPARK-58753] Add Cargo workspace skeleton, vendored protos, and genproto crate Introduces the Cargo workspace for the Spark Connect Gateway: the root Cargo.toml (with the full [workspace.package] and [workspace.dependencies] tables; members lists only crates/genproto for now), Cargo.lock, rust-toolchain.toml, the vendored spark.connect.* protos under proto/, and the scg-genproto crate that generates tonic bindings from them at build time. This is the first of a series of small, bottom-up PRs importing the gateway reference implementation. Subsequent PRs append one or more crates to the workspace members list, keeping main green under the full CI matrix (fmt / clippy --all-targets / build / test --workspace) at each step. The proto files are a read-only mirror of the corresponding files in apache/spark; genproto's build.rs compiles them via tonic-prost-build. Co-authored-by: Claude Code --- Cargo.lock | 937 ++++++++++++++ Cargo.toml | 96 ++ crates/genproto/Cargo.toml | 36 + crates/genproto/build.rs | 40 + crates/genproto/src/lib.rs | 39 + proto/spark/connect/base.proto | 1367 +++++++++++++++++++++ proto/spark/connect/catalog.proto | 324 +++++ proto/spark/connect/commands.proto | 560 +++++++++ proto/spark/connect/common.proto | 179 +++ proto/spark/connect/example_plugins.proto | 42 + proto/spark/connect/expressions.proto | 557 +++++++++ proto/spark/connect/ml.proto | 147 +++ proto/spark/connect/ml_common.proto | 64 + proto/spark/connect/pipelines.proto | 343 ++++++ proto/spark/connect/relations.proto | 1278 +++++++++++++++++++ proto/spark/connect/types.proto | 227 ++++ rust-toolchain.toml | 18 + 17 files changed, 6254 insertions(+) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 crates/genproto/Cargo.toml create mode 100644 crates/genproto/build.rs create mode 100644 crates/genproto/src/lib.rs create mode 100644 proto/spark/connect/base.proto create mode 100644 proto/spark/connect/catalog.proto create mode 100644 proto/spark/connect/commands.proto create mode 100644 proto/spark/connect/common.proto create mode 100644 proto/spark/connect/example_plugins.proto create mode 100644 proto/spark/connect/expressions.proto create mode 100644 proto/spark/connect/ml.proto create mode 100644 proto/spark/connect/ml_common.proto create mode 100644 proto/spark/connect/pipelines.proto create mode 100644 proto/spark/connect/relations.proto create mode 100644 proto/spark/connect/types.proto create mode 100644 rust-toolchain.toml diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..5356f40 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,937 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "async-trait" +version = "0.1.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "scg-genproto" +version = "0.1.0" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-prost", + "tonic-prost-build", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.119", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..f26c9b3 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[workspace] +resolver = "2" +members = [ + "crates/genproto", +] + +[workspace.package] +version = "0.1.0" +edition = "2021" +authors = ["Apache Spark "] +license = "Apache-2.0" +repository = "https://github.com/apache/spark-connect-gateway" +readme = "README.md" +keywords = ["spark", "spark-connect", "gateway", "proxy", "grpc"] +categories = ["network-programming", "database"] +description = "Stateless gRPC proxy fronting Apache Spark Connect servers: session affinity, multi-tenant routing, auth, observability." +publish = false +rust-version = "1.82" + +[workspace.dependencies] +# Async + gRPC +tokio = { version = "1.41", features = ["full"] } +tokio-stream = "0.1" +tonic = "0.14" +tonic-prost = "0.14" +tonic-prost-build = "0.14" +tonic-health = "0.14" +tower = "0.5" +prost = "0.14" +prost-types = "0.14" + +# Logging + telemetry +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } +opentelemetry = "0.31" +opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] } +opentelemetry-otlp = { version = "0.31", default-features = false, features = ["trace", "grpc-tonic", "tls-ring", "tls-webpki-roots"] } +opentelemetry-semantic-conventions = "0.31" +opentelemetry-stdout = "0.31" +opentelemetry-proto = { version = "0.31", default-features = false, features = ["gen-tonic", "trace"] } +tracing-opentelemetry = "0.32" + +# Config + utilities +serde = { version = "1", features = ["derive"] } +serde_yaml = "0.9" +clap = { version = "4", features = ["derive"] } +anyhow = "1" +thiserror = "2" +async-trait = "0.1" +futures = "0.3" +parking_lot = "0.12" + +# K8s +kube = { version = "3.1", features = ["runtime", "client"] } +k8s-openapi = { version = "0.27", features = ["latest"] } + +# Affinity store backings +redis = { version = "1", default-features = false, features = ["tokio-comp", "connection-manager"] } + +# Auth +jsonwebtoken = "9" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } +subtle = "2" + +# Observability +prometheus = "0.14" +uuid = { version = "1", features = ["v4"] } +hyper = { version = "1", features = ["server", "http1"] } +hyper-util = { version = "0.1", features = ["tokio"] } +http-body-util = "0.1" +bytes = "1" + +# Test-only +tempfile = "3" +hdrhistogram = { version = "7", default-features = false } +testcontainers-modules = { version = "0.15", features = ["redis"] } + +[profile.release] +lto = "thin" +codegen-units = 1 +strip = "debuginfo" diff --git a/crates/genproto/Cargo.toml b/crates/genproto/Cargo.toml new file mode 100644 index 0000000..13be391 --- /dev/null +++ b/crates/genproto/Cargo.toml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "scg-genproto" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +description = "tonic-generated Rust bindings for the Apache Spark Connect protobufs." +publish = false + +[dependencies] +tonic = { workspace = true } +tonic-prost = { workspace = true } +prost = { workspace = true } +prost-types = { workspace = true } + +[build-dependencies] +tonic-prost-build = { workspace = true } diff --git a/crates/genproto/build.rs b/crates/genproto/build.rs new file mode 100644 index 0000000..04d10b2 --- /dev/null +++ b/crates/genproto/build.rs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::path::PathBuf; + +fn main() { + let proto_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join("proto"); + let connect_dir = proto_root.join("spark").join("connect"); + let entries: Vec = std::fs::read_dir(&connect_dir) + .expect("read proto dir") + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x == "proto")) + .collect(); + + for p in &entries { + println!("cargo:rerun-if-changed={}", p.display()); + } + + tonic_prost_build::configure() + .build_server(true) + .build_client(true) + .compile_protos(&entries, &[proto_root]) + .expect("tonic-prost-build compile_protos"); +} diff --git a/crates/genproto/src/lib.rs b/crates/genproto/src/lib.rs new file mode 100644 index 0000000..d588649 --- /dev/null +++ b/crates/genproto/src/lib.rs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Generated tonic bindings for the upstream `spark.connect.*` proto surface. +//! +//! The proto files under `proto/spark/connect/` are a read-only mirror of the +//! corresponding files in `apache/spark`. Bindings are regenerated at build +//! time by `build.rs`. +//! +//! Lints disabled here are intrinsic to prost-generated code, not bugs we +//! should fix: +//! * `large_enum_variant` — Spark Connect oneofs are inherently sized by +//! their largest variant; boxing every variant would be invasive and +//! would make the proto API noticeably worse to use. +//! * `clippy::all` more broadly: generated code is not human-authored, so +//! lint findings are not actionable. + +#![allow(clippy::all)] +#![allow(clippy::pedantic)] + +pub mod spark { + pub mod connect { + tonic::include_proto!("spark.connect"); + } +} + +pub use spark::connect as pb; diff --git a/proto/spark/connect/base.proto b/proto/spark/connect/base.proto new file mode 100644 index 0000000..c724712 --- /dev/null +++ b/proto/spark/connect/base.proto @@ -0,0 +1,1367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "google/protobuf/any.proto"; +import "spark/connect/commands.proto"; +import "spark/connect/common.proto"; +import "spark/connect/expressions.proto"; +import "spark/connect/relations.proto"; +import "spark/connect/types.proto"; +import "spark/connect/ml.proto"; +import "spark/connect/pipelines.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// A [[Plan]] is the structure that carries the runtime information for the execution from the +// client to the server. A [[Plan]] can be one of the following: +// - [[Relation]]: a reference to the underlying logical plan. +// - [[Command]]: used to execute commands on the server. +// - [[CompressedOperation]]: a compressed representation of either a Relation or a Command. +message Plan { + oneof op_type { + Relation root = 1; + Command command = 2; + CompressedOperation compressed_operation = 3; + } + + message CompressedOperation { + bytes data = 1; + OpType op_type = 2; + CompressionCodec compression_codec = 3; + + enum OpType { + OP_TYPE_UNSPECIFIED = 0; + OP_TYPE_RELATION = 1; + OP_TYPE_COMMAND = 2; + } + } +} + +// Compression codec for plan compression. +enum CompressionCodec { + COMPRESSION_CODEC_UNSPECIFIED = 0; + COMPRESSION_CODEC_ZSTD = 1; +} + +// User Context is used to refer to one particular user session that is executing +// queries in the backend. +message UserContext { + string user_id = 1; + string user_name = 2; + + // To extend the existing user context message that is used to identify incoming requests, + // Spark Connect leverages the Any protobuf type that can be used to inject arbitrary other + // messages into this message. Extensions are stored as a `repeated` type to be able to + // handle multiple active extensions. + repeated google.protobuf.Any extensions = 999; +} + +// Request to perform plan analyze, optionally to explain the plan. +message AnalyzePlanRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 17; + + // (Required) User context + UserContext user_context = 2; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + oneof analyze { + Schema schema = 4; + Explain explain = 5; + TreeString tree_string = 6; + IsLocal is_local = 7; + IsStreaming is_streaming = 8; + InputFiles input_files = 9; + SparkVersion spark_version = 10; + DDLParse ddl_parse = 11; + SameSemantics same_semantics = 12; + SemanticHash semantic_hash = 13; + Persist persist = 14; + Unpersist unpersist = 15; + GetStorageLevel get_storage_level = 16; + JsonToDDL json_to_ddl = 18; + } + + message Schema { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + } + + // Explains the input plan based on a configurable mode. + message Explain { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + + // (Required) For analyzePlan rpc calls, configure the mode to explain plan in strings. + ExplainMode explain_mode = 2; + + // Plan explanation mode. + enum ExplainMode { + EXPLAIN_MODE_UNSPECIFIED = 0; + + // Generates only physical plan. + EXPLAIN_MODE_SIMPLE = 1; + + // Generates parsed logical plan, analyzed logical plan, optimized logical plan and physical plan. + // Parsed Logical plan is a unresolved plan that extracted from the query. Analyzed logical plans + // transforms which translates unresolvedAttribute and unresolvedRelation into fully typed objects. + // The optimized logical plan transforms through a set of optimization rules, resulting in the + // physical plan. + EXPLAIN_MODE_EXTENDED = 2; + + // Generates code for the statement, if any and a physical plan. + EXPLAIN_MODE_CODEGEN = 3; + + // If plan node statistics are available, generates a logical plan and also the statistics. + EXPLAIN_MODE_COST = 4; + + // Generates a physical plan outline and also node details. + EXPLAIN_MODE_FORMATTED = 5; + } + } + + message TreeString { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + + // (Optional) Max level of the schema. + optional int32 level = 2; + } + + message IsLocal { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + } + + message IsStreaming { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + } + + message InputFiles { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + } + + message SparkVersion { } + + message DDLParse { + // (Required) The DDL formatted string to be parsed. + string ddl_string = 1; + } + + + // Returns `true` when the logical query plans are equal and therefore return same results. + message SameSemantics { + // (Required) The plan to be compared. + Plan target_plan = 1; + + // (Required) The other plan to be compared. + Plan other_plan = 2; + } + + message SemanticHash { + // (Required) The logical plan to get a hashCode. + Plan plan = 1; + } + + message Persist { + // (Required) The logical plan to persist. + Relation relation = 1; + + // (Optional) The storage level. + optional StorageLevel storage_level = 2; + } + + message Unpersist { + // (Required) The logical plan to unpersist. + Relation relation = 1; + + // (Optional) Whether to block until all blocks are deleted. + optional bool blocking = 2; + } + + message GetStorageLevel { + // (Required) The logical plan to get the storage level. + Relation relation = 1; + } + + message JsonToDDL { + // (Required) The JSON formatted string to be converted to DDL. + string json_string = 1; + } +} + +// Response to performing analysis of the query. Contains relevant metadata to be able to +// reason about the performance. +// Next ID: 16 +message AnalyzePlanResponse { + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 15; + + oneof result { + Schema schema = 2; + Explain explain = 3; + TreeString tree_string = 4; + IsLocal is_local = 5; + IsStreaming is_streaming = 6; + InputFiles input_files = 7; + SparkVersion spark_version = 8; + DDLParse ddl_parse = 9; + SameSemantics same_semantics = 10; + SemanticHash semantic_hash = 11; + Persist persist = 12; + Unpersist unpersist = 13; + GetStorageLevel get_storage_level = 14; + JsonToDDL json_to_ddl = 16; + } + + message Schema { + DataType schema = 1; + } + + message Explain { + string explain_string = 1; + } + + message TreeString { + string tree_string = 1; + } + + message IsLocal { + bool is_local = 1; + } + + message IsStreaming { + bool is_streaming = 1; + } + + message InputFiles { + // A best-effort snapshot of the files that compose this Dataset + repeated string files = 1; + } + + message SparkVersion { + string version = 1; + } + + message DDLParse { + DataType parsed = 1; + } + + message SameSemantics { + bool result = 1; + } + + message SemanticHash { + int32 result = 1; + } + + message Persist { } + + message Unpersist { } + + message GetStorageLevel { + // (Required) The StorageLevel as a result of get_storage_level request. + StorageLevel storage_level = 1; + } + + message JsonToDDL { + string ddl_string = 1; + } +} + +// A request to be executed by the service. +message ExecutePlanRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 8; + + // (Required) User context + // + // user_context.user_id and session+id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // (Optional) + // Provide an id for this request. If not provided, it will be generated by the server. + // It is returned in every ExecutePlanResponse.operation_id of the ExecutePlan response stream. + // The id must be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + optional string operation_id = 6; + + // (Required) The logical plan to be executed / analyzed. + Plan plan = 3; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 4; + + // Repeated element for options that can be passed to the request. This element is currently + // unused but allows to pass in an extension value used for arbitrary options. + repeated RequestOption request_options = 5; + + message RequestOption { + oneof request_option { + ReattachOptions reattach_options = 1; + ResultChunkingOptions result_chunking_options = 2; + // Extension type for request options + google.protobuf.Any extension = 999; + } + } + + // Tags to tag the given execution with. + // Tags cannot contain ',' character and cannot be empty strings. + // Used by Interrupt with interrupt.tag. + repeated string tags = 7; +} + +// The response of a query, can be one or more for each request. Responses belonging to the +// same input query, carry the same `session_id`. +// Next ID: 17 +message ExecutePlanResponse { + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 15; + + // Identifies the ExecutePlan execution. + // If set by the client in ExecutePlanRequest.operationId, that value is returned. + // Otherwise generated by the server. + // It is an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string operation_id = 12; + + // Identified the response in the stream. + // The id is an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string response_id = 13; + + // Union type for the different response messages. + oneof response_type { + ArrowBatch arrow_batch = 2; + + // Special case for executing SQL commands. + SqlCommandResult sql_command_result = 5; + + // Response for a streaming query. + WriteStreamOperationStartResult write_stream_operation_start_result = 8; + + // Response for commands on a streaming query. + StreamingQueryCommandResult streaming_query_command_result = 9; + + // Response for 'SparkContext.resources'. + GetResourcesCommandResult get_resources_command_result = 10; + + // Response for commands on the streaming query manager. + StreamingQueryManagerCommandResult streaming_query_manager_command_result = 11; + + // Response for commands on the client side streaming query listener. + StreamingQueryListenerEventsResult streaming_query_listener_events_result = 16; + + // Response type informing if the stream is complete in reattachable execution. + ResultComplete result_complete = 14; + + // Response for command that creates ResourceProfile. + CreateResourceProfileCommandResult create_resource_profile_command_result = 17; + + // (Optional) Intermediate query progress reports. + ExecutionProgress execution_progress = 18; + + // Response for command that checkpoints a DataFrame. + CheckpointCommandResult checkpoint_command_result = 19; + + // ML command response + MlCommandResult ml_command_result = 20; + + // Response containing pipeline event that is streamed back to the client during a pipeline run + PipelineEventResult pipeline_event_result = 21; + + // Pipeline command response + PipelineCommandResult pipeline_command_result = 22; + + // A signal from the server to the client to execute the query function for a flow, and to + // register its result with the server. + PipelineQueryFunctionExecutionSignal pipeline_query_function_execution_signal = 23; + + // Support arbitrary result objects. + google.protobuf.Any extension = 999; + } + + // Metrics for the query execution. Typically, this field is only present in the last + // batch of results and then represent the overall state of the query execution. + Metrics metrics = 4; + + // The metrics observed during the execution of the query plan. + repeated ObservedMetrics observed_metrics = 6; + + // (Optional) The Spark schema. This field is available when `collect` is called. + DataType schema = 7; + + // A SQL command returns an opaque Relation that can be directly used as input for the next + // call. + message SqlCommandResult { + Relation relation = 1; + } + + // Batch results of metrics. + message ArrowBatch { + // Count rows in `data`. Must match the number of rows inside `data`. + int64 row_count = 1; + // Serialized Arrow data. + bytes data = 2; + + // If set, row offset of the start of this ArrowBatch in execution results. + optional int64 start_offset = 3; + + // Index of this chunk in the batch if chunking is enabled. The index starts from 0. + optional int64 chunk_index = 4; + + // Total number of chunks in this batch if chunking is enabled. + // It is missing when chunking is disabled - the batch is returned whole + // and client will treat this response as the batch. + optional int64 num_chunks_in_batch = 5; + } + + message Metrics { + + repeated MetricObject metrics = 1; + + message MetricObject { + string name = 1; + int64 plan_id = 2; + int64 parent = 3; + map execution_metrics = 4; + } + + message MetricValue { + string name = 1; + int64 value = 2; + string metric_type = 3; + } + } + + message ObservedMetrics { + string name = 1; + repeated Expression.Literal values = 2; + repeated string keys = 3; + int64 plan_id = 4; + // (Optional) The index of the root error in errors. + // The field will not be set if there are no errors. + optional int32 root_error_idx = 5; + // A list of errors that occurred while collecting the observed metrics. + // If the length is 0, it means no errors occurred. + repeated FetchErrorDetailsResponse.Error errors = 6; + } + + message ResultComplete { + // If present, in a reattachable execution this means that after server sends onComplete, + // the execution is complete. If the server sends onComplete without sending a ResultComplete, + // it means that there is more, and the client should use ReattachExecute RPC to continue. + } + + // This message is used to communicate progress about the query progress during the execution. + message ExecutionProgress { + // Captures the progress of each individual stage. + repeated StageInfo stages = 1; + + // Captures the currently in progress tasks. + int64 num_inflight_tasks = 2; + + message StageInfo { + int64 stage_id = 1; + int64 num_tasks = 2; + int64 num_completed_tasks = 3; + int64 input_bytes_read = 4; + bool done = 5; + } + } +} + +// The key-value pair for the config request and response. +message KeyValue { + // (Required) The key. + string key = 1; + // (Optional) The value. + optional string value = 2; +} + +// Request to update or fetch the configurations. +message ConfigRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 8; + + // (Required) User context + UserContext user_context = 2; + + // (Required) The operation for the config. + Operation operation = 3; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 4; + + message Operation { + oneof op_type { + Set set = 1; + Get get = 2; + GetWithDefault get_with_default = 3; + GetOption get_option = 4; + GetAll get_all = 5; + Unset unset = 6; + IsModifiable is_modifiable = 7; + } + } + + message Set { + // (Required) The config key-value pairs to set. + repeated KeyValue pairs = 1; + + // (Optional) Whether to ignore failures. + optional bool silent = 2; + } + + message Get { + // (Required) The config keys to get. + repeated string keys = 1; + } + + message GetWithDefault { + // (Required) The config key-value pairs to get. The value will be used as the default value. + repeated KeyValue pairs = 1; + } + + message GetOption { + // (Required) The config keys to get optionally. + repeated string keys = 1; + } + + message GetAll { + // (Optional) The prefix of the config key to get. + optional string prefix = 1; + } + + message Unset { + // (Required) The config keys to unset. + repeated string keys = 1; + } + + message IsModifiable { + // (Required) The config keys to check the config is modifiable. + repeated string keys = 1; + } +} + +// Response to the config request. +// Next ID: 5 +message ConfigResponse { + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 4; + + // (Optional) The result key-value pairs. + // + // Available when the operation is 'Get', 'GetWithDefault', 'GetOption', 'GetAll'. + // Also available for the operation 'IsModifiable' with boolean string "true" and "false". + repeated KeyValue pairs = 2; + + // (Optional) + // + // Warning messages for deprecated or unsupported configurations. + repeated string warnings = 3; +} + +// Request to transfer client-local artifacts. +message AddArtifactsRequest { + + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // User context + UserContext user_context = 2; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 7; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 6; + + // A chunk of an Artifact. + message ArtifactChunk { + // Data chunk. + bytes data = 1; + // CRC to allow server to verify integrity of the chunk. + int64 crc = 2; + } + + // An artifact that is contained in a single `ArtifactChunk`. + // Generally, this message represents tiny artifacts such as REPL-generated class files. + message SingleChunkArtifact { + // The name of the artifact is expected in the form of a "Relative Path" that is made up of a + // sequence of directories and the final file element. + // Examples of "Relative Path"s: "jars/test.jar", "classes/xyz.class", "abc.xyz", "a/b/X.jar". + // The server is expected to maintain the hierarchy of files as defined by their name. (i.e + // The relative path of the file on the server's filesystem will be the same as the name of + // the provided artifact) + string name = 1; + // A single data chunk. + ArtifactChunk data = 2; + } + + // A number of `SingleChunkArtifact` batched into a single RPC. + message Batch { + repeated SingleChunkArtifact artifacts = 1; + } + + // Signals the beginning/start of a chunked artifact. + // A large artifact is transferred through a payload of `BeginChunkedArtifact` followed by a + // sequence of `ArtifactChunk`s. + message BeginChunkedArtifact { + // Name of the artifact undergoing chunking. Follows the same conventions as the `name` in + // the `Artifact` message. + string name = 1; + // Total size of the artifact in bytes. + int64 total_bytes = 2; + // Number of chunks the artifact is split into. + // This includes the `initial_chunk`. + int64 num_chunks = 3; + // The first/initial chunk. + ArtifactChunk initial_chunk = 4; + } + + // The payload is either a batch of artifacts or a partial chunk of a large artifact. + oneof payload { + Batch batch = 3; + // The metadata and the initial chunk of a large artifact chunked into multiple requests. + // The server side is notified about the total size of the large artifact as well as the + // number of chunks to expect. + BeginChunkedArtifact begin_chunk = 4; + // A chunk of an artifact excluding metadata. This can be any chunk of a large artifact + // excluding the first chunk (which is included in `BeginChunkedArtifact`). + ArtifactChunk chunk = 5; + } +} + +// Response to adding an artifact. Contains relevant metadata to verify successful transfer of +// artifact(s). +// Next ID: 4 +message AddArtifactsResponse { + // Session id in which the AddArtifact was running. + string session_id = 2; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 3; + + // The list of artifact(s) seen by the server. + repeated ArtifactSummary artifacts = 1; + + // Metadata of an artifact. + message ArtifactSummary { + string name = 1; + // Whether the CRC (Cyclic Redundancy Check) is successful on server verification. + // The server discards any artifact that fails the CRC. + // If false, the client may choose to resend the artifact specified by `name`. + bool is_crc_successful = 2; + } +} + +// Request to get current statuses of artifacts at the server side. +message ArtifactStatusesRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 5; + + // User context + UserContext user_context = 2; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // The name of the artifact is expected in the form of a "Relative Path" that is made up of a + // sequence of directories and the final file element. + // Examples of "Relative Path"s: "jars/test.jar", "classes/xyz.class", "abc.xyz", "a/b/X.jar". + // The server is expected to maintain the hierarchy of files as defined by their name. (i.e + // The relative path of the file on the server's filesystem will be the same as the name of + // the provided artifact) + repeated string names = 4; +} + +// Response to checking artifact statuses. +// Next ID: 4 +message ArtifactStatusesResponse { + // Session id in which the ArtifactStatus was running. + string session_id = 2; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 3; + // A map of artifact names to their statuses. + map statuses = 1; + + message ArtifactStatus { + // Exists or not particular artifact at the server. + bool exists = 1; + } +} + +message InterruptRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 7; + + // (Required) User context + UserContext user_context = 2; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // (Required) The type of interrupt to execute. + InterruptType interrupt_type = 4; + + enum InterruptType { + INTERRUPT_TYPE_UNSPECIFIED = 0; + + // Interrupt all running executions within the session with the provided session_id. + INTERRUPT_TYPE_ALL = 1; + + // Interrupt all running executions within the session with the provided operation_tag. + INTERRUPT_TYPE_TAG = 2; + + // Interrupt the running execution within the session with the provided operation_id. + INTERRUPT_TYPE_OPERATION_ID = 3; + } + + oneof interrupt { + // if interrupt_tag == INTERRUPT_TYPE_TAG, interrupt operation with this tag. + string operation_tag = 5; + + // if interrupt_tag == INTERRUPT_TYPE_OPERATION_ID, interrupt operation with this operation_id. + string operation_id = 6; + } +} + +// Next ID: 4 +message InterruptResponse { + // Session id in which the interrupt was running. + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 3; + + // Operation ids of the executions which were interrupted. + repeated string interrupted_ids = 2; + +} + +message ReattachOptions { + // If true, the request can be reattached to using ReattachExecute. + // ReattachExecute can be used either if the stream broke with a GRPC network error, + // or if the server closed the stream without sending a response with StreamStatus.complete=true. + // The server will keep a buffer of responses in case a response is lost, and + // ReattachExecute needs to back-track. + // + // If false, the execution response stream will will not be reattachable, and all responses are + // immediately released by the server after being sent. + bool reattachable = 1; +} + +message ResultChunkingOptions { + // Although Arrow results are split into batches with a size limit according to estimation, the + // size of the batches is not guaranteed to be less than the limit, especially when a single row + // is larger than the limit, in which case the server will fail to split it further into smaller + // batches. As a result, the client may encounter a gRPC error stating “Received message larger + // than max” when a batch is too large. + // If allow_arrow_batch_chunking=true, the server will split large Arrow batches into smaller chunks, + // and the client is expected to handle the chunked Arrow batches. + // + // If false, the server will not chunk large Arrow batches. + bool allow_arrow_batch_chunking = 1; + + // Optional preferred Arrow batch size in bytes for the server to use when sending Arrow results. + // The server will attempt to use this size if it is set and within the valid range + // ([1KB, max batch size on server]). Otherwise, the server's maximum batch size is used. + optional int64 preferred_arrow_chunk_size = 2; +} + +message ReattachExecuteRequest { + // (Required) + // + // The session_id of the request to reattach to. + // This must be an id of existing session. + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 6; + + // (Required) User context + // + // user_context.user_id and session+id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // (Required) + // Provide an id of the request to reattach to. + // This must be an id of existing operation. + string operation_id = 3; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 4; + + // (Optional) + // Last already processed response id from the response stream. + // After reattach, server will resume the response stream after that response. + // If not specified, server will restart the stream from the start. + // + // Note: server controls the amount of responses that it buffers and it may drop responses, + // that are far behind the latest returned response, so this can't be used to arbitrarily + // scroll back the cursor. If the response is no longer available, this will result in an error. + optional string last_response_id = 5; +} + +message ReleaseExecuteRequest { + // (Required) + // + // The session_id of the request to reattach to. + // This must be an id of existing session. + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 7; + + // (Required) User context + // + // user_context.user_id and session+id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // (Required) + // Provide an id of the request to reattach to. + // This must be an id of existing operation. + string operation_id = 3; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 4; + + // Release and close operation completely. + // This will also interrupt the query if it is running execution, and wait for it to be torn down. + message ReleaseAll {} + + // Release all responses from the operation response stream up to and including + // the response with the given by response_id. + // While server determines by itself how much of a buffer of responses to keep, client providing + // explicit release calls will help reduce resource consumption. + // Noop if response_id not found in cached responses. + message ReleaseUntil { + string response_id = 1; + } + + oneof release { + ReleaseAll release_all = 5; + ReleaseUntil release_until = 6; + } +} + +// Next ID: 4 +message ReleaseExecuteResponse { + // Session id in which the release was running. + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 3; + + // Operation id of the operation on which the release executed. + // If the operation couldn't be found (because e.g. it was concurrently released), will be unset. + // Otherwise, it will be equal to the operation_id from request. + optional string operation_id = 2; +} + +message ReleaseSessionRequest { + // (Required) + // + // The session_id of the request to reattach to. + // This must be an id of existing session. + string session_id = 1; + + // (Required) User context + // + // user_context.user_id and session+id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // Signals the server to allow the client to reconnect to the session after it is released. + // + // By default, the server tombstones the session upon release, preventing reconnections and + // fully cleaning the session state. + // + // If this flag is set to true, the server may permit the client to reconnect to the session + // post-release, even if the session state has been cleaned. This can result in missing state, + // such as Temporary Views, Temporary UDFs, or the Current Catalog, in the reconnected session. + // + // Use this option sparingly and only when the client fully understands the implications of + // reconnecting to a released session. The client must ensure that any queries executed do not + // rely on the session state prior to its release. + bool allow_reconnect = 4; +} + +// Next ID: 3 +message ReleaseSessionResponse { + // Session id of the session on which the release executed. + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 2; +} + +message FetchErrorDetailsRequest { + + // (Required) + // The session_id specifies a Spark session for a user identified by user_context.user_id. + // The id should be a UUID string of the format `00112233-4455-6677-8899-aabbccddeeff`. + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 5; + + // User context + UserContext user_context = 2; + + // (Required) + // The id of the error. + string error_id = 3; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 4; +} + +// Next ID: 5 +message FetchErrorDetailsResponse { + + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 3; + + string session_id = 4; + + // The index of the root error in errors. The field will not be set if the error is not found. + optional int32 root_error_idx = 1; + + // A list of errors. + repeated Error errors = 2; + + message StackTraceElement { + // The fully qualified name of the class containing the execution point. + string declaring_class = 1; + + // The name of the method containing the execution point. + string method_name = 2; + + // The name of the file containing the execution point. + optional string file_name = 3; + + // The line number of the source line containing the execution point. + int32 line_number = 4; + } + + // QueryContext defines the schema for the query context of a SparkThrowable. + // It helps users understand where the error occurs while executing queries. + message QueryContext { + // The type of this query context. + enum ContextType { + SQL = 0; + DATAFRAME = 1; + } + ContextType context_type = 10; + + // The object type of the query which throws the exception. + // If the exception is directly from the main query, it should be an empty string. + // Otherwise, it should be the exact object type in upper case. For example, a "VIEW". + string object_type = 1; + + // The object name of the query which throws the exception. + // If the exception is directly from the main query, it should be an empty string. + // Otherwise, it should be the object name. For example, a view name "V1". + string object_name = 2; + + // The starting index in the query text which throws the exception. The index starts from 0. + int32 start_index = 3; + + // The stopping index in the query which throws the exception. The index starts from 0. + int32 stop_index = 4; + + // The corresponding fragment of the query which throws the exception. + string fragment = 5; + + // The user code (call site of the API) that caused throwing the exception. + string call_site = 6; + + // Summary of the exception cause. + string summary = 7; + } + + // SparkThrowable defines the schema for SparkThrowable exceptions. + message SparkThrowable { + // Succinct, human-readable, unique, and consistent representation of the error category. + optional string error_class = 1; + + // The message parameters for the error framework. + map message_parameters = 2; + + // The query context of a SparkThrowable. + repeated QueryContext query_contexts = 3; + + // Portable error identifier across SQL engines + // If null, error class or SQLSTATE is not set. + optional string sql_state = 4; + + // Additional information if the error was caused by a breaking change. + optional BreakingChangeInfo breaking_change_info = 5; + } + + // BreakingChangeInfo defines the schema for breaking change information. + message BreakingChangeInfo { + // A message explaining how the user can migrate their job to work + // with the breaking change. + repeated string migration_message = 1; + + // A spark config flag that can be used to mitigate the breaking change. + optional MitigationConfig mitigation_config = 2; + + // If true, the breaking change should be inspected manually. + // If false, the spark job should be retried by setting the mitigationConfig. + optional bool needs_audit = 3; + } + + // MitigationConfig defines a spark config flag that can be used to mitigate a breaking change. + message MitigationConfig { + // The spark config key. + string key = 1; + + // The spark config value that mitigates the breaking change. + string value = 2; + } + + // Error defines the schema for the representing exception. + message Error { + // The fully qualified names of the exception class and its parent classes. + repeated string error_type_hierarchy = 1; + + // The detailed message of the exception. + string message = 2; + + // The stackTrace of the exception. It will be set + // if the SQLConf spark.sql.connect.serverStacktrace.enabled is true. + repeated StackTraceElement stack_trace = 3; + + // The index of the cause error in errors. + optional int32 cause_idx = 4; + + // The structured data of a SparkThrowable exception. + optional SparkThrowable spark_throwable = 5; + } +} + +message CheckpointCommandResult { + // (Required) The logical plan checkpointed. + CachedRemoteRelation relation = 1; +} + +message CloneSessionRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 5; + + // (Required) User context + // + // user_context.user_id and session_id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // (Optional) + // The session_id for the new cloned session. If not provided, a new UUID will be generated. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + optional string new_session_id = 4; +} + +// Next ID: 5 +message CloneSessionResponse { + // Session id of the original session that was cloned. + string session_id = 1; + + // Server-side generated idempotency key that the client can use to assert that the server side + // session (parent of the cloned session) has not changed. + string server_side_session_id = 2; + + // Session id of the new cloned session. + string new_session_id = 3; + + // Server-side session ID of the new cloned session. + string new_server_side_session_id = 4; +} + +// Next ID: 6 +message GetStatusRequest { + // (Required) + // + // The session_id specifies a Spark session for a user identified by user_context.user_id. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Required) + // + // user_context.user_id and session_id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // (Optional) + // + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 4; + + // (Optional) + // + // Get status of operations in the session. + optional OperationStatusRequest operation_status = 5; + + // Extension point for custom status request types. + repeated google.protobuf.Any extensions = 999; + + message OperationStatusRequest { + // Get status of operations with these operation_ids. + // If unset or empty, returns status of all operations in the session. + repeated string operation_ids = 1; + + // Extension point for custom operation-level status requests. + repeated google.protobuf.Any extensions = 999; + } +} + +// Next ID: 4 +message GetStatusResponse { + // Session id of the session for which the status was requested. + string session_id = 1; + + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 2; + + // Status information about requested operations. + repeated OperationStatus operation_statuses = 3; + + // Extension point for custom status response types. + repeated google.protobuf.Any extensions = 999; + + // Status information for a single operation. + message OperationStatus { + // The operation_id of the operation. + string operation_id = 1; + + // The current status of the operation. + OperationState state = 2; + + // Extension point for custom operation-level status fields. + repeated google.protobuf.Any extensions = 999; + + enum OperationState { + OPERATION_STATE_UNSPECIFIED = 0; + OPERATION_STATE_UNKNOWN = 1; + OPERATION_STATE_RUNNING = 2; + OPERATION_STATE_TERMINATING = 3; + OPERATION_STATE_SUCCEEDED = 4; + OPERATION_STATE_FAILED = 5; + OPERATION_STATE_CANCELLED = 6; + } + } +} + +// Main interface for the SparkConnect service. +service SparkConnectService { + + // Executes a request that contains the query and returns a stream of [[Response]]. + // + // It is guaranteed that there is at least one ARROW batch returned even if the result set is empty. + rpc ExecutePlan(ExecutePlanRequest) returns (stream ExecutePlanResponse) {} + + // Analyzes a query and returns a [[AnalyzeResponse]] containing metadata about the query. + rpc AnalyzePlan(AnalyzePlanRequest) returns (AnalyzePlanResponse) {} + + // Update or fetch the configurations and returns a [[ConfigResponse]] containing the result. + rpc Config(ConfigRequest) returns (ConfigResponse) {} + + // Add artifacts to the session and returns a [[AddArtifactsResponse]] containing metadata about + // the added artifacts. + rpc AddArtifacts(stream AddArtifactsRequest) returns (AddArtifactsResponse) {} + + // Check statuses of artifacts in the session and returns them in a [[ArtifactStatusesResponse]] + rpc ArtifactStatus(ArtifactStatusesRequest) returns (ArtifactStatusesResponse) {} + + // Interrupts running executions + rpc Interrupt(InterruptRequest) returns (InterruptResponse) {} + + // Reattach to an existing reattachable execution. + // The ExecutePlan must have been started with ReattachOptions.reattachable=true. + // If the ExecutePlanResponse stream ends without a ResultComplete message, there is more to + // continue. If there is a ResultComplete, the client should use ReleaseExecute with + rpc ReattachExecute(ReattachExecuteRequest) returns (stream ExecutePlanResponse) {} + + // Release an reattachable execution, or parts thereof. + // The ExecutePlan must have been started with ReattachOptions.reattachable=true. + // Non reattachable executions are released automatically and immediately after the ExecutePlan + // RPC and ReleaseExecute may not be used. + rpc ReleaseExecute(ReleaseExecuteRequest) returns (ReleaseExecuteResponse) {} + + // Release a session. + // All the executions in the session will be released. Any further requests for the session with + // that session_id for the given user_id will fail. If the session didn't exist or was already + // released, this is a noop. + rpc ReleaseSession(ReleaseSessionRequest) returns (ReleaseSessionResponse) {} + + // FetchErrorDetails retrieves the matched exception with details based on a provided error id. + rpc FetchErrorDetails(FetchErrorDetailsRequest) returns (FetchErrorDetailsResponse) {} + + // Create a clone of a Spark Connect session on the server side. The server-side session + // is cloned with all its current state (SQL configurations, temporary views, registered + // functions, catalog state) copied over to a new independent session. The cloned session + // is isolated from the source session - any subsequent changes to either session's + // server-side state will not be reflected in the other. + // + // The request can optionally specify a custom session ID for the cloned session (must be + // a valid UUID). If not provided, a new UUID will be generated automatically. + rpc CloneSession(CloneSessionRequest) returns (CloneSessionResponse) {} + + // Get status information of different types. + rpc GetStatus(GetStatusRequest) returns (GetStatusResponse) {} +} diff --git a/proto/spark/connect/catalog.proto b/proto/spark/connect/catalog.proto new file mode 100644 index 0000000..b5341d1 --- /dev/null +++ b/proto/spark/connect/catalog.proto @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "spark/connect/common.proto"; +import "spark/connect/types.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// Catalog messages are marked as unstable. +message Catalog { + oneof cat_type { + CurrentDatabase current_database = 1; + SetCurrentDatabase set_current_database = 2; + ListDatabases list_databases = 3; + ListTables list_tables = 4; + ListFunctions list_functions = 5; + ListColumns list_columns = 6; + GetDatabase get_database = 7; + GetTable get_table = 8; + GetFunction get_function = 9; + DatabaseExists database_exists = 10; + TableExists table_exists = 11; + FunctionExists function_exists = 12; + CreateExternalTable create_external_table = 13; + CreateTable create_table = 14; + DropTempView drop_temp_view = 15; + DropGlobalTempView drop_global_temp_view = 16; + RecoverPartitions recover_partitions = 17; + IsCached is_cached = 18; + CacheTable cache_table = 19; + UncacheTable uncache_table = 20; + ClearCache clear_cache = 21; + RefreshTable refresh_table = 22; + RefreshByPath refresh_by_path = 23; + CurrentCatalog current_catalog = 24; + SetCurrentCatalog set_current_catalog = 25; + ListCatalogs list_catalogs = 26; + DropTable drop_table = 27; + DropView drop_view = 28; + CreateDatabase create_database = 29; + DropDatabase drop_database = 30; + ListPartitions list_partitions = 31; + ListViews list_views = 32; + GetTableProperties get_table_properties = 33; + GetCreateTableString get_create_table_string = 34; + TruncateTable truncate_table = 35; + AnalyzeTable analyze_table = 36; + } +} + +// See `spark.catalog.currentDatabase` +message CurrentDatabase { } + +// See `spark.catalog.setCurrentDatabase` +message SetCurrentDatabase { + // (Required) + string db_name = 1; +} + +// See `spark.catalog.listDatabases` +message ListDatabases { + // (Optional) The pattern that the database name needs to match + optional string pattern = 1; +} + +// See `spark.catalog.listTables` +message ListTables { + // (Optional) + optional string db_name = 1; + // (Optional) The pattern that the table name needs to match + optional string pattern = 2; +} + +// See `spark.catalog.listFunctions` +message ListFunctions { + // (Optional) + optional string db_name = 1; + // (Optional) The pattern that the function name needs to match + optional string pattern = 2; +} + +// See `spark.catalog.listColumns` +message ListColumns { + // (Required) + string table_name = 1; + // (Optional) + optional string db_name = 2; +} + +// See `spark.catalog.getDatabase` +message GetDatabase { + // (Required) + string db_name = 1; +} + +// See `spark.catalog.getTable` +message GetTable { + // (Required) + string table_name = 1; + // (Optional) + optional string db_name = 2; +} + +// See `spark.catalog.getFunction` +message GetFunction { + // (Required) + string function_name = 1; + // (Optional) + optional string db_name = 2; +} + +// See `spark.catalog.databaseExists` +message DatabaseExists { + // (Required) + string db_name = 1; +} + +// See `spark.catalog.tableExists` +message TableExists { + // (Required) + string table_name = 1; + // (Optional) + optional string db_name = 2; +} + +// See `spark.catalog.functionExists` +message FunctionExists { + // (Required) + string function_name = 1; + // (Optional) + optional string db_name = 2; +} + +// See `spark.catalog.createExternalTable` +message CreateExternalTable { + // (Required) + string table_name = 1; + // (Optional) + optional string path = 2; + // (Optional) + optional string source = 3; + // (Optional) + optional DataType schema = 4; + // Options could be empty for valid data source format. + // The map key is case insensitive. + map options = 5; +} + +// See `spark.catalog.createTable` +message CreateTable { + // (Required) + string table_name = 1; + // (Optional) + optional string path = 2; + // (Optional) + optional string source = 3; + // (Optional) + optional string description = 4; + // (Optional) + optional DataType schema = 5; + // Options could be empty for valid data source format. + // The map key is case insensitive. + map options = 6; +} + +// See `spark.catalog.dropTempView` +message DropTempView { + // (Required) + string view_name = 1; +} + +// See `spark.catalog.dropGlobalTempView` +message DropGlobalTempView { + // (Required) + string view_name = 1; +} + +// See `spark.catalog.recoverPartitions` +message RecoverPartitions { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.isCached` +message IsCached { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.cacheTable` +message CacheTable { + // (Required) + string table_name = 1; + + // (Optional) + optional StorageLevel storage_level = 2; +} + +// See `spark.catalog.uncacheTable` +message UncacheTable { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.clearCache` +message ClearCache { } + +// See `spark.catalog.refreshTable` +message RefreshTable { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.refreshByPath` +message RefreshByPath { + // (Required) + string path = 1; +} + +// See `spark.catalog.currentCatalog` +message CurrentCatalog { } + +// See `spark.catalog.setCurrentCatalog` +message SetCurrentCatalog { + // (Required) + string catalog_name = 1; +} + +// See `spark.catalog.listCatalogs` +message ListCatalogs { + // (Optional) The pattern that the catalog name needs to match + optional string pattern = 1; +} + +// See `spark.catalog.dropTable` +message DropTable { + // (Required) + string table_name = 1; + bool if_exists = 2; + bool purge = 3; +} + +// See `spark.catalog.dropView` +message DropView { + // (Required) + string view_name = 1; + bool if_exists = 2; +} + +// See `spark.catalog.createDatabase` +message CreateDatabase { + // (Required) + string db_name = 1; + bool if_not_exists = 2; + map properties = 3; +} + +// See `spark.catalog.dropDatabase` +message DropDatabase { + // (Required) + string db_name = 1; + bool if_exists = 2; + bool cascade = 3; +} + +// See `spark.catalog.listPartitions` +message ListPartitions { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.listViews` +message ListViews { + // (Optional) + optional string db_name = 1; + // (Optional) The pattern that the view name needs to match + optional string pattern = 2; +} + +// See `spark.catalog.getTableProperties` +message GetTableProperties { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.getCreateTableString` +message GetCreateTableString { + // (Required) + string table_name = 1; + bool as_serde = 2; +} + +// See `spark.catalog.truncateTable` +message TruncateTable { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.analyzeTable` +message AnalyzeTable { + // (Required) + string table_name = 1; + bool no_scan = 2; +} diff --git a/proto/spark/connect/commands.proto b/proto/spark/connect/commands.proto new file mode 100644 index 0000000..dcf5aff --- /dev/null +++ b/proto/spark/connect/commands.proto @@ -0,0 +1,560 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +import "google/protobuf/any.proto"; +import "spark/connect/common.proto"; +import "spark/connect/expressions.proto"; +import "spark/connect/relations.proto"; +import "spark/connect/ml.proto"; +import "spark/connect/pipelines.proto"; + +package spark.connect; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// A [[Command]] is an operation that is executed by the server that does not directly consume or +// produce a relational result. +message Command { + oneof command_type { + CommonInlineUserDefinedFunction register_function = 1; + WriteOperation write_operation = 2; + CreateDataFrameViewCommand create_dataframe_view = 3; + WriteOperationV2 write_operation_v2 = 4; + SqlCommand sql_command = 5; + WriteStreamOperationStart write_stream_operation_start = 6; + StreamingQueryCommand streaming_query_command = 7; + GetResourcesCommand get_resources_command = 8; + StreamingQueryManagerCommand streaming_query_manager_command = 9; + CommonInlineUserDefinedTableFunction register_table_function = 10; + StreamingQueryListenerBusCommand streaming_query_listener_bus_command = 11; + CommonInlineUserDefinedDataSource register_data_source = 12; + CreateResourceProfileCommand create_resource_profile_command = 13; + CheckpointCommand checkpoint_command = 14; + RemoveCachedRemoteRelationCommand remove_cached_remote_relation_command = 15; + MergeIntoTableCommand merge_into_table_command = 16; + MlCommand ml_command = 17; + ExecuteExternalCommand execute_external_command = 18; + PipelineCommand pipeline_command = 19; + + // This field is used to mark extensions to the protocol. When plugins generate arbitrary + // Commands they can add them here. During the planning the correct resolution is done. + google.protobuf.Any extension = 999; + + } +} + +// A SQL Command is used to trigger the eager evaluation of SQL commands in Spark. +// +// When the SQL provide as part of the message is a command it will be immediately evaluated +// and the result will be collected and returned as part of a LocalRelation. If the result is +// not a command, the operation will simply return a SQL Relation. This allows the client to be +// almost oblivious to the server-side behavior. +message SqlCommand { + // (Required) SQL Query. + string sql = 1 [deprecated=true]; + + // (Optional) A map of parameter names to literal expressions. + map args = 2 [deprecated=true]; + + // (Optional) A sequence of literal expressions for positional parameters in the SQL query text. + repeated Expression.Literal pos_args = 3 [deprecated=true]; + + // (Optional) A map of parameter names to expressions. + // It cannot coexist with `pos_arguments`. + map named_arguments = 4 [deprecated=true]; + + // (Optional) A sequence of expressions for positional parameters in the SQL query text. + // It cannot coexist with `named_arguments`. + repeated Expression pos_arguments = 5 [deprecated=true]; + + // (Optional) The relation that this SQL command will be built on. + Relation input = 6; +} + +// A command that can create DataFrame global temp view or local temp view. +message CreateDataFrameViewCommand { + // (Required) The relation that this view will be built on. + Relation input = 1; + + // (Required) View name. + string name = 2; + + // (Required) Whether this is global temp view or local temp view. + bool is_global = 3; + + // (Required) + // + // If true, and if the view already exists, updates it; if false, and if the view + // already exists, throws exception. + bool replace = 4; +} + +// As writes are not directly handled during analysis and planning, they are modeled as commands. +message WriteOperation { + // (Required) The output of the `input` relation will be persisted according to the options. + Relation input = 1; + + // (Optional) Format value according to the Spark documentation. Examples are: text, parquet, delta. + optional string source = 2; + + // (Optional) + // + // The destination of the write operation can be either a path or a table. + // If the destination is neither a path nor a table, such as jdbc and noop, + // the `save_type` should not be set. + oneof save_type { + string path = 3; + SaveTable table = 4; + } + + // (Required) the save mode. + SaveMode mode = 5; + + // (Optional) List of columns to sort the output by. + repeated string sort_column_names = 6; + + // (Optional) List of columns for partitioning. + repeated string partitioning_columns = 7; + + // (Optional) Bucketing specification. Bucketing must set the number of buckets and the columns + // to bucket by. + BucketBy bucket_by = 8; + + // (Optional) A list of configuration options. + map options = 9; + + // (Optional) Columns used for clustering the table. + repeated string clustering_columns = 10; + + // (Optional) Whether schema evolution is enabled for the write. + bool with_schema_evolution = 11; + + message SaveTable { + // (Required) The table name. + string table_name = 1; + // (Required) The method to be called to write to the table. + TableSaveMethod save_method = 2; + + enum TableSaveMethod { + TABLE_SAVE_METHOD_UNSPECIFIED = 0; + TABLE_SAVE_METHOD_SAVE_AS_TABLE = 1; + TABLE_SAVE_METHOD_INSERT_INTO = 2; + } + } + + message BucketBy { + repeated string bucket_column_names = 1; + int32 num_buckets = 2; + } + + enum SaveMode { + SAVE_MODE_UNSPECIFIED = 0; + SAVE_MODE_APPEND = 1; + SAVE_MODE_OVERWRITE = 2; + SAVE_MODE_ERROR_IF_EXISTS = 3; + SAVE_MODE_IGNORE = 4; + } +} + +// As writes are not directly handled during analysis and planning, they are modeled as commands. +message WriteOperationV2 { + // (Required) The output of the `input` relation will be persisted according to the options. + Relation input = 1; + + // (Required) The destination of the write operation must be either a path or a table. + string table_name = 2; + + // (Optional) A provider for the underlying output data source. Spark's default catalog supports + // "parquet", "json", etc. + optional string provider = 3; + + // (Optional) List of columns for partitioning for output table created by `create`, + // `createOrReplace`, or `replace` + repeated Expression partitioning_columns = 4; + + // (Optional) A list of configuration options. + map options = 5; + + // (Optional) A list of table properties. + map table_properties = 6; + + // (Required) Write mode. + Mode mode = 7; + + enum Mode { + MODE_UNSPECIFIED = 0; + MODE_CREATE = 1; + MODE_OVERWRITE = 2; + MODE_OVERWRITE_PARTITIONS = 3; + MODE_APPEND = 4; + MODE_REPLACE = 5; + MODE_CREATE_OR_REPLACE = 6; + } + + // (Optional) A condition for overwrite saving mode + Expression overwrite_condition = 8; + + // (Optional) Columns used for clustering the table. + repeated string clustering_columns = 9; + + // (Optional) Whether schema evolution is enabled for the write. + bool with_schema_evolution = 10; +} + +// Starts write stream operation as streaming query. Query ID and Run ID of the streaming +// query are returned. +message WriteStreamOperationStart { + + // (Required) The output of the `input` streaming relation will be written. + Relation input = 1; + + // The following fields directly map to API for DataStreamWriter(). + // Consult API documentation unless explicitly documented here. + + string format = 2; + map options = 3; + repeated string partitioning_column_names = 4; + + oneof trigger { + string processing_time_interval = 5; + bool available_now = 6; + bool once = 7; + string continuous_checkpoint_interval = 8; + string real_time_batch_duration = 100; + } + + string output_mode = 9; + string query_name = 10; + + // The destination is optional. When set, it can be a path or a table name. + oneof sink_destination { + string path = 11; + string table_name = 12; + } + + StreamingForeachFunction foreach_writer = 13; + StreamingForeachFunction foreach_batch = 14; + + // (Optional) Columns used for clustering the table. + repeated string clustering_column_names = 15; +} + +message StreamingForeachFunction { + oneof function { + PythonUDF python_function = 1; + ScalarScalaUDF scala_function = 2; + } +} + +message WriteStreamOperationStartResult { + + // (Required) Query instance. See `StreamingQueryInstanceId`. + StreamingQueryInstanceId query_id = 1; + + // An optional query name. + string name = 2; + + // Optional query started event if there is any listener registered on the client side. + optional string query_started_event_json = 3; + + // TODO: How do we indicate errors? + // TODO: Consider adding status, last progress etc here. +} + +// A tuple that uniquely identifies an instance of streaming query run. It consists of `id` that +// persists across the streaming runs and `run_id` that changes between each run of the +// streaming query that resumes from the checkpoint. +message StreamingQueryInstanceId { + + // (Required) The unique id of this query that persists across restarts from checkpoint data. + // That is, this id is generated when a query is started for the first time, and + // will be the same every time it is restarted from checkpoint data. + string id = 1; + + // (Required) The unique id of this run of the query. That is, every start/restart of a query + // will generate a unique run_id. Therefore, every time a query is restarted from + // checkpoint, it will have the same `id` but different `run_id`s. + string run_id = 2; +} + +// Commands for a streaming query. +message StreamingQueryCommand { + + // (Required) Query instance. See `StreamingQueryInstanceId`. + StreamingQueryInstanceId query_id = 1; + + // See documentation for the corresponding API method in StreamingQuery. + oneof command { + // status() API. + bool status = 2; + // lastProgress() API. + bool last_progress = 3; + // recentProgress() API. + bool recent_progress = 4; + // stop() API. Stops the query. + bool stop = 5; + // processAllAvailable() API. Waits till all the available data is processed + bool process_all_available = 6; + // explain() API. Returns logical and physical plans. + ExplainCommand explain = 7; + // exception() API. Returns the exception in the query if any. + bool exception = 8; + // awaitTermination() API. Waits for the termination of the query. + AwaitTerminationCommand await_termination = 9; + } + + message ExplainCommand { + // TODO: Consider reusing Explain from AnalyzePlanRequest message. + // We can not do this right now since it base.proto imports this file. + bool extended = 1; + } + + message AwaitTerminationCommand { + optional int64 timeout_ms = 2; + } +} + +// Response for commands on a streaming query. +message StreamingQueryCommandResult { + // (Required) Query instance id. See `StreamingQueryInstanceId`. + StreamingQueryInstanceId query_id = 1; + + oneof result_type { + StatusResult status = 2; + RecentProgressResult recent_progress = 3; + ExplainResult explain = 4; + ExceptionResult exception = 5; + AwaitTerminationResult await_termination = 6; + } + + message StatusResult { + // See documentation for these Scala 'StreamingQueryStatus' struct + string status_message = 1; + bool is_data_available = 2; + bool is_trigger_active = 3; + bool is_active = 4; + } + + message RecentProgressResult { + // Progress reports as an array of json strings. + repeated string recent_progress_json = 5; + } + + message ExplainResult { + // Logical and physical plans as string + string result = 1; + } + + message ExceptionResult { + // (Optional) Exception message as string, maps to the return value of original + // StreamingQueryException's toString method + optional string exception_message = 1; + // (Optional) Exception error class as string + optional string error_class = 2; + // (Optional) Exception stack trace as string + optional string stack_trace = 3; + } + + message AwaitTerminationResult { + bool terminated = 1; + } +} + +// Commands for the streaming query manager. +message StreamingQueryManagerCommand { + + // See documentation for the corresponding API method in StreamingQueryManager. + oneof command { + // active() API, returns a list of active queries. + bool active = 1; + // get() API, returns the StreamingQuery identified by id. + string get_query = 2; + // awaitAnyTermination() API, wait until any query terminates or timeout. + AwaitAnyTerminationCommand await_any_termination = 3; + // resetTerminated() API. + bool reset_terminated = 4; + // addListener API. + StreamingQueryListenerCommand add_listener = 5; + // removeListener API. + StreamingQueryListenerCommand remove_listener = 6; + // listListeners() API, returns a list of streaming query listeners. + bool list_listeners = 7; + } + + message AwaitAnyTerminationCommand { + // (Optional) The waiting time in milliseconds to wait for any query to terminate. + optional int64 timeout_ms = 1; + } + + message StreamingQueryListenerCommand { + bytes listener_payload = 1; + optional PythonUDF python_listener_payload = 2; + string id = 3; + } +} + +// Response for commands on the streaming query manager. +message StreamingQueryManagerCommandResult { + oneof result_type { + ActiveResult active = 1; + StreamingQueryInstance query = 2; + AwaitAnyTerminationResult await_any_termination = 3; + bool reset_terminated = 4; + bool add_listener = 5; + bool remove_listener = 6; + ListStreamingQueryListenerResult list_listeners = 7; + } + + message ActiveResult { + repeated StreamingQueryInstance active_queries = 1; + } + + message StreamingQueryInstance { + // (Required) The id and runId of this query. + StreamingQueryInstanceId id = 1; + // (Optional) The name of this query. + optional string name = 2; + } + + message AwaitAnyTerminationResult { + bool terminated = 1; + } + + message StreamingQueryListenerInstance { + bytes listener_payload = 1; + } + + message ListStreamingQueryListenerResult { + // (Required) Reference IDs of listener instances. + repeated string listener_ids = 1; + } +} + +// The protocol for client-side StreamingQueryListener. +// This command will only be set when either the first listener is added to the client, or the last +// listener is removed from the client. +// The add_listener_bus_listener command will only be set true in the first case. +// The remove_listener_bus_listener command will only be set true in the second case. +message StreamingQueryListenerBusCommand { + oneof command { + bool add_listener_bus_listener = 1; + bool remove_listener_bus_listener = 2; + } +} + +// The enum used for client side streaming query listener event +// There is no QueryStartedEvent defined here, +// it is added as a field in WriteStreamOperationStartResult +enum StreamingQueryEventType { + QUERY_PROGRESS_UNSPECIFIED = 0; + QUERY_PROGRESS_EVENT = 1; + QUERY_TERMINATED_EVENT = 2; + QUERY_IDLE_EVENT = 3; +} + +// The protocol for the returned events in the long-running response channel. +message StreamingQueryListenerEvent { + // (Required) The json serialized event, all StreamingQueryListener events have a json method + string event_json = 1; + // (Required) Query event type used by client to decide how to deserialize the event_json + StreamingQueryEventType event_type = 2; +} + +message StreamingQueryListenerEventsResult { + repeated StreamingQueryListenerEvent events = 1; + optional bool listener_bus_listener_added = 2; +} + +// Command to get the output of 'SparkContext.resources' +message GetResourcesCommand { } + +// Response for command 'GetResourcesCommand'. +message GetResourcesCommandResult { + map resources = 1; +} + +// Command to create ResourceProfile +message CreateResourceProfileCommand { + // (Required) The ResourceProfile to be built on the server-side. + ResourceProfile profile = 1; +} + +// Response for command 'CreateResourceProfileCommand'. +message CreateResourceProfileCommandResult { + // (Required) Server-side generated resource profile id. + int32 profile_id = 1; +} + +// Command to remove `CashedRemoteRelation` +message RemoveCachedRemoteRelationCommand { + // (Required) The remote to be related + CachedRemoteRelation relation = 1; +} + +message CheckpointCommand { + // (Required) The logical plan to checkpoint. + Relation relation = 1; + + // (Required) Locally checkpoint using a local temporary + // directory in Spark Connect server (Spark Driver) + bool local = 2; + + // (Required) Whether to checkpoint this dataframe immediately. + bool eager = 3; + + // (Optional) For local checkpoint, the storage level to use. + optional StorageLevel storage_level = 4; +} + +message MergeIntoTableCommand { + // (Required) The name of the target table. + string target_table_name = 1; + + // (Required) The relation of the source table. + Relation source_table_plan = 2; + + // (Required) The condition to match the source and target. + Expression merge_condition = 3; + + // (Optional) The actions to be taken when the condition is matched. + repeated Expression match_actions = 4; + + // (Optional) The actions to be taken when the condition is not matched. + repeated Expression not_matched_actions = 5; + + // (Optional) The actions to be taken when the condition is not matched by source. + repeated Expression not_matched_by_source_actions = 6; + + // (Required) Whether to enable schema evolution. + bool with_schema_evolution = 7; +} + +// Execute an arbitrary string command inside an external execution engine +message ExecuteExternalCommand { + // (Required) The class name of the runner that implements `ExternalCommandRunner` + string runner = 1; + + // (Required) The target command to be executed. + string command = 2; + + // (Optional) The options for the runner. + map options = 3; +} diff --git a/proto/spark/connect/common.proto b/proto/spark/connect/common.proto new file mode 100644 index 0000000..c547053 --- /dev/null +++ b/proto/spark/connect/common.proto @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// StorageLevel for persisting Datasets/Tables. +message StorageLevel { + // (Required) Whether the cache should use disk or not. + bool use_disk = 1; + // (Required) Whether the cache should use memory or not. + bool use_memory = 2; + // (Required) Whether the cache should use off-heap or not. + bool use_off_heap = 3; + // (Required) Whether the cached data is deserialized or not. + bool deserialized = 4; + // (Required) The number of replicas. + int32 replication = 5; +} + + +// ResourceInformation to hold information about a type of Resource. +// The corresponding class is 'org.apache.spark.resource.ResourceInformation' +message ResourceInformation { + // (Required) The name of the resource + string name = 1; + // (Required) An array of strings describing the addresses of the resource. + repeated string addresses = 2; +} + +// An executor resource request. +message ExecutorResourceRequest { + // (Required) resource name. + string resource_name = 1; + + // (Required) resource amount requesting. + int64 amount = 2; + + // Optional script used to discover the resources. + optional string discovery_script = 3; + + // Optional vendor, required for some cluster managers. + optional string vendor = 4; +} + +// A task resource request. +message TaskResourceRequest { + // (Required) resource name. + string resource_name = 1; + + // (Required) resource amount requesting as a double to support fractional + // resource requests. + double amount = 2; +} + +message ResourceProfile { + // (Optional) Resource requests for executors. Mapped from the resource name + // (e.g., cores, memory, CPU) to its specific request. + map executor_resources = 1; + + // (Optional) Resource requests for tasks. Mapped from the resource name + // (e.g., cores, memory, CPU) to its specific request. + map task_resources = 2; +} + +message Origin { + // (Required) Indicate the origin type. + oneof function { + PythonOrigin python_origin = 1; + JvmOrigin jvm_origin = 2; + } +} + +message PythonOrigin { + // (Required) Name of the origin, for example, the name of the function + string fragment = 1; + + // (Required) Callsite to show to end users, for example, stacktrace. + string call_site = 2; +} + +message JvmOrigin { + // (Optional) Line number in the source file. + optional int32 line = 1; + + // (Optional) Start position in the source file. + optional int32 start_position = 2; + + // (Optional) Start index in the source file. + optional int32 start_index = 3; + + // (Optional) Stop index in the source file. + optional int32 stop_index = 4; + + // (Optional) SQL text. + optional string sql_text = 5; + + // (Optional) Object type. + optional string object_type = 6; + + // (Optional) Object name. + optional string object_name = 7; + + // (Optional) Stack trace. + repeated StackTraceElement stack_trace = 8; +} + +// A message to hold a [[java.lang.StackTraceElement]]. +message StackTraceElement { + // (Optional) Class loader name + optional string class_loader_name = 1; + + // (Optional) Module name + optional string module_name = 2; + + // (Optional) Module version + optional string module_version = 3; + + // (Required) Declaring class + string declaring_class = 4; + + // (Required) Method name + string method_name = 5; + + // (Optional) File name + optional string file_name = 6; + + // (Required) Line number + int32 line_number = 7; +} + +message ResolvedIdentifier { + string catalog_name = 1; + repeated string namespace = 2; + string table_name = 3; +} + +message Bools { + repeated bool values = 1; +} + +message Ints { + repeated int32 values = 1; +} + +message Longs { + repeated int64 values = 1; +} + +message Floats { + repeated float values = 1; +} + +message Doubles { + repeated double values = 1; +} + +message Strings { + repeated string values = 1; +} diff --git a/proto/spark/connect/example_plugins.proto b/proto/spark/connect/example_plugins.proto new file mode 100644 index 0000000..7ad171d --- /dev/null +++ b/proto/spark/connect/example_plugins.proto @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +import "spark/connect/relations.proto"; +import "spark/connect/expressions.proto"; +option go_package = "internal/generated"; + +package spark.connect; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; + +message ExamplePluginRelation { + Relation input = 1; + string custom_field = 2; + +} + +message ExamplePluginExpression { + Expression child = 1; + string custom_field = 2; +} + +message ExamplePluginCommand { + string custom_field = 1; +} \ No newline at end of file diff --git a/proto/spark/connect/expressions.proto b/proto/spark/connect/expressions.proto new file mode 100644 index 0000000..f74c5af --- /dev/null +++ b/proto/spark/connect/expressions.proto @@ -0,0 +1,557 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +import "google/protobuf/any.proto"; +import "spark/connect/types.proto"; +import "spark/connect/common.proto"; + +package spark.connect; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// Expression used to refer to fields, functions and similar. This can be used everywhere +// expressions in SQL appear. +message Expression { + + ExpressionCommon common = 18; + oneof expr_type { + Literal literal = 1; + UnresolvedAttribute unresolved_attribute = 2; + UnresolvedFunction unresolved_function = 3; + ExpressionString expression_string = 4; + UnresolvedStar unresolved_star = 5; + Alias alias = 6; + Cast cast = 7; + UnresolvedRegex unresolved_regex = 8; + SortOrder sort_order = 9; + LambdaFunction lambda_function = 10; + Window window = 11; + UnresolvedExtractValue unresolved_extract_value = 12; + UpdateFields update_fields = 13; + UnresolvedNamedLambdaVariable unresolved_named_lambda_variable = 14; + CommonInlineUserDefinedFunction common_inline_user_defined_function = 15; + CallFunction call_function = 16; + NamedArgumentExpression named_argument_expression = 17; + MergeAction merge_action = 19; + TypedAggregateExpression typed_aggregate_expression = 20; + SubqueryExpression subquery_expression = 21; + DirectShufflePartitionID direct_shuffle_partition_id = 22; + + // This field is used to mark extensions to the protocol. When plugins generate arbitrary + // relations they can add them here. During the planning the correct resolution is done. + google.protobuf.Any extension = 999; + } + + + // Expression for the OVER clause or WINDOW clause. + message Window { + + // (Required) The window function. + Expression window_function = 1; + + // (Optional) The way that input rows are partitioned. + repeated Expression partition_spec = 2; + + // (Optional) Ordering of rows in a partition. + repeated SortOrder order_spec = 3; + + // (Optional) Window frame in a partition. + // + // If not set, it will be treated as 'UnspecifiedFrame'. + WindowFrame frame_spec = 4; + + // The window frame + message WindowFrame { + + // (Required) The type of the frame. + FrameType frame_type = 1; + + // (Required) The lower bound of the frame. + FrameBoundary lower = 2; + + // (Required) The upper bound of the frame. + FrameBoundary upper = 3; + + enum FrameType { + FRAME_TYPE_UNDEFINED = 0; + + // RowFrame treats rows in a partition individually. + FRAME_TYPE_ROW = 1; + + // RangeFrame treats rows in a partition as groups of peers. + // All rows having the same 'ORDER BY' ordering are considered as peers. + FRAME_TYPE_RANGE = 2; + } + + message FrameBoundary { + oneof boundary { + // CURRENT ROW boundary + bool current_row = 1; + + // UNBOUNDED boundary. + // For lower bound, it will be converted to 'UnboundedPreceding'. + // for upper bound, it will be converted to 'UnboundedFollowing'. + bool unbounded = 2; + + // This is an expression for future proofing. We are expecting literals on the server side. + Expression value = 3; + } + } + } + } + + // SortOrder is used to specify the data ordering, it is normally used in Sort and Window. + // It is an unevaluable expression and cannot be evaluated, so can not be used in Projection. + message SortOrder { + // (Required) The expression to be sorted. + Expression child = 1; + + // (Required) The sort direction, should be ASCENDING or DESCENDING. + SortDirection direction = 2; + + // (Required) How to deal with NULLs, should be NULLS_FIRST or NULLS_LAST. + NullOrdering null_ordering = 3; + + enum SortDirection { + SORT_DIRECTION_UNSPECIFIED = 0; + SORT_DIRECTION_ASCENDING = 1; + SORT_DIRECTION_DESCENDING = 2; + } + + enum NullOrdering { + SORT_NULLS_UNSPECIFIED = 0; + SORT_NULLS_FIRST = 1; + SORT_NULLS_LAST = 2; + } + } + + // Expression that takes a partition ID value and passes it through directly for use in + // shuffle partitioning. This is used with RepartitionByExpression to allow users to + // directly specify target partition IDs. + message DirectShufflePartitionID { + // (Required) The expression that evaluates to the partition ID. + Expression child = 1; + } + + message Cast { + // (Required) the expression to be casted. + Expression expr = 1; + + // (Required) the data type that the expr to be casted to. + oneof cast_to_type { + DataType type = 2; + // If this is set, Server will use Catalyst parser to parse this string to DataType. + string type_str = 3; + } + + // (Optional) The expression evaluation mode. + EvalMode eval_mode = 4; + + enum EvalMode { + EVAL_MODE_UNSPECIFIED = 0; + EVAL_MODE_LEGACY = 1; + EVAL_MODE_ANSI = 2; + EVAL_MODE_TRY = 3; + } + } + + message Literal { + oneof literal_type { + DataType null = 1; + bytes binary = 2; + bool boolean = 3; + + int32 byte = 4; + int32 short = 5; + int32 integer = 6; + int64 long = 7; + float float = 10; + double double = 11; + Decimal decimal = 12; + + string string = 13; + + // Date in units of days since the UNIX epoch. + int32 date = 16; + // Timestamp in units of microseconds since the UNIX epoch. + int64 timestamp = 17; + // Timestamp in units of microseconds since the UNIX epoch (without timezone information). + int64 timestamp_ntz = 18; + + CalendarInterval calendar_interval = 19; + int32 year_month_interval = 20; + int64 day_time_interval = 21; + Array array = 22; + Map map = 23; + Struct struct = 24; + + SpecializedArray specialized_array = 25; + Time time = 26; + } + + // Reserved for Geometry and Geography. + reserved 27, 28; + + // Data type information for the literal. + // This field is required only in the root literal message for null values or + // for data types (e.g., array, map, or struct) with non-trivial information. + // If the data_type field is not set at the root level, the data type will be + // inferred or retrieved from the deprecated data type fields using best efforts. + DataType data_type = 100; + + message Decimal { + // the string representation. + string value = 1; + // The maximum number of digits allowed in the value. + // the maximum precision is 38. + optional int32 precision = 2; + // declared scale of decimal literal + optional int32 scale = 3; + } + + message CalendarInterval { + int32 months = 1; + int32 days = 2; + int64 microseconds = 3; + } + + message Array { + // (Deprecated) The element type of the array. + // + // This field is deprecated since Spark 4.1+. Use data_type field instead. + DataType element_type = 1 [deprecated = true]; + + // The literal values that make up the array elements. + repeated Literal elements = 2; + } + + message Map { + // (Deprecated) The key type of the map. + // + // This field is deprecated since Spark 4.1+. Use data_type field instead. + DataType key_type = 1 [deprecated = true]; + + // (Deprecated) The value type of the map. + // + // This field is deprecated since Spark 4.1+ and should only be set + // if the data_type field is not set. Use data_type field instead. + DataType value_type = 2 [deprecated = true]; + + // The literal keys that make up the map. + repeated Literal keys = 3; + + // The literal values that make up the map. + repeated Literal values = 4; + } + + message Struct { + // (Deprecated) The type of the struct. + // + // This field is deprecated since Spark 4.1+ because using DataType as the type of a struct + // is ambiguous. Use data_type field instead. + DataType struct_type = 1 [deprecated = true]; + + // The literal values that make up the struct elements. + repeated Literal elements = 2; + } + + message SpecializedArray { + oneof value_type { + Bools bools = 1; + Ints ints = 2; + Longs longs = 3; + Floats floats = 4; + Doubles doubles = 5; + Strings strings = 6; + } + } + + message Time { + int64 nano = 1; + // The precision of this time, if omitted, uses the default value of MICROS_PRECISION. + optional int32 precision = 2; + } + } + + // An unresolved attribute that is not explicitly bound to a specific column, but the column + // is resolved during analysis by name. + message UnresolvedAttribute { + // (Required) An identifier that will be parsed by Catalyst parser. This should follow the + // Spark SQL identifier syntax. + string unparsed_identifier = 1; + + // (Optional) The id of corresponding connect plan. + optional int64 plan_id = 2; + + // (Optional) The requested column is a metadata column. + optional bool is_metadata_column = 3; + } + + // An unresolved function is not explicitly bound to one explicit function, but the function + // is resolved during analysis following Sparks name resolution rules. + message UnresolvedFunction { + // (Required) name (or unparsed name for user defined function) for the unresolved function. + string function_name = 1; + + // (Optional) Function arguments. Empty arguments are allowed. + repeated Expression arguments = 2; + + // (Required) Indicate if this function should be applied on distinct values. + bool is_distinct = 3; + + // (Required) Indicate if this is a user defined function. + // + // When it is not a user defined function, Connect will use the function name directly. + // When it is a user defined function, Connect will parse the function name first. + bool is_user_defined_function = 4; + + // (Optional) Indicate if this function is defined in the internal function registry. + // If not set, the server will try to look up the function in the internal function registry + // and decide appropriately. + optional bool is_internal = 5; + } + + // Expression as string. + message ExpressionString { + // (Required) A SQL expression that will be parsed by Catalyst parser. + string expression = 1; + } + + // UnresolvedStar is used to expand all the fields of a relation or struct. + message UnresolvedStar { + + // (Optional) The target of the expansion. + // + // If set, it should end with '.*' and will be parsed by 'parseAttributeName' + // in the server side. + optional string unparsed_target = 1; + + // (Optional) The id of corresponding connect plan. + optional int64 plan_id = 2; + } + + // Represents all of the input attributes to a given relational operator, for example in + // "SELECT `(id)?+.+` FROM ...". + message UnresolvedRegex { + // (Required) The column name used to extract column with regex. + string col_name = 1; + + // (Optional) The id of corresponding connect plan. + optional int64 plan_id = 2; + } + + // Extracts a value or values from an Expression + message UnresolvedExtractValue { + // (Required) The expression to extract value from, can be + // Map, Array, Struct or array of Structs. + Expression child = 1; + + // (Required) The expression to describe the extraction, can be + // key of Map, index of Array, field name of Struct. + Expression extraction = 2; + } + + // Add, replace or drop a field of `StructType` expression by name. + message UpdateFields { + // (Required) The struct expression. + Expression struct_expression = 1; + + // (Required) The field name. + string field_name = 2; + + // (Optional) The expression to add or replace. + // + // When not set, it means this field will be dropped. + Expression value_expression = 3; + } + + message Alias { + // (Required) The expression that alias will be added on. + Expression expr = 1; + + // (Required) a list of name parts for the alias. + // + // Scalar columns only has one name that presents. + repeated string name = 2; + + // (Optional) Alias metadata expressed as a JSON map. + optional string metadata = 3; + } + + message LambdaFunction { + // (Required) The lambda function. + // + // The function body should use 'UnresolvedAttribute' as arguments, the sever side will + // replace 'UnresolvedAttribute' with 'UnresolvedNamedLambdaVariable'. + Expression function = 1; + + // (Required) Function variables. Must contains 1 ~ 3 variables. + repeated Expression.UnresolvedNamedLambdaVariable arguments = 2; + } + + message UnresolvedNamedLambdaVariable { + + // (Required) a list of name parts for the variable. Must not be empty. + repeated string name_parts = 1; + } +} + +message ExpressionCommon { + // (Required) Keep the information of the origin for this expression such as stacktrace. + Origin origin = 1; +} + +message CommonInlineUserDefinedFunction { + // (Required) Name of the user-defined function. + string function_name = 1; + // (Optional) Indicate if the user-defined function is deterministic. + bool deterministic = 2; + // (Optional) Function arguments. Empty arguments are allowed. + repeated Expression arguments = 3; + // (Required) Indicate the function type of the user-defined function. + oneof function { + PythonUDF python_udf = 4; + ScalarScalaUDF scalar_scala_udf = 5; + JavaUDF java_udf = 6; + } + // (Required) Indicate if this function should be applied on distinct values. + bool is_distinct = 7; +} + +message PythonUDF { + // (Required) Output type of the Python UDF + DataType output_type = 1; + // (Required) EvalType of the Python UDF + int32 eval_type = 2; + // (Required) The encoded commands of the Python UDF + bytes command = 3; + // (Required) Python version being used in the client. + string python_ver = 4; + // (Optional) Additional includes for the Python UDF. + repeated string additional_includes = 5; +} + +message ScalarScalaUDF { + // (Required) Serialized JVM object containing UDF definition, input encoders and output encoder + bytes payload = 1; + // (Optional) Input type(s) of the UDF + repeated DataType inputTypes = 2; + // (Required) Output type of the UDF + DataType outputType = 3; + // (Required) True if the UDF can return null value + bool nullable = 4; + // (Required) Indicate if the UDF is an aggregate function + bool aggregate = 5; +} + +message JavaUDF { + // (Required) Fully qualified name of Java class + string class_name = 1; + + // (Optional) Output type of the Java UDF + optional DataType output_type = 2; + + // (Required) Indicate if the Java user-defined function is an aggregate function + bool aggregate = 3; +} + +message TypedAggregateExpression { + // (Required) The aggregate function object packed into bytes. + ScalarScalaUDF scalar_scala_udf = 1; +} + +message CallFunction { + // (Required) Unparsed name of the SQL function. + string function_name = 1; + + // (Optional) Function arguments. Empty arguments are allowed. + repeated Expression arguments = 2; +} + +message NamedArgumentExpression { + // (Required) The key of the named argument. + string key = 1; + + // (Required) The value expression of the named argument. + Expression value = 2; +} + +message MergeAction { + // (Required) The action type of the merge action. + ActionType action_type = 1; + + // (Optional) The condition expression of the merge action. + optional Expression condition = 2; + + // (Optional) The assignments of the merge action. Required for ActionTypes INSERT and UPDATE. + repeated Assignment assignments = 3; + + enum ActionType { + ACTION_TYPE_INVALID = 0; + ACTION_TYPE_DELETE = 1; + ACTION_TYPE_INSERT = 2; + ACTION_TYPE_INSERT_STAR = 3; + ACTION_TYPE_UPDATE = 4; + ACTION_TYPE_UPDATE_STAR = 5; + } + + message Assignment { + // (Required) The key of the assignment. + Expression key = 1; + + // (Required) The value of the assignment. + Expression value = 2; + } +} + +message SubqueryExpression { + // (Required) The ID of the corresponding connect plan. + int64 plan_id = 1; + + // (Required) The type of the subquery. + SubqueryType subquery_type = 2; + + // (Optional) Options specific to table arguments. + optional TableArgOptions table_arg_options = 3; + + // (Optional) IN subquery values. + repeated Expression in_subquery_values = 4; + + enum SubqueryType { + SUBQUERY_TYPE_UNKNOWN = 0; + SUBQUERY_TYPE_SCALAR = 1; + SUBQUERY_TYPE_EXISTS = 2; + SUBQUERY_TYPE_TABLE_ARG = 3; + SUBQUERY_TYPE_IN = 4; + } + + // Nested message for table argument options. + message TableArgOptions { + // (Optional) The way that input rows are partitioned. + repeated Expression partition_spec = 1; + + // (Optional) Ordering of rows in a partition. + repeated Expression.SortOrder order_spec = 2; + + // (Optional) Whether this is a single partition. + optional bool with_single_partition = 3; + } +} diff --git a/proto/spark/connect/ml.proto b/proto/spark/connect/ml.proto new file mode 100644 index 0000000..ef5c406 --- /dev/null +++ b/proto/spark/connect/ml.proto @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "spark/connect/relations.proto"; +import "spark/connect/expressions.proto"; +import "spark/connect/ml_common.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// Command for ML +message MlCommand { + oneof command { + Fit fit = 1; + Fetch fetch = 2; + Delete delete = 3; + Write write = 4; + Read read = 5; + Evaluate evaluate = 6; + CleanCache clean_cache = 7; + GetCacheInfo get_cache_info = 8; + CreateSummary create_summary = 9; + GetModelSize get_model_size = 10; + } + + // Command for estimator.fit(dataset) + message Fit { + // (Required) Estimator information (its type should be OPERATOR_TYPE_ESTIMATOR) + MlOperator estimator = 1; + // (Optional) parameters of the Estimator + optional MlParams params = 2; + // (Required) the training dataset + Relation dataset = 3; + } + + // Command to delete the cached objects which could be a model + // or summary evaluated by a model + message Delete { + repeated ObjectRef obj_refs = 1; + // if set `evict_only` to true, only evict the cached model from memory, + // but keep the offloaded model in Spark driver local disk. + optional bool evict_only = 2; + } + + // Force to clean up all the ML cached objects + message CleanCache { } + + // Get the information of all the ML cached objects + message GetCacheInfo { } + + // Command to write ML operator + message Write { + // It could be an estimator/evaluator or the cached model + oneof type { + // Estimator or evaluator + MlOperator operator = 1; + // The cached model + ObjectRef obj_ref = 2; + } + // (Optional) The parameters of operator which could be estimator/evaluator or a cached model + optional MlParams params = 3; + // (Required) Save the ML instance to the path + string path = 4; + // (Optional) Overwrites if the output path already exists. + optional bool should_overwrite = 5; + // (Optional) The options of the writer + map options = 6; + } + + // Command to load ML operator. + message Read { + // (Required) ML operator information + MlOperator operator = 1; + // (Required) Load the ML instance from the input path + string path = 2; + } + + // Command for evaluator.evaluate(dataset) + message Evaluate { + // (Required) Evaluator information (its type should be OPERATOR_TYPE_EVALUATOR) + MlOperator evaluator = 1; + // (Optional) parameters of the Evaluator + optional MlParams params = 2; + // (Required) the evaluating dataset + Relation dataset = 3; + } + + // This is for re-creating the model summary when the model summary is lost + // (model summary is lost when the model is offloaded and then loaded back) + message CreateSummary { + ObjectRef model_ref = 1; + Relation dataset = 2; + } + + // This is for query the model estimated in-memory size + message GetModelSize { + ObjectRef model_ref = 1; + } +} + +// The result of MlCommand +message MlCommandResult { + oneof result_type { + // The result of the attribute + Expression.Literal param = 1; + // Evaluate a Dataset in a model and return the cached ID of summary + string summary = 2; + // Operator information + MlOperatorInfo operator_info = 3; + } + + // Represents an operator info + message MlOperatorInfo { + oneof type { + // The cached object which could be a model or summary evaluated by a model + ObjectRef obj_ref = 1; + // Operator name + string name = 2; + } + // (Optional) the 'uid' of a ML object + // Note it is different from the 'id' of a cached object. + optional string uid = 3; + // (Optional) parameters + optional MlParams params = 4; + // (Optional) warning message generated during the ML command execution + optional string warning_message = 5; + } +} diff --git a/proto/spark/connect/ml_common.proto b/proto/spark/connect/ml_common.proto new file mode 100644 index 0000000..06ca4e5 --- /dev/null +++ b/proto/spark/connect/ml_common.proto @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "spark/connect/expressions.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// MlParams stores param settings for ML Estimator / Transformer / Evaluator +message MlParams { + // User-supplied params + map params = 1; +} + +// MLOperator represents the ML operators like (Estimator, Transformer or Evaluator) +message MlOperator { + // (Required) The qualified name of the ML operator. + string name = 1; + + // (Required) Unique id of the ML operator + string uid = 2; + + // (Required) Represents what the ML operator is + OperatorType type = 3; + + enum OperatorType { + OPERATOR_TYPE_UNSPECIFIED = 0; + // ML estimator + OPERATOR_TYPE_ESTIMATOR = 1; + // ML transformer (non-model) + OPERATOR_TYPE_TRANSFORMER = 2; + // ML evaluator + OPERATOR_TYPE_EVALUATOR = 3; + // ML model + OPERATOR_TYPE_MODEL = 4; + } +} + +// Represents a reference to the cached object which could be a model +// or summary evaluated by a model +message ObjectRef { + // (Required) The ID is used to lookup the object on the server side. + // Note it is different from the 'uid' of a ML object. + string id = 1; +} diff --git a/proto/spark/connect/pipelines.proto b/proto/spark/connect/pipelines.proto new file mode 100644 index 0000000..6438583 --- /dev/null +++ b/proto/spark/connect/pipelines.proto @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package spark.connect; + +import "google/protobuf/any.proto"; +import "google/protobuf/timestamp.proto"; +import "spark/connect/common.proto"; +import "spark/connect/relations.proto"; +import "spark/connect/types.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// Dispatch object for pipelines commands. See each individual command for documentation. +message PipelineCommand { + oneof command_type { + CreateDataflowGraph create_dataflow_graph = 1; + DefineOutput define_output = 2; + DefineFlow define_flow = 3; + DropDataflowGraph drop_dataflow_graph = 4; + StartRun start_run = 5; + DefineSqlGraphElements define_sql_graph_elements = 6; + GetQueryFunctionExecutionSignalStream get_query_function_execution_signal_stream = 7; + DefineFlowQueryFunctionResult define_flow_query_function_result = 8; + ExecuteOutputFlows execute_output_flows = 9; + + // Reserved field for protocol extensions. + // Used to support forward-compatibility by carrying additional command types + // that are not yet defined in this version of the proto. During planning, the + // engine will resolve and dispatch the concrete command contained in this field. + google.protobuf.Any extension = 999; + } + + // Request to create a new dataflow graph. + message CreateDataflowGraph { + // The default catalog. + optional string default_catalog = 1; + + // The default database. + optional string default_database = 2; + + // SQL configurations for all flows in this graph. + map sql_conf = 5; + } + + // Drops the graph and stops any running attached flows. + message DropDataflowGraph { + // The graph to drop. + optional string dataflow_graph_id = 1; + } + + // Request to define an output: a table, a materialized view, a temporary view or a sink. + message DefineOutput { + // The graph to attach this output to. + optional string dataflow_graph_id = 1; + + // Name of the output. Can be partially or fully qualified. + optional string output_name = 2; + + // The type of the output. + optional OutputType output_type = 3; + + // Optional comment for the output. + optional string comment = 4; + + // The location in source code that this output was defined. + optional SourceCodeLocation source_code_location = 5; + + oneof details { + TableDetails table_details = 6; + SinkDetails sink_details = 7; + google.protobuf.Any extension = 999; + } + + // Metadata that's only applicable to tables and materialized views. + message TableDetails { + // Optional table properties. + map table_properties = 1; + + // Optional partition columns for the table. + repeated string partition_cols = 2; + + // The output table format for the table. + optional string format = 3; + + // Schema for the table. If unset, this will be inferred from incoming flows. + oneof schema { + spark.connect.DataType schema_data_type = 4; + string schema_string = 5; + } + + // Optional cluster columns for the table. + repeated string clustering_columns = 6; + } + + // Metadata that's only applicable to sinks. + message SinkDetails { + // Streaming write options + map options = 1; + + // Streaming write format + optional string format = 2; + } + } + + // Request to define a flow targeting a dataset. + message DefineFlow { + // The graph to attach this flow to. + optional string dataflow_graph_id = 1; + + // Name of the flow. For standalone flows, this must be a single-part name. + optional string flow_name = 2; + + // Name of the dataset this flow writes to. Can be partially or fully qualified. + optional string target_dataset_name = 3; + + // SQL configurations set when running this flow. + map sql_conf = 4; + + // Identifier for the client making the request. The server uses this to determine what flow + // evaluation request stream to dispatch evaluation requests to for this flow. + optional string client_id = 5; + + // The location in source code that this flow was defined. + optional SourceCodeLocation source_code_location = 6; + + oneof details { + WriteRelationFlowDetails relation_flow_details = 7; + google.protobuf.Any extension = 999; + } + + // A flow that is that takes the contents of a relation and writes it to the target dataset. + message WriteRelationFlowDetails { + // An unresolved relation that defines the dataset's flow. Empty if the query function + // that defines the flow cannot be analyzed at the time of flow definition. + optional spark.connect.Relation relation = 1; + } + + // If true, define the flow as a one-time flow, such as for backfill. + // Set to true changes the flow in two ways: + // - The flow is run one time by default. If the pipeline is ran with a full refresh, + // the flow will run again. + // - The flow function must be a batch DataFrame, not a streaming DataFrame. + optional bool once = 8; + + message Response { + // Fully qualified flow name that uniquely identify a flow in the Dataflow graph. + optional string flow_name = 1; + } + } + + // Request to execute all flows for a single output (dataset or sink) remotely. + message ExecuteOutputFlows { + + // The output (table or materialized view or sink) definition. + optional DefineOutput define_output = 1; + + // The flows to execute for this table. + repeated DefineFlow define_flows = 2; + + // Whether to perform a full refresh instead of an incremental update. + optional bool full_refresh = 3; + + // Storage location for pipeline checkpoints and metadata. + optional string storage = 4; + + // Reserved field for protocol extensions. + repeated google.protobuf.Any extension = 999; + } + + // Resolves all datasets and flows and start a pipeline update. Should be called after all + // graph elements are registered. + message StartRun { + // The graph to start. + optional string dataflow_graph_id = 1; + + // List of dataset to reset and recompute. + repeated string full_refresh_selection = 2; + + // Perform a full graph reset and recompute. + optional bool full_refresh_all = 3; + + // List of dataset to update. + repeated string refresh_selection = 4; + + // If true, the run will not actually execute any flows, but will only validate the graph and + // check for any errors. This is useful for testing and validation purposes. + optional bool dry = 5; + + // storage location for pipeline checkpoints and metadata. + optional string storage = 6; + } + + // Parses the SQL file and registers all datasets and flows. + message DefineSqlGraphElements { + // The graph to attach this dataset to. + optional string dataflow_graph_id = 1; + + // The full path to the SQL file. Can be relative or absolute. + optional string sql_file_path = 2; + + // The contents of the SQL file. + optional string sql_text = 3; + } + + // Request to get the stream of query function execution signals for a graph. Responses should + // be a stream of PipelineQueryFunctionExecutionSignal messages. + message GetQueryFunctionExecutionSignalStream { + // The graph to get the query function execution signal stream for. + optional string dataflow_graph_id = 1; + + // Identifier for the client that is requesting the stream. + optional string client_id = 2; + } + + // Request from the client to update the flow function evaluation result + // for a previously un-analyzed flow. + message DefineFlowQueryFunctionResult { + // (Deprecated) The fully qualified name of the flow being updated. + // + // This field is deprecated since Spark 4.2+. Use flow_identifier field instead. + optional string flow_name = 1 [deprecated = true]; + + // The fully qualified identifier of the flow being updated. + optional ResolvedIdentifier flow_identifier = 4; + + // The ID of the graph this flow belongs to. + optional string dataflow_graph_id = 2; + + // An unresolved relation that defines the dataset's flow. + optional spark.connect.Relation relation = 3; + } +} + +// Dispatch object for pipelines command results. +message PipelineCommandResult { + oneof result_type { + CreateDataflowGraphResult create_dataflow_graph_result = 1; + DefineOutputResult define_output_result = 2; + DefineFlowResult define_flow_result = 3; + } + message CreateDataflowGraphResult { + // The ID of the created graph. + optional string dataflow_graph_id = 1; + } + message DefineOutputResult { + // Resolved identifier of the output + optional ResolvedIdentifier resolved_identifier = 1; + } + message DefineFlowResult { + // Resolved identifier of the flow + optional ResolvedIdentifier resolved_identifier = 1; + } +} + +// The type of output. +enum OutputType { + // Safe default value. Should not be used. + OUTPUT_TYPE_UNSPECIFIED = 0; + // A materialized view which is published to the catalog + MATERIALIZED_VIEW = 1; + // A table which is published to the catalog + TABLE = 2; + // A view which is not published to the catalog + TEMPORARY_VIEW = 3; + // A sink which is not published to the catalog + SINK = 4; +} + +// A response containing an event emitted during the run of a pipeline. +message PipelineEventResult { + PipelineEvent event = 1; +} + +message PipelineEvent { + // The timestamp corresponding to when the event occurred. + google.protobuf.Timestamp timestamp = 1; + // The message that should be displayed to users. + optional string message = 2; +} + +// Source code location information associated with a particular dataset or flow. +message SourceCodeLocation { + // The file that this pipeline source code was defined in. + optional string file_name = 1; + // The specific line number that this pipeline source code is located at, if applicable. + optional int32 line_number = 2; + // The path of the top-level pipeline file determined at runtime during pipeline initialization. + optional string definition_path = 3; + + // Reserved field for protocol extensions. + // Used to support forward-compatibility by carrying additional fields + // that are not yet defined in this version of the proto. During planning, the + // engine will resolve and dispatch the concrete command contained in this field. + repeated google.protobuf.Any extension = 999; +} + +// A signal from the server to the client to execute the query function for one or more flows, and +// to register their results with the server. +message PipelineQueryFunctionExecutionSignal { + // (Deprecated) The name of flows that are ready to be re-evaluated. + // + // This field is deprecated since Spark 4.2+. Use flow_identifiers field instead. + repeated string flow_names = 1 [deprecated = true]; + + // The identifier of flows that are ready to be re-evaluated + repeated ResolvedIdentifier flow_identifiers = 2; +} + +// Metadata providing context about the pipeline during Spark Connect query analysis. +message PipelineAnalysisContext { + // Unique identifier of the dataflow graph associated with this pipeline. + optional string dataflow_graph_id = 1; + // The path of the top-level pipeline file determined at runtime during pipeline initialization. + optional string definition_path = 2; + // (Deprecated) The name of the Flow involved in this analysis + // + // This field is deprecated since Spark 4.2+. Use flow_identifier field instead. + optional string flow_name = 3 [deprecated = true]; + // The identifier of the Flow involved in this analysis + optional ResolvedIdentifier flow_identifier = 4; + + // Reserved field for protocol extensions. + repeated google.protobuf.Any extension = 999; +} diff --git a/proto/spark/connect/relations.proto b/proto/spark/connect/relations.proto new file mode 100644 index 0000000..57c4ed7 --- /dev/null +++ b/proto/spark/connect/relations.proto @@ -0,0 +1,1278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "google/protobuf/any.proto"; +import "spark/connect/expressions.proto"; +import "spark/connect/types.proto"; +import "spark/connect/catalog.proto"; +import "spark/connect/common.proto"; +import "spark/connect/ml_common.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// The main [[Relation]] type. Fundamentally, a relation is a typed container +// that has exactly one explicit relation type set. +// +// When adding new relation types, they have to be registered here. +message Relation { + RelationCommon common = 1; + oneof rel_type { + Read read = 2; + Project project = 3; + Filter filter = 4; + Join join = 5; + SetOperation set_op = 6; + Sort sort = 7; + Limit limit = 8; + Aggregate aggregate = 9; + SQL sql = 10; + LocalRelation local_relation = 11; + Sample sample = 12; + Offset offset = 13; + Deduplicate deduplicate = 14; + Range range = 15; + SubqueryAlias subquery_alias = 16; + Repartition repartition = 17; + ToDF to_df = 18; + WithColumnsRenamed with_columns_renamed = 19; + ShowString show_string = 20; + Drop drop = 21; + Tail tail = 22; + WithColumns with_columns = 23; + Hint hint = 24; + Unpivot unpivot = 25; + ToSchema to_schema = 26; + RepartitionByExpression repartition_by_expression = 27; + MapPartitions map_partitions = 28; + CollectMetrics collect_metrics = 29; + Parse parse = 30; + GroupMap group_map = 31; + CoGroupMap co_group_map = 32; + WithWatermark with_watermark = 33; + ApplyInPandasWithState apply_in_pandas_with_state = 34; + HtmlString html_string = 35; + CachedLocalRelation cached_local_relation = 36; + CachedRemoteRelation cached_remote_relation = 37; + CommonInlineUserDefinedTableFunction common_inline_user_defined_table_function = 38; + AsOfJoin as_of_join = 39; + CommonInlineUserDefinedDataSource common_inline_user_defined_data_source = 40; + WithRelations with_relations = 41; + Transpose transpose = 42; + UnresolvedTableValuedFunction unresolved_table_valued_function = 43; + LateralJoin lateral_join = 44; + ChunkedCachedLocalRelation chunked_cached_local_relation = 45; + RelationChanges relation_changes = 46; + + // NA functions + NAFill fill_na = 90; + NADrop drop_na = 91; + NAReplace replace = 92; + + // stat functions + StatSummary summary = 100; + StatCrosstab crosstab = 101; + StatDescribe describe = 102; + StatCov cov = 103; + StatCorr corr = 104; + StatApproxQuantile approx_quantile = 105; + StatFreqItems freq_items = 106; + StatSampleBy sample_by = 107; + + // Catalog API (experimental / unstable) + Catalog catalog = 200; + + // ML relation + MlRelation ml_relation = 300; + + // This field is used to mark extensions to the protocol. When plugins generate arbitrary + // relations they can add them here. During the planning the correct resolution is done. + google.protobuf.Any extension = 998; + Unknown unknown = 999; + } +} + +// Relation to represent ML world +message MlRelation { + oneof ml_type { + Transform transform = 1; + Fetch fetch = 2; + } + // (Optional) the dataset for restoring the model summary + optional Relation model_summary_dataset = 3; + + // Relation to represent transform(input) of the operator + // which could be a cached model or a new transformer + message Transform { + oneof operator { + // Object reference + ObjectRef obj_ref = 1; + // Could be an ML transformer like VectorAssembler + MlOperator transformer = 2; + } + // the input dataframe + Relation input = 3; + // the operator specific parameters + MlParams params = 4; + } +} + +// Message for fetching attribute from object on the server side. +// Fetch can be represented as a Relation or a ML command +// Command: model.coefficients, model.summary.weightedPrecision which +// returns the final literal result +// Relation: model.summary.roc which returns a DataFrame (Relation) +message Fetch { + // (Required) reference to the object on the server side + ObjectRef obj_ref = 1; + // (Required) the calling method chains + repeated Method methods = 2; + + // Represents a method with inclusion of method name and its arguments + message Method { + // (Required) the method name + string method = 1; + // (Optional) the arguments of the method + repeated Args args = 2; + + message Args { + oneof args_type { + Expression.Literal param = 1; + Relation input = 2; + } + } + } +} + +// Used for testing purposes only. +message Unknown {} + +// Common metadata of all relations. +message RelationCommon { + // (Required) Shared relation metadata. + string source_info = 1 [deprecated=true]; + + // (Optional) A per-client globally unique id for a given connect plan. + optional int64 plan_id = 2; + + // (Optional) Keep the information of the origin for this expression such as stacktrace. + Origin origin = 3; +} + +// Relation that uses a SQL query to generate the output. +message SQL { + // (Required) The SQL query. + string query = 1; + + // (Optional) A map of parameter names to literal expressions. + map args = 2 [deprecated=true]; + + // (Optional) A sequence of literal expressions for positional parameters in the SQL query text. + repeated Expression.Literal pos_args = 3 [deprecated=true]; + + // (Optional) A map of parameter names to expressions. + // It cannot coexist with `pos_arguments`. + map named_arguments = 4; + + // (Optional) A sequence of expressions for positional parameters in the SQL query text. + // It cannot coexist with `named_arguments`. + repeated Expression pos_arguments = 5; +} + +// Relation of type [[WithRelations]]. +// +// This relation contains a root plan, and one or more references that are used by the root plan. +// There are two ways of referencing a relation, by name (through a subquery alias), or by plan_id +// (using RelationCommon.plan_id). +// +// This relation can be used to implement CTEs, describe DAGs, or to reduce tree depth. +message WithRelations { + // (Required) Plan at the root of the query tree. This plan is expected to contain one or more + // references. Those references get expanded later on by the engine. + Relation root = 1; + + // (Required) Plans referenced by the root plan. Relations in this list are also allowed to + // contain references to other relations in this list, as long they do not form cycles. + repeated Relation references = 2; +} + +// Relation that reads from a file / table or other data source. Does not have additional +// inputs. +message Read { + oneof read_type { + NamedTable named_table = 1; + DataSource data_source = 2; + } + + // (Optional) Indicates if this is a streaming read. + bool is_streaming = 3; + + message NamedTable { + // (Required) Unparsed identifier for the table. + string unparsed_identifier = 1; + + // Options for the named table. The map key is case insensitive. + map options = 2; + } + + message DataSource { + // (Optional) Supported formats include: parquet, orc, text, json, parquet, csv, avro. + // + // If not set, the value from SQL conf 'spark.sql.sources.default' will be used. + optional string format = 1; + + // (Optional) If not set, Spark will infer the schema. + // + // This schema string should be either DDL-formatted or JSON-formatted. + optional string schema = 2; + + // Options for the data source. The context of this map varies based on the + // data source format. This options could be empty for valid data source format. + // The map key is case insensitive. + map options = 3; + + // (Optional) A list of path for file-system backed data sources. + repeated string paths = 4; + + // (Optional) Condition in the where clause for each partition. + // + // This is only supported by the JDBC data source. + repeated string predicates = 5; + + // (Optional) A user-provided name for the streaming source. + // This name is used in checkpoint metadata and enables stable checkpoint locations + // for source evolution. + optional string source_name = 6; + } +} + +// Reads Change Data Capture (CDC) changes for a named table. +// +// This corresponds to the `DataFrameReader.changes()` or `DataStreamReader.changes()` API. +// CDC-specific options (startingVersion, endingVersion, startingTimestamp, endingTimestamp, +// deduplicationMode, computeUpdates, etc.) are passed in the options map. +message RelationChanges { + // (Required) Unparsed identifier for the table. + string unparsed_identifier = 1; + + // Options for the CDC query. The map key is case insensitive. + // Supported keys include: startingVersion, endingVersion, startingTimestamp, + // endingTimestamp, deduplicationMode, computeUpdates, startingBoundInclusive, + // endingBoundInclusive. + map options = 2; + + // (Optional) Indicates if this is a streaming CDC read. + bool is_streaming = 3; +} + +// Projection of a bag of expressions for a given input relation. +// +// The input relation must be specified. +// The projected expression can be an arbitrary expression. +message Project { + // (Optional) Input relation is optional for Project. + // + // For example, `SELECT ABS(-1)` is valid plan without an input plan. + Relation input = 1; + + // (Required) A Project requires at least one expression. + repeated Expression expressions = 3; +} + +// Relation that applies a boolean expression `condition` on each row of `input` to produce +// the output result. +message Filter { + // (Required) Input relation for a Filter. + Relation input = 1; + + // (Required) A Filter must have a condition expression. + Expression condition = 2; +} + +// Relation of type [[Join]]. +// +// `left` and `right` must be present. +message Join { + // (Required) Left input relation for a Join. + Relation left = 1; + + // (Required) Right input relation for a Join. + Relation right = 2; + + // (Optional) The join condition. Could be unset when `using_columns` is utilized. + // + // This field does not co-exist with using_columns. + Expression join_condition = 3; + + // (Required) The join type. + JoinType join_type = 4; + + // Optional. using_columns provides a list of columns that should present on both sides of + // the join inputs that this Join will join on. For example A JOIN B USING col_name is + // equivalent to A JOIN B on A.col_name = B.col_name. + // + // This field does not co-exist with join_condition. + repeated string using_columns = 5; + + enum JoinType { + JOIN_TYPE_UNSPECIFIED = 0; + JOIN_TYPE_INNER = 1; + JOIN_TYPE_FULL_OUTER = 2; + JOIN_TYPE_LEFT_OUTER = 3; + JOIN_TYPE_RIGHT_OUTER = 4; + JOIN_TYPE_LEFT_ANTI = 5; + JOIN_TYPE_LEFT_SEMI = 6; + JOIN_TYPE_CROSS = 7; + } + + // (Optional) Only used by joinWith. Set the left and right join data types. + optional JoinDataType join_data_type = 6; + + message JoinDataType { + // If the left data type is a struct. + bool is_left_struct = 1; + // If the right data type is a struct. + bool is_right_struct = 2; + } +} + +// Relation of type [[SetOperation]] +message SetOperation { + // (Required) Left input relation for a Set operation. + Relation left_input = 1; + + // (Required) Right input relation for a Set operation. + Relation right_input = 2; + + // (Required) The Set operation type. + SetOpType set_op_type = 3; + + // (Optional) If to remove duplicate rows. + // + // True to preserve all results. + // False to remove duplicate rows. + optional bool is_all = 4; + + // (Optional) If to perform the Set operation based on name resolution. + // + // Only UNION supports this option. + optional bool by_name = 5; + + // (Optional) If to perform the Set operation and allow missing columns. + // + // Only UNION supports this option. + optional bool allow_missing_columns = 6; + + enum SetOpType { + SET_OP_TYPE_UNSPECIFIED = 0; + SET_OP_TYPE_INTERSECT = 1; + SET_OP_TYPE_UNION = 2; + SET_OP_TYPE_EXCEPT = 3; + } +} + +// Relation of type [[Limit]] that is used to `limit` rows from the input relation. +message Limit { + // (Required) Input relation for a Limit. + Relation input = 1; + + // (Required) the limit. + int32 limit = 2; +} + +// Relation of type [[Offset]] that is used to read rows staring from the `offset` on +// the input relation. +message Offset { + // (Required) Input relation for an Offset. + Relation input = 1; + + // (Required) the limit. + int32 offset = 2; +} + +// Relation of type [[Tail]] that is used to fetch `limit` rows from the last of the input relation. +message Tail { + // (Required) Input relation for an Tail. + Relation input = 1; + + // (Required) the limit. + int32 limit = 2; +} + +// Relation of type [[Aggregate]]. +message Aggregate { + // (Required) Input relation for a RelationalGroupedDataset. + Relation input = 1; + + // (Required) How the RelationalGroupedDataset was built. + GroupType group_type = 2; + + // (Required) Expressions for grouping keys + repeated Expression grouping_expressions = 3; + + // (Required) List of values that will be translated to columns in the output DataFrame. + repeated Expression aggregate_expressions = 4; + + // (Optional) Pivots a column of the current `DataFrame` and performs the specified aggregation. + Pivot pivot = 5; + + // (Optional) List of values that will be translated to columns in the output DataFrame. + repeated GroupingSets grouping_sets = 6; + + enum GroupType { + GROUP_TYPE_UNSPECIFIED = 0; + GROUP_TYPE_GROUPBY = 1; + GROUP_TYPE_ROLLUP = 2; + GROUP_TYPE_CUBE = 3; + GROUP_TYPE_PIVOT = 4; + GROUP_TYPE_GROUPING_SETS = 5; + } + + message Pivot { + // (Required) The column to pivot + Expression col = 1; + + // (Optional) List of values that will be translated to columns in the output DataFrame. + // + // Note that if it is empty, the server side will immediately trigger a job to collect + // the distinct values of the column. + repeated Expression.Literal values = 2; + } + + message GroupingSets { + // (Required) Individual grouping set + repeated Expression grouping_set = 1; + } +} + +// Relation of type [[Sort]]. +message Sort { + // (Required) Input relation for a Sort. + Relation input = 1; + + // (Required) The ordering expressions + repeated Expression.SortOrder order = 2; + + // (Optional) if this is a global sort. + optional bool is_global = 3; +} + + +// Drop specified columns. +message Drop { + // (Required) The input relation. + Relation input = 1; + + // (Optional) columns to drop. + repeated Expression columns = 2; + + // (Optional) names of columns to drop. + repeated string column_names = 3; +} + + +// Relation of type [[Deduplicate]] which have duplicate rows removed, could consider either only +// the subset of columns or all the columns. +message Deduplicate { + // (Required) Input relation for a Deduplicate. + Relation input = 1; + + // (Optional) Deduplicate based on a list of column names. + // + // This field does not co-use with `all_columns_as_keys`. + repeated string column_names = 2; + + // (Optional) Deduplicate based on all the columns of the input relation. + // + // This field does not co-use with `column_names`. + optional bool all_columns_as_keys = 3; + + // (Optional) Deduplicate within the time range of watermark. + optional bool within_watermark = 4; +} + +// A relation that does not need to be qualified by name. +message LocalRelation { + // (Optional) Local collection data serialized into Arrow IPC streaming format which contains + // the schema of the data. + optional bytes data = 1; + + // (Optional) The schema of local data. + // It should be either a DDL-formatted type string or a JSON string. + // + // The server side will update the column names and data types according to this schema. + // If the 'data' is not provided, then this schema will be required. + optional string schema = 2; +} + +// A local relation that has been cached already. +// CachedLocalRelation doesn't support LocalRelations of size over 2GB. +message CachedLocalRelation { + // `userId` and `sessionId` fields are deleted since the server must always use the active + // session/user rather than arbitrary values provided by the client. It is never valid to access + // a local relation from a different session/user. + reserved 1, 2; + reserved "userId", "sessionId"; + + // (Required) A sha-256 hash of the serialized local relation in proto, see LocalRelation. + string hash = 3; +} + +// A local relation that has been cached already. +message ChunkedCachedLocalRelation { + // (Required) A list of sha-256 hashes for representing LocalRelation.data. + // Data is serialized in Arrow IPC streaming format, each batch is cached on the server as + // a separate artifact. Each hash represents one batch stored on the server. + // Hashes are hex-encoded strings (e.g., "a3b2c1d4..."). + repeated string dataHashes = 1; + + // (Optional) A sha-256 hash of the serialized LocalRelation.schema. + // Scala clients always provide the schema, Python clients can omit it. + // Hash is a hex-encoded string (e.g., "a3b2c1d4..."). + optional string schemaHash = 2; +} + +// Represents a remote relation that has been cached on server. +message CachedRemoteRelation { + // (Required) ID of the remote related (assigned by the service). + string relation_id = 1; +} + +// Relation of type [[Sample]] that samples a fraction of the dataset. +message Sample { + // (Required) Input relation for a Sample. + Relation input = 1; + + // (Required) lower bound. + double lower_bound = 2; + + // (Required) upper bound. + double upper_bound = 3; + + // (Optional) Whether to sample with replacement. + optional bool with_replacement = 4; + + // (Required) The random seed. + // This field is required to avoid generating mutable dataframes (see SPARK-48184 for details), + // however, still keep it 'optional' here for backward compatibility. + optional int64 seed = 5; + + // (Required) Explicitly sort the underlying plan to make the ordering deterministic or cache it. + // This flag is true when invoking `dataframe.randomSplit` to randomly splits DataFrame with the + // provided weights. Otherwise, it is false. + bool deterministic_order = 6; +} + +// Relation of type [[Range]] that generates a sequence of integers. +message Range { + // (Optional) Default value = 0 + optional int64 start = 1; + + // (Required) + int64 end = 2; + + // (Required) + int64 step = 3; + + // Optional. Default value is assigned by 1) SQL conf "spark.sql.leafNodeDefaultParallelism" if + // it is set, or 2) spark default parallelism. + optional int32 num_partitions = 4; +} + +// Relation alias. +message SubqueryAlias { + // (Required) The input relation of SubqueryAlias. + Relation input = 1; + + // (Required) The alias. + string alias = 2; + + // (Optional) Qualifier of the alias. + repeated string qualifier = 3; +} + +// Relation repartition. +message Repartition { + // (Required) The input relation of Repartition. + Relation input = 1; + + // (Required) Must be positive. + int32 num_partitions = 2; + + // (Optional) Default value is false. + optional bool shuffle = 3; +} + +// Compose the string representing rows for output. +// It will invoke 'Dataset.showString' to compute the results. +message ShowString { + // (Required) The input relation. + Relation input = 1; + + // (Required) Number of rows to show. + int32 num_rows = 2; + + // (Required) If set to more than 0, truncates strings to + // `truncate` characters and all cells will be aligned right. + int32 truncate = 3; + + // (Required) If set to true, prints output rows vertically (one line per column value). + bool vertical = 4; +} + +// Compose the string representing rows for output. +// It will invoke 'Dataset.htmlString' to compute the results. +message HtmlString { + // (Required) The input relation. + Relation input = 1; + + // (Required) Number of rows to show. + int32 num_rows = 2; + + // (Required) If set to more than 0, truncates strings to + // `truncate` characters and all cells will be aligned right. + int32 truncate = 3; +} + +// Computes specified statistics for numeric and string columns. +// It will invoke 'Dataset.summary' (same as 'StatFunctions.summary') +// to compute the results. +message StatSummary { + // (Required) The input relation. + Relation input = 1; + + // (Optional) Statistics from to be computed. + // + // Available statistics are: + // count + // mean + // stddev + // min + // max + // arbitrary approximate percentiles specified as a percentage (e.g. 75%) + // count_distinct + // approx_count_distinct + // + // If no statistics are given, this function computes 'count', 'mean', 'stddev', 'min', + // 'approximate quartiles' (percentiles at 25%, 50%, and 75%), and 'max'. + repeated string statistics = 2; +} + +// Computes basic statistics for numeric and string columns, including count, mean, stddev, min, +// and max. If no columns are given, this function computes statistics for all numerical or +// string columns. +message StatDescribe { + // (Required) The input relation. + Relation input = 1; + + // (Optional) Columns to compute statistics on. + repeated string cols = 2; +} + +// Computes a pair-wise frequency table of the given columns. Also known as a contingency table. +// It will invoke 'Dataset.stat.crosstab' (same as 'StatFunctions.crossTabulate') +// to compute the results. +message StatCrosstab { + // (Required) The input relation. + Relation input = 1; + + // (Required) The name of the first column. + // + // Distinct items will make the first item of each row. + string col1 = 2; + + // (Required) The name of the second column. + // + // Distinct items will make the column names of the DataFrame. + string col2 = 3; +} + +// Calculate the sample covariance of two numerical columns of a DataFrame. +// It will invoke 'Dataset.stat.cov' (same as 'StatFunctions.calculateCov') to compute the results. +message StatCov { + // (Required) The input relation. + Relation input = 1; + + // (Required) The name of the first column. + string col1 = 2; + + // (Required) The name of the second column. + string col2 = 3; +} + +// Calculates the correlation of two columns of a DataFrame. Currently only supports the Pearson +// Correlation Coefficient. It will invoke 'Dataset.stat.corr' (same as +// 'StatFunctions.pearsonCorrelation') to compute the results. +message StatCorr { + // (Required) The input relation. + Relation input = 1; + + // (Required) The name of the first column. + string col1 = 2; + + // (Required) The name of the second column. + string col2 = 3; + + // (Optional) Default value is 'pearson'. + // + // Currently only supports the Pearson Correlation Coefficient. + optional string method = 4; +} + +// Calculates the approximate quantiles of numerical columns of a DataFrame. +// It will invoke 'Dataset.stat.approxQuantile' (same as 'StatFunctions.approxQuantile') +// to compute the results. +message StatApproxQuantile { + // (Required) The input relation. + Relation input = 1; + + // (Required) The names of the numerical columns. + repeated string cols = 2; + + // (Required) A list of quantile probabilities. + // + // Each number must belong to [0, 1]. + // For example 0 is the minimum, 0.5 is the median, 1 is the maximum. + repeated double probabilities = 3; + + // (Required) The relative target precision to achieve (greater than or equal to 0). + // + // If set to zero, the exact quantiles are computed, which could be very expensive. + // Note that values greater than 1 are accepted but give the same result as 1. + double relative_error = 4; +} + +// Finding frequent items for columns, possibly with false positives. +// It will invoke 'Dataset.stat.freqItems' (same as 'StatFunctions.freqItems') +// to compute the results. +message StatFreqItems { + // (Required) The input relation. + Relation input = 1; + + // (Required) The names of the columns to search frequent items in. + repeated string cols = 2; + + // (Optional) The minimum frequency for an item to be considered `frequent`. + // Should be greater than 1e-4. + optional double support = 3; +} + + +// Returns a stratified sample without replacement based on the fraction +// given on each stratum. +// It will invoke 'Dataset.stat.freqItems' (same as 'StatFunctions.freqItems') +// to compute the results. +message StatSampleBy { + // (Required) The input relation. + Relation input = 1; + + // (Required) The column that defines strata. + Expression col = 2; + + // (Required) Sampling fraction for each stratum. + // + // If a stratum is not specified, we treat its fraction as zero. + repeated Fraction fractions = 3; + + // (Required) The random seed. + // This field is required to avoid generating mutable dataframes (see SPARK-48184 for details), + // however, still keep it 'optional' here for backward compatibility. + optional int64 seed = 5; + + message Fraction { + // (Required) The stratum. + Expression.Literal stratum = 1; + + // (Required) The fraction value. Must be in [0, 1]. + double fraction = 2; + } +} + + +// Replaces null values. +// It will invoke 'Dataset.na.fill' (same as 'DataFrameNaFunctions.fill') to compute the results. +// Following 3 parameter combinations are supported: +// 1, 'values' only contains 1 item, 'cols' is empty: +// replaces null values in all type-compatible columns. +// 2, 'values' only contains 1 item, 'cols' is not empty: +// replaces null values in specified columns. +// 3, 'values' contains more than 1 items, then 'cols' is required to have the same length: +// replaces each specified column with corresponding value. +message NAFill { + // (Required) The input relation. + Relation input = 1; + + // (Optional) Optional list of column names to consider. + repeated string cols = 2; + + // (Required) Values to replace null values with. + // + // Should contain at least 1 item. + // Only 4 data types are supported now: bool, long, double, string + repeated Expression.Literal values = 3; +} + + +// Drop rows containing null values. +// It will invoke 'Dataset.na.drop' (same as 'DataFrameNaFunctions.drop') to compute the results. +message NADrop { + // (Required) The input relation. + Relation input = 1; + + // (Optional) Optional list of column names to consider. + // + // When it is empty, all the columns in the input relation will be considered. + repeated string cols = 2; + + // (Optional) The minimum number of non-null and non-NaN values required to keep. + // + // When not set, it is equivalent to the number of considered columns, which means + // a row will be kept only if all columns are non-null. + // + // 'how' options ('all', 'any') can be easily converted to this field: + // - 'all' -> set 'min_non_nulls' 1; + // - 'any' -> keep 'min_non_nulls' unset; + optional int32 min_non_nulls = 3; +} + + +// Replaces old values with the corresponding values. +// It will invoke 'Dataset.na.replace' (same as 'DataFrameNaFunctions.replace') +// to compute the results. +message NAReplace { + // (Required) The input relation. + Relation input = 1; + + // (Optional) List of column names to consider. + // + // When it is empty, all the type-compatible columns in the input relation will be considered. + repeated string cols = 2; + + // (Optional) The value replacement mapping. + repeated Replacement replacements = 3; + + message Replacement { + // (Required) The old value. + // + // Only 4 data types are supported now: null, bool, double, string. + Expression.Literal old_value = 1; + + // (Required) The new value. + // + // Should be of the same data type with the old value. + Expression.Literal new_value = 2; + } +} + + +// Rename columns on the input relation by the same length of names. +message ToDF { + // (Required) The input relation of RenameColumnsBySameLengthNames. + Relation input = 1; + + // (Required) + // + // The number of columns of the input relation must be equal to the length + // of this field. If this is not true, an exception will be returned. + repeated string column_names = 2; +} + + +// Rename columns on the input relation by a map with name to name mapping. +message WithColumnsRenamed { + // (Required) The input relation. + Relation input = 1; + + + // (Optional) + // + // Renaming column names of input relation from A to B where A is the map key + // and B is the map value. This is a no-op if schema doesn't contain any A. It + // does not require that all input relation column names to present as keys. + // duplicated B are not allowed. + map rename_columns_map = 2 [deprecated=true]; + + repeated Rename renames = 3; + + message Rename { + // (Required) The existing column name. + string col_name = 1; + + // (Required) The new column name. + string new_col_name = 2; + } +} + +// Adding columns or replacing the existing columns that have the same names. +message WithColumns { + // (Required) The input relation. + Relation input = 1; + + // (Required) + // + // Given a column name, apply the corresponding expression on the column. If column + // name exists in the input relation, then replace the column. If the column name + // does not exist in the input relation, then adds it as a new column. + // + // Only one name part is expected from each Expression.Alias. + // + // An exception is thrown when duplicated names are present in the mapping. + repeated Expression.Alias aliases = 2; +} + +message WithWatermark { + + // (Required) The input relation + Relation input = 1; + + // (Required) Name of the column containing event time. + string event_time = 2; + + // (Required) + string delay_threshold = 3; +} + +// Specify a hint over a relation. Hint should have a name and optional parameters. +message Hint { + // (Required) The input relation. + Relation input = 1; + + // (Required) Hint name. + // + // Supported Join hints include BROADCAST, MERGE, SHUFFLE_HASH, SHUFFLE_REPLICATE_NL. + // + // Supported partitioning hints include COALESCE, REPARTITION, REPARTITION_BY_RANGE. + string name = 2; + + // (Optional) Hint parameters. + repeated Expression parameters = 3; +} + +// Unpivot a DataFrame from wide format to long format, optionally leaving identifier columns set. +message Unpivot { + // (Required) The input relation. + Relation input = 1; + + // (Required) Id columns. + repeated Expression ids = 2; + + // (Optional) Value columns to unpivot. + optional Values values = 3; + + // (Required) Name of the variable column. + string variable_column_name = 4; + + // (Required) Name of the value column. + string value_column_name = 5; + + message Values { + repeated Expression values = 1; + } +} + +// Transpose a DataFrame, switching rows to columns. +// Transforms the DataFrame such that the values in the specified index column +// become the new columns of the DataFrame. +message Transpose { + // (Required) The input relation. + Relation input = 1; + + // (Optional) A list of columns that will be treated as the indices. + // Only single column is supported now. + repeated Expression index_columns = 2; +} + +message UnresolvedTableValuedFunction { + // (Required) name (or unparsed name for user defined function) for the unresolved function. + string function_name = 1; + + // (Optional) Function arguments. Empty arguments are allowed. + repeated Expression arguments = 2; +} + +message ToSchema { + // (Required) The input relation. + Relation input = 1; + + // (Required) The user provided schema. + // + // The Sever side will update the dataframe with this schema. + DataType schema = 2; +} + +message RepartitionByExpression { + // (Required) The input relation. + Relation input = 1; + + // (Required) The partitioning expressions. + repeated Expression partition_exprs = 2; + + // (Optional) number of partitions, must be positive. + optional int32 num_partitions = 3; +} + +message MapPartitions { + // (Required) Input relation for a mapPartitions-equivalent API: mapInPandas, mapInArrow. + Relation input = 1; + + // (Required) Input user-defined function. + CommonInlineUserDefinedFunction func = 2; + + // (Optional) Whether to use barrier mode execution or not. + optional bool is_barrier = 3; + + // (Optional) ResourceProfile id used for the stage level scheduling. + optional int32 profile_id = 4; +} + +message GroupMap { + // (Required) Input relation for Group Map API: apply, applyInPandas. + Relation input = 1; + + // (Required) Expressions for grouping keys. + repeated Expression grouping_expressions = 2; + + // (Required) Input user-defined function. + CommonInlineUserDefinedFunction func = 3; + + // (Optional) Expressions for sorting. Only used by Scala Sorted Group Map API. + repeated Expression sorting_expressions = 4; + + // Below fields are only used by (Flat)MapGroupsWithState + // (Optional) Input relation for initial State. + Relation initial_input = 5; + + // (Optional) Expressions for grouping keys of the initial state input relation. + repeated Expression initial_grouping_expressions = 6; + + // (Optional) True if MapGroupsWithState, false if FlatMapGroupsWithState. + optional bool is_map_groups_with_state = 7; + + // (Optional) The output mode of the function. + optional string output_mode = 8; + + // (Optional) Timeout configuration for groups that do not receive data for a while. + optional string timeout_conf = 9; + + // (Optional) The schema for the grouped state. + optional DataType state_schema = 10; + + // Below fields are used by TransformWithState and TransformWithStateInPandas + // (Optional) TransformWithState related parameters. + optional TransformWithStateInfo transform_with_state_info = 11; +} + +// Additional input parameters used for TransformWithState operator. +message TransformWithStateInfo { + // (Required) Time mode string for transformWithState. + string time_mode = 1; + + // (Optional) Event time column name. + optional string event_time_column_name = 2; + + // (Optional) Schema for the output DataFrame. + // Only required used for TransformWithStateInPandas. + optional DataType output_schema = 3; +} + +message CoGroupMap { + // (Required) One input relation for CoGroup Map API - applyInPandas. + Relation input = 1; + + // Expressions for grouping keys of the first input relation. + repeated Expression input_grouping_expressions = 2; + + // (Required) The other input relation. + Relation other = 3; + + // Expressions for grouping keys of the other input relation. + repeated Expression other_grouping_expressions = 4; + + // (Required) Input user-defined function. + CommonInlineUserDefinedFunction func = 5; + + // (Optional) Expressions for sorting. Only used by Scala Sorted CoGroup Map API. + repeated Expression input_sorting_expressions = 6; + + // (Optional) Expressions for sorting. Only used by Scala Sorted CoGroup Map API. + repeated Expression other_sorting_expressions = 7; +} + +message ApplyInPandasWithState { + // (Required) Input relation for applyInPandasWithState. + Relation input = 1; + + // (Required) Expressions for grouping keys. + repeated Expression grouping_expressions = 2; + + // (Required) Input user-defined function. + CommonInlineUserDefinedFunction func = 3; + + // (Required) Schema for the output DataFrame. + string output_schema = 4; + + // (Required) Schema for the state. + string state_schema = 5; + + // (Required) The output mode of the function. + string output_mode = 6; + + // (Required) Timeout configuration for groups that do not receive data for a while. + string timeout_conf = 7; +} + +message CommonInlineUserDefinedTableFunction { + // (Required) Name of the user-defined table function. + string function_name = 1; + + // (Optional) Whether the user-defined table function is deterministic. + bool deterministic = 2; + + // (Optional) Function input arguments. Empty arguments are allowed. + repeated Expression arguments = 3; + + // (Required) Type of the user-defined table function. + oneof function { + PythonUDTF python_udtf = 4; + } +} + +message PythonUDTF { + // (Optional) Return type of the Python UDTF. + optional DataType return_type = 1; + + // (Required) EvalType of the Python UDTF. + int32 eval_type = 2; + + // (Required) The encoded commands of the Python UDTF. + bytes command = 3; + + // (Required) Python version being used in the client. + string python_ver = 4; +} + +message CommonInlineUserDefinedDataSource { + // (Required) Name of the data source. + string name = 1; + + // (Required) The data source type. + oneof data_source { + PythonDataSource python_data_source = 2; + } +} + +message PythonDataSource { + // (Required) The encoded commands of the Python data source. + bytes command = 1; + + // (Required) Python version being used in the client. + string python_ver = 2; +} + +// Collect arbitrary (named) metrics from a dataset. +message CollectMetrics { + // (Required) The input relation. + Relation input = 1; + + // (Required) Name of the metrics. + string name = 2; + + // (Required) The metric sequence. + repeated Expression metrics = 3; +} + +message Parse { + // (Required) Input relation to Parse. The input is expected to have single text column. + Relation input = 1; + // (Required) The expected format of the text. + ParseFormat format = 2; + + // (Optional) DataType representing the schema. If not set, Spark will infer the schema. + optional DataType schema = 3; + + // Options for the csv/json/xml parser. The map key is case insensitive. + map options = 4; + enum ParseFormat { + PARSE_FORMAT_UNSPECIFIED = 0; + PARSE_FORMAT_CSV = 1; + PARSE_FORMAT_JSON = 2; + PARSE_FORMAT_XML = 3; + } +} + +// Relation of type [[AsOfJoin]]. +// +// `left` and `right` must be present. +message AsOfJoin { + // (Required) Left input relation for a Join. + Relation left = 1; + + // (Required) Right input relation for a Join. + Relation right = 2; + + // (Required) Field to join on in left DataFrame + Expression left_as_of = 3; + + // (Required) Field to join on in right DataFrame + Expression right_as_of = 4; + + // (Optional) The join condition. Could be unset when `using_columns` is utilized. + // + // This field does not co-exist with using_columns. + Expression join_expr = 5; + + // Optional. using_columns provides a list of columns that should present on both sides of + // the join inputs that this Join will join on. For example A JOIN B USING col_name is + // equivalent to A JOIN B on A.col_name = B.col_name. + // + // This field does not co-exist with join_condition. + repeated string using_columns = 6; + + // (Required) The join type. + string join_type = 7; + + // (Optional) The asof tolerance within this range. + Expression tolerance = 8; + + // (Required) Whether allow matching with the same value or not. + bool allow_exact_matches = 9; + + // (Required) Whether to search for prior, subsequent, or closest matches. + string direction = 10; +} + +// Relation of type [[LateralJoin]]. +// +// `left` and `right` must be present. +message LateralJoin { + // (Required) Left input relation for a Join. + Relation left = 1; + + // (Required) Right input relation for a Join. + Relation right = 2; + + // (Optional) The join condition. + Expression join_condition = 3; + + // (Required) The join type. + Join.JoinType join_type = 4; +} diff --git a/proto/spark/connect/types.proto b/proto/spark/connect/types.proto new file mode 100644 index 0000000..caaa234 --- /dev/null +++ b/proto/spark/connect/types.proto @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// This message describes the logical [[DataType]] of something. It does not carry the value +// itself but only describes it. +message DataType { + oneof kind { + NULL null = 1; + + Binary binary = 2; + + Boolean boolean = 3; + + // Numeric types + Byte byte = 4; + Short short = 5; + Integer integer = 6; + Long long = 7; + + Float float = 8; + Double double = 9; + Decimal decimal = 10; + + // String types + String string = 11; + Char char = 12; + VarChar var_char = 13; + + // Datatime types + Date date = 14; + Timestamp timestamp = 15; + TimestampNTZ timestamp_ntz = 16; + + // Interval types + CalendarInterval calendar_interval = 17; + YearMonthInterval year_month_interval = 18; + DayTimeInterval day_time_interval = 19; + + // Complex types + Array array = 20; + Struct struct = 21; + Map map = 22; + Variant variant = 25; + + // UserDefinedType + UDT udt = 23; + + // Geospatial types + Geometry geometry = 26; + + Geography geography = 27; + + // UnparsedDataType + Unparsed unparsed = 24; + + Time time = 28; + } + + message Boolean { + uint32 type_variation_reference = 1; + } + + message Byte { + uint32 type_variation_reference = 1; + } + + message Short { + uint32 type_variation_reference = 1; + } + + message Integer { + uint32 type_variation_reference = 1; + } + + message Long { + uint32 type_variation_reference = 1; + } + + message Float { + uint32 type_variation_reference = 1; + } + + message Double { + uint32 type_variation_reference = 1; + } + + message String { + uint32 type_variation_reference = 1; + string collation = 2; + } + + message Binary { + uint32 type_variation_reference = 1; + } + + message NULL { + uint32 type_variation_reference = 1; + } + + message Timestamp { + uint32 type_variation_reference = 1; + } + + message Date { + uint32 type_variation_reference = 1; + } + + message TimestampNTZ { + uint32 type_variation_reference = 1; + } + + message Time { + optional int32 precision = 1; + uint32 type_variation_reference = 2; + } + + message CalendarInterval { + uint32 type_variation_reference = 1; + } + + message YearMonthInterval { + optional int32 start_field = 1; + optional int32 end_field = 2; + uint32 type_variation_reference = 3; + } + + message DayTimeInterval { + optional int32 start_field = 1; + optional int32 end_field = 2; + uint32 type_variation_reference = 3; + } + + // Start compound types. + message Char { + int32 length = 1; + uint32 type_variation_reference = 2; + } + + message VarChar { + int32 length = 1; + uint32 type_variation_reference = 2; + } + + message Decimal { + optional int32 scale = 1; + optional int32 precision = 2; + uint32 type_variation_reference = 3; + } + + message StructField { + string name = 1; + DataType data_type = 2; + bool nullable = 3; + optional string metadata = 4; + } + + message Struct { + repeated StructField fields = 1; + uint32 type_variation_reference = 2; + } + + message Array { + DataType element_type = 1; + bool contains_null = 2; + uint32 type_variation_reference = 3; + } + + message Map { + DataType key_type = 1; + DataType value_type = 2; + bool value_contains_null = 3; + uint32 type_variation_reference = 4; + } + + message Geometry { + int32 srid = 1; + uint32 type_variation_reference = 2; + } + + message Geography { + int32 srid = 1; + uint32 type_variation_reference = 2; + } + + message Variant { + uint32 type_variation_reference = 1; + } + + message UDT { + string type = 1; + // Required for Scala/Java UDT + optional string jvm_class = 2; + // Required for Python UDT + optional string python_class = 3; + // Required for Python UDT + optional string serialized_python_class = 4; + // Required for Python UDT + optional DataType sql_type = 5; + } + + message Unparsed { + // (Required) The unparsed data type string + string data_type_string = 1; + } +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..92337f5 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[toolchain] +channel = "stable" +components = ["rustfmt", "clippy"]