From 9a9c41f1857dd8c2f6ceaaa58757f488ec70448f Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Wed, 15 Jul 2026 13:57:31 -0700 Subject: [PATCH 1/7] [KYUUBI #7593][AUTHZ] Add paranoid mode for unclassified plan nodes --- docs/security/authorization/spark/install.md | 30 ++ extensions/spark/kyuubi-spark-authz/README.md | 9 + .../kyuubi-spark-authz/docs/paranoid-mode.md | 339 +++++++++++++++ .../main/resources/database_command_spec.json | 45 +- .../main/resources/function_command_spec.json | 12 +- .../main/resources/known_harmless_spec.json | 61 +++ .../src/main/resources/scan_command_spec.json | 24 +- .../main/resources/table_command_spec.json | 396 ++++++++++++------ .../plugin/spark/authz/ParanoidMode.scala | 155 +++++++ .../spark/authz/PrivilegesBuilder.scala | 261 ++++++++---- .../spark/authz/serde/CommandSpec.scala | 133 +++++- .../plugin/spark/authz/serde/package.scala | 34 ++ .../plugin/spark/authz/util/AuthZUtils.scala | 23 + .../classification_backlog_spark_3.5.txt | 136 ++++++ .../classification_backlog_spark_4.1.txt | 181 ++++++++ .../authz/ClassificationCoverageSuite.scala | 249 +++++++++++ .../spark/authz/ParanoidModeSuite.scala | 206 +++++++++ .../spark/authz/SparkSessionProvider.scala | 4 + .../authz/gen/JsonSpecFileGenerator.scala | 57 ++- .../spark/authz/gen/KnownHarmlessNodes.scala | 119 ++++++ 20 files changed, 2224 insertions(+), 250 deletions(-) create mode 100644 extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md create mode 100644 extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json create mode 100644 extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ParanoidMode.scala create mode 100644 extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_3.5.txt create mode 100644 extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt create mode 100644 extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala create mode 100644 extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ParanoidModeSuite.scala create mode 100644 extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala diff --git a/docs/security/authorization/spark/install.md b/docs/security/authorization/spark/install.md index 94419ff91a3..0680d7e462d 100644 --- a/docs/security/authorization/spark/install.md +++ b/docs/security/authorization/spark/install.md @@ -153,3 +153,33 @@ Add `org.apache.kyuubi.plugin.spark.authz.ranger.RangerSparkExtension` to the sp spark.sql.extensions=org.apache.kyuubi.plugin.spark.authz.ranger.RangerSparkExtension ``` +### Handling of unclassified plan nodes + +The plugin builds access requests by recognizing Spark logical plan nodes. A plan node it +does not recognize — for example a command introduced by a newer Spark version or by a +third-party catalog plugin — carries no access request, so by default it would execute +without any authorization check. The `spark.kyuubi.authz.unclassifiedNode.behavior` +configuration controls what happens when such a node is encountered: + +```properties +# allow | warn | deny (default: warn) +spark.kyuubi.authz.unclassifiedNode.behavior=deny +``` + +- `allow`: legacy behavior; the node is silently treated as not authorization-relevant. +- `warn` (default): the query proceeds, but a warning naming the unclassified plan node + class is logged once per class per JVM. +- `deny`: the query fails with an `AccessControlException` naming the unclassified class. + This makes the plugin fail closed and is the recommended setting for security-sensitive + deployments. + +The same setting governs extraction failures against recognized commands (e.g. after a +Spark upgrade changes a plan node's shape): `warn` logs them, `deny` fails the query. + +Plan nodes that are genuinely not authorization-relevant (e.g. `SELECT` without a table, +session `SET` commands) are declared in the plugin's `known_harmless_spec.json` resource. +Each entry carries a human-reviewed reason and names the exact Spark `major.minor` +versions that review applies to; on any other Spark version the entry is inert and the +node is treated as unclassified, so upgrading Spark past the reviewed versions surfaces +each entry for re-review instead of silently trusting it. + diff --git a/extensions/spark/kyuubi-spark-authz/README.md b/extensions/spark/kyuubi-spark-authz/README.md index 55291122de3..f1762744b1e 100644 --- a/extensions/spark/kyuubi-spark-authz/README.md +++ b/extensions/spark/kyuubi-spark-authz/README.md @@ -22,6 +22,15 @@ - [x] Column-level fine-grained authorization - [x] Row-level fine-grained authorization, a.k.a. Row-level filtering - [x] Data masking +- [x] Fail-closed handling of unclassified plan nodes ("paranoid mode"), + via `spark.kyuubi.authz.unclassifiedNode.behavior=allow|warn|deny` + +## Design Notes + +- [Paranoid mode](docs/paranoid-mode.md) — why non-recognition of a plan node must not + silently authorize it, the runtime `allow|warn|deny` mechanism, the + `known_harmless_spec.json` allowlist policy, and the per-Spark-profile build-time + coverage checks (`ClassificationCoverageSuite`). ## Build diff --git a/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md b/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md new file mode 100644 index 00000000000..cd1e8538f72 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md @@ -0,0 +1,339 @@ + + +# Paranoid Mode: Fail-Closed Handling of Unclassified Plan Nodes + +**Status:** Implemented. Runtime mechanism in `ParanoidMode.scala` / `PrivilegesBuilder.scala`; +build-time checks in `ClassificationCoverageSuite`; user-facing configuration documented in +`docs/security/authorization/spark/install.md`. + +## 1. Background and Motivation + +The authz plugin is the enforcement point for fine-grained access control on Spark SQL. It +works by walking each Catalyst logical plan, pattern-matching plan nodes against a set of +known node types (the JSON spec files, plus a few `nodeName` string matches), and building +Ranger access requests from the nodes it recognizes. + +The structural weakness is that **recognition is the security boundary, and non-recognition +fails open**. Any plan node that falls through the pattern match is implicitly classified as +not-authz-relevant and contributes no access request. Spark's logical plan space is open — +new commands and relations appear in every Spark minor release, and third-party catalogs +(Iceberg, Delta, Hudi, Paimon) inject their own nodes at user runtime — so the set of +unrecognized nodes grows silently over time. + +This is not hypothetical. During the Spark 4.1 porting effort, several cases were found +where adjusting the pattern match converted false-negative authorization decisions into +true positives — found by accident, in the course of other work. And when the build-time +enumeration check (§6) first ran, it counted **136 unclassified authz-relevant plan classes +on the Spark 3.5 classpath and 170 on Spark 4.1** — the population that fail-open behavior +had been hiding. + +A green test suite does not bound this risk: every test was written for a node type someone +had already classified, so the suite certifies that the *previously known* sample still +authorizes correctly and says nothing about nodes nobody thought to test. + +The stakes are what make this urgent. Deployments that take the plugin's security claims at +face value put regulated workloads (PII, PHI, financial data) behind it; for them a +fail-open authorization gap is not a bug ticket but a potential compliance incident, and +the failure mode is silent by construction. The gap between "policies are enforced on every +operation the plugin recognizes" and "policies are enforced on every operation" is +precisely the part an operator cannot audit from the outside. + +## 2. The Fail-Open Layers + +Four distinct layers, all addressed by this change. The first three were identified up +front; the fourth was discovered during implementation. + +1. **Unknown commands.** `PrivilegesBuilder.buildCommand` dispatches on class name against + the three command spec maps. The fallthrough returned `OperationType.QUERY` with zero + privilege objects, so `RuleAuthorization` built zero access requests and never called + `verify` — the command executed with no check at all. + +2. **Unknown leaf relations.** In `buildQuery`, a leaf node not matched by a scan spec (or + the `UnresolvedRelation` nodeName match) fell into the generic recursive arm, has no + children, and contributed nothing. A *known* scan node that was not `resolved` fell + through the same way. + +3. **Extractor drift on known commands.** Descriptor extraction was wrapped in + `catch { case e: Exception => LOG.debug(...); Nil }`. A command that *has* a spec still + failed open when its extractors broke against a new Spark version — the class name + matched, but the field the extractor reads had moved. The spec's existence created the + appearance of coverage; the only trace was a DEBUG log line. + +4. **Constant-projection pruning.** `buildQuery` deliberately skips the subtree of a + `Project` whose output has no relation to its input (a constant projection reads no + columns). But the subtree still *executes* — `SELECT 'x' FROM t` — so an unclassified + node under a constant projection was invisible to privilege building entirely. + +### Case study: CALL on Spark 4 + +The sharpest real example, verified against the Spark 4.1 classpath. On Spark 3.x with +Iceberg, `CALL` resolves to `org.apache.spark.sql.catalyst.plans.logical.Call` — a class +Iceberg injects, extending Catalyst's `Command`, so dispatch reaches the spec in +`table_command_spec.json` and authorizes the call. Spark 4.x ships *its own* `Call` under +the **identical fully-qualified class name** with a different type hierarchy: a `UnaryNode` +implementing `ExecutableDuringAnalysis` — not a `Command`. Dispatch never reached the spec +lookup; the procedure executed *during analysis*, before any optimizer-phase check could +run. The spec entry still existed and still named the right class; it was simply +unreachable. + +Three properties make this the canonical motivating case: + +- **Drift by type hierarchy under a stable class name.** Neither a class-name check nor the + presence of a spec entry detects it. +- **It is neither a `Command` nor a `LeafNode`.** An invariant scoped to those two shapes + would have missed it — which is why the invariant (§3) has its third clause, and why the + build-time enumeration (§6) covers `ExecutableDuringAnalysis`. +- **No runtime check in this plugin can stop it.** `RuleAuthorization` is an optimizer + rule; an analysis-time-executable node has already run by then. The build-time check is + the only net for this shape, and it currently reports `Call` as an **acknowledged, + tracked vulnerability** on Spark 4 (§6). Closing it requires an analysis-time rule or + blocking the operation on Spark 4 outright. + +## 3. The Invariant + +Naively flagging every fallthrough would be unusably noisy: intermediate operators +(`Project`, `Filter`, `Join`, …) legitimately recurse, because privileges are carried by +leaves and commands. The invariant paranoid mode enforces is narrower: + +> Every `Command`, every `LeafNode`, every node that can execute or mutate state outside +> the checked path (e.g. Spark 4's `ExecutableDuringAnalysis`), and every node whose class +> name has a spec that dispatch did not consult, encountered during privilege building — +> including in subtrees pruned by constant-projection elimination — must be either +> (a) matched by a spec, or (b) present on the explicit allowlist. Ordinary non-leaf query +> operators recurse freely. + +The honest caveat: "side-effecting non-leaf, non-command node" is not a category Catalyst +exposes as a single stable supertype. `ExecutableDuringAnalysis` covers the case we know +about; nothing guarantees a future Spark version won't introduce another. This is a core +reason the runtime deny mode cannot be replaced by build-time checks alone — the set of +dangerous shapes is open, so the only sound default is "unrecognized ⇒ deny," not +"unrecognized-and-matching-a-known-dangerous-supertype ⇒ deny." + +Additionally: + +> An extraction failure against a matched spec is a violation **when no descriptor of the +> command completes at all** (§5). A single descriptor failing while a sibling succeeds is +> expected version variance, not drift. + +## 4. Runtime Design + +### 4.1 Configuration + +``` +spark.kyuubi.authz.unclassifiedNode.behavior = allow | warn | deny (default: warn) +``` + +- **allow** — legacy behavior, for deployments that cannot tolerate new noise. Violations + are still counted and visible at DEBUG. +- **warn** — enforce nothing, but log at WARN once per (class name, violation kind) per + JVM, and count every occurrence. This is the accretion mode used to build the allowlist + from real workloads. +- **deny** — throw `AccessControlException` naming the unclassified class and the config + key. This is the mode documented for regulated deployments, and the mode every authz + test suite runs in (`SparkSessionProvider` sets it unconditionally). + +An invalid value fails loudly (`IllegalArgumentException`) rather than defaulting: a +security knob that silently absorbs typos is itself a fail-open. + +Violation kinds (see `ParanoidMode.ViolationKind`): unclassified command, unclassified +leaf, unresolved known scan, unreachable spec, analysis-time execution, extraction failure. + +### 4.2 Dispatch hardening + +`PrivilegesBuilder.build` gained a fallback arm: any node whose class name has a command +spec routes to `buildCommand` even if it is not a `Command` on this Spark version. This is +the direct, general fix for the CALL supertype-drift shape — the spec becomes reachable by +membership, not by supertype. (For `CALL` specifically it is still too late at optimizer +time, per §2; the arm exists so the *next* drift of this shape degrades to an ordinary +spec-driven check instead of silence.) + +### 4.3 The allowlist + +`known_harmless_spec.json` lives alongside the command spec files, is loaded the same way, +and is maintained by the same generator (`KnownHarmlessNodes` in the test tree, written by +`JsonSpecFileGenerator`). Entries are exact class names with a **required `reason` field** +— enforced by a `require` in `HarmlessNodeSpec` — so each entry is a reviewed decision an +auditor can read, not a reflexive silencing. Each entry also names the exact Spark minor +versions its review applies to (§4.5); on any other version the entry is inert. + +Two patterns emerged during triage that future entries should be checked against: + +- **"Reads no stored data"** — `LocalRelation`, `OneRowRelation`, `Range`, + `CTERelationRef`, `CommandResult`, session-conf commands. The straightforward kind. +- **"Enforced elsewhere"** — v2 `ShowNamespaces` / `ShowTables`. These are row-filtered by + the `ObjectFilterPlaceHolder` + `FilterDataSourceV2Strategy` machinery, but Spark's + `QueryExecution.eagerlyExecuteCommands` *also* executes the bare inner command in a + nested QueryExecution whose unfiltered result the placeholder deliberately discards + (`withNewChildInternal` refuses child swaps that change `nodeName`). That nested run hits + `RuleAuthorization` with the bare command as root and **must be allowed** — enforcement + lives in the placeholder machinery, not in `PrivilegesBuilder`. Denying it breaks every + SHOW query. This subtlety is load-bearing; do not "fix" it. + +Allowlisting a `Command` is higher-stakes than allowlisting a leaf relation, so it takes a +second, colocated review: the entry must also be exempted by name in +`ClassificationCoverageSuite`'s "not Commands in disguise" test, forcing every such +addition to touch the coverage suite where the reviewer sees the policy. + +`nodeName`-string matching is not extended: allowlist and specs key on fully qualified +class names only. The existing `nodeName == "UnresolvedRelation"` match stays and counts +as classified. + +### 4.4 Extraction-failure semantics (layer 3) + +Specs are written so an object "wins at least once" across Spark versions and command +shapes: a command may carry several descriptors for the same object, of which only one is +expected to succeed on any given version. Descriptors also fail legitimately by *shape* — +for Hudi's path-based `CALL` procedures, every table descriptor fails while the URI +descriptors carry the enforcement. + +The rule is therefore **per command, not per descriptor or per descriptor family**: a +violation is reported only when at least one descriptor threw and *no descriptor of the +whole command completed* (`DescOutcomes` in `PrivilegesBuilder`). That is the true drift +shape — the spec matches by name but can no longer extract anything from this Spark's plan +layout. + +Residual gap, accepted deliberately: partial drift (a broken table descriptor alongside a +still-working query descriptor) is not reported at runtime, because it is +indistinguishable from legitimate shape variance. The build-time checks are the net for +that class of drift. + +Two guardrails around the tracking: `AccessControlException` is always rethrown (an +authorization verdict bubbling up from nested privilege building must never be recorded as +an extraction failure), and recursion into extracted queries happens *outside* the tracked +region so nested violations surface as themselves. + +The same per-command logic applies to scan specs in `buildQuery` via +`ScanSpec.tablesWithFailures` / `urisWithFailures`: a matched scan that yields no table, no +URI, and at least one exception is a violation. + +### 4.5 Version-scoped audits (`verifiedSparkVersions`) + +The CALL case (§2) shows that "known" and "harmless" are assertions about a class *on a +specific Spark version*: the class under the same fully qualified name is free to become +something else in the next minor release. Every spec entry — allowlist and command/scan +specs alike — therefore carries a `verifiedSparkVersions` field naming the Spark versions +its review applies to. + +The field is an **explicit enumeration of exact `major.minor` pairs, never a range**. +Ranges invite boundary misreadings ("less than 4.0, exclusive" read as inclusive); a list +has no boundary to misread. The format is validated at construction (`SparkVersionAudit`), +and the enumeration gives the right default for free: a new Spark minor is unverified until +a human adds it — which is exactly when the re-review should happen. A version joins an +entry's list by being tested or reviewed, never by interpolation (this is why an entry can +legitimately list `3.3, 3.4, 3.5, 4.1` without `4.0`). + +The field's force differs by spec kind, deliberately: + +- **Allowlist entries gate.** An entry not verified for the running Spark's `major.minor` + is inert: the node counts as unclassified and paranoid mode applies (fail closed, + per node). The violation message says so explicitly — "verified for 3.3, 3.4, 3.5 but + not for 4.2; re-review the entry" is far more actionable than "unclassified". An + allowlist entry *grants silence*, so its scope must be exactly as wide as its review. +- **Command and scan spec entries are advisory.** A spec still engages on an unverified + version. A spec *imposes checks*, so staying active on an unaudited version is the safe + direction — going inert there would fail everything closed and make new Spark versions + unusable, while actual drift is caught by the extraction-failure tracking (§4.4) and the + build-time checks (§6). The metadata records what was audited when, and gives the + build-time tooling a place to grow (e.g. flagging specs engaged far outside their + audited range). + +The build-time enumeration (§6) respects the gate: on each profile, only allowlist entries +verified for that profile's Spark minor count as classified, so porting to a new Spark +version surfaces every entry awaiting re-review in that profile's backlog at PR time. + +## 5. Test Posture + +All authz test sessions run with `deny` set in `SparkSessionProvider` — fallout goes to a +spec or to the allowlist, never to relaxing the default. Behavior-specific tests live in +`ParanoidModeSuite` (synthetic unclassified leaf/command nodes, all three behaviors, the +constant-projection sweep, allowlist loading and the required-reason rule). + +Note a suite green in `deny` mode establishes coverage only of the plans the suite +exercises. It is the enumeration check (§6), not the test suite, that approximates closure +over the classpath. + +## 6. Build-Time Coverage Checks + +`ClassificationCoverageSuite`, run per Spark profile, catches classification drift at PR +time instead of in production: + +1. **Analysis-time reachability.** For every command spec classname resolvable on this + classpath, assert it does not implement `ExecutableDuringAnalysis` without being a + `Command` — the shape whose spec can never enforce anything (§2). Known instances live + in an explicit `acknowledgedGaps` set with tracking comments; an entry there is an + **acknowledged vulnerability on that Spark version, not a pass**. Currently: + `logical.Call` on Spark 4. +2. **Allowlist re-review.** Allowlisted classes that are (or became) `Command`s on this + classpath fail unless explicitly exempted in the suite (§4.3), and no class may have + both a spec and an allowlist entry. +3. **Enumeration.** Scan the jars that can contribute plan nodes (spark-catalyst, sql-core, + hive, plus whichever catalog plugins are on this profile's classpath), enumerate every + concrete class that is a `Command`, `LeafNode`, or `ExecutableDuringAnalysis`, and diff + against specs ∪ the allowlist entries verified for this profile's Spark minor (§4.5). + The unclassified remainder is pinned, one classname per line, in + `src/test/resources/classification_backlog_spark_.txt` (currently 136 entries + for 3.5, 181 for 4.1 — the 4.1 figure includes allowlist entries awaiting 4.x + re-review). A class **new** to the diff fails the build with an actionable + message: classify it, allowlist it with a reason, or consciously regenerate the backlog. + A class that leaves the diff must also leave the backlog, so the backlog only ever + shrinks by being triaged, never silently. + +Regeneration: `KYUUBI_UPDATE=1 build/mvn test -pl extensions/spark/kyuubi-spark-authz +-DwildcardSuites=org.apache.kyuubi.plugin.spark.authz.ClassificationCoverageSuite` (spec +and allowlist JSON via `dev/gen/gen_ranger_spec_json.sh`). Note the generator reads specs +from `target/classes`, so regenerate the allowlist first, then the backlog in a second +pass. + +These checks do **not** replace runtime paranoid mode, for two independent reasons: +build-time enumeration cannot see third-party catalog plugins loaded only in the user's +environment, and the set of dangerous node shapes is open (§3), so no fixed enumeration is +provably complete. + +## 7. Known Limitations and Follow-Ups + +- **Analysis-time execution is unreachable at runtime.** `CALL` on Spark 4 runs before the + optimizer; only check §6.1 names it. Closing the gap needs an analysis-time rule or a + hard block on Spark 4 — tracked via the `acknowledgedGaps` entry. +- **A startup Spark-major-version assertion** (refuse to initialize on an unsupported + Spark major) is a cheap, orthogonal hardening recommended for released branch lines: the + released jar today loads on Spark 4 and *visibly enforces* policies on ordinary + statements while silently allowing `CALL` — apparent enforcement plus a silent gap, in + exactly the deployment the docs disclaim. +- **Partial extractor drift** with a surviving sibling descriptor is not reported at + runtime (§4.4). +- **Row-filter and data-masking rule paths** (`RuleApplyRowFilter`, + `RuleApplyDataMaskingStage0/1`) share the recognition machinery but have their own + traversals; they are not covered by this change and should get the same treatment as a + follow-up. +- **Out of scope by design:** wrong (as opposed to missing) classification in existing + specs; physical-plan and RDD-level escape hatches (`df.rdd`, `ExternalRDD` / + `LogicalRDD` are allowlisted with exactly this reasoning); subquery-expression traversal. +- **The backlogs are triage lists, not archives.** 136 + 170 classes await a + spec-or-allowlist decision each; allowlist review must be part of every Spark version + bump, since a node harmless today can gain authz-relevant behavior in a later release. + +## 8. Open Questions + +1. Should `deny` eventually be the default? Fail-closed-by-default is the defensible + posture for a security plugin, but it changes behavior for every existing deployment. + Shipping with `warn` first is the safe rollout; revisit after a release of soak time. +2. Granularity: is a single behavior knob enough, or do the violation kinds need + independent settings? (Current position: one knob until real usage proves otherwise.) +3. This changes user-visible security posture, so it should land discuss-first: framed as + "deny-by-default mode for unclassified plan nodes," with a search of existing issues + and discussions for prior art before opening a new one. diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json index 5891fb1e548..96c44e0a5e6 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json @@ -8,7 +8,8 @@ "comment" : "" } ], "opType" : "ALTERDATABASE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CreateNamespace", "databaseDescs" : [ { @@ -40,7 +41,8 @@ "fieldExtractor" : "PropertiesLocationUriExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DescribeNamespace", "databaseDescs" : [ { @@ -51,7 +53,8 @@ "comment" : "" } ], "opType" : "DESCDATABASE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropNamespace", "databaseDescs" : [ { @@ -62,7 +65,8 @@ "comment" : "" } ], "opType" : "DROPDATABASE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.SetCatalogAndNamespace", "databaseDescs" : [ { @@ -89,7 +93,8 @@ "comment" : "" } ], "opType" : "SWITCHDATABASE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.SetNamespaceLocation", "databaseDescs" : [ { @@ -105,7 +110,8 @@ "fieldExtractor" : "StringURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.SetNamespaceProperties", "databaseDescs" : [ { @@ -116,7 +122,8 @@ "comment" : "" } ], "opType" : "ALTERDATABASE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterDatabasePropertiesCommand", "databaseDescs" : [ { @@ -127,7 +134,8 @@ "comment" : "" } ], "opType" : "ALTERDATABASE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterDatabaseSetLocationCommand", "databaseDescs" : [ { @@ -143,7 +151,8 @@ "fieldExtractor" : "StringURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AnalyzeTablesCommand", "databaseDescs" : [ { @@ -154,7 +163,8 @@ "comment" : "" } ], "opType" : "ANALYZE_TABLE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateDatabaseCommand", "databaseDescs" : [ { @@ -170,7 +180,8 @@ "fieldExtractor" : "StringURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.DescribeDatabaseCommand", "databaseDescs" : [ { @@ -181,7 +192,8 @@ "comment" : "" } ], "opType" : "DESCDATABASE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.DropDatabaseCommand", "databaseDescs" : [ { @@ -192,7 +204,8 @@ "comment" : "" } ], "opType" : "DROPDATABASE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.SetDatabaseCommand", "databaseDescs" : [ { @@ -203,7 +216,8 @@ "comment" : "" } ], "opType" : "SWITCHDATABASE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.SetNamespaceCommand", "databaseDescs" : [ { @@ -214,5 +228,6 @@ "comment" : "" } ], "opType" : "SWITCHDATABASE", - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] } ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json index b0da1a95199..677cd54db7d 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json @@ -31,7 +31,8 @@ "isInput" : false, "comment" : "" } ], - "opType" : "CREATEFUNCTION" + "opType" : "CREATEFUNCTION", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.DescribeFunctionCommand", "functionDescs" : [ { @@ -59,7 +60,8 @@ "isInput" : true, "comment" : "" } ], - "opType" : "DESCFUNCTION" + "opType" : "DESCFUNCTION", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.DropFunctionCommand", "functionDescs" : [ { @@ -93,7 +95,8 @@ "isInput" : false, "comment" : "" } ], - "opType" : "DROPFUNCTION" + "opType" : "DROPFUNCTION", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.RefreshFunctionCommand", "functionDescs" : [ { @@ -117,5 +120,6 @@ "isInput" : false, "comment" : "" } ], - "opType" : "RELOADFUNCTION" + "opType" : "RELOADFUNCTION", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] } ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json new file mode 100644 index 00000000000..fe698cbf48c --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json @@ -0,0 +1,61 @@ +[ { + "classname" : "org.apache.spark.sql.catalyst.analysis.ResolvedNamespace", + "reason" : "Analysis-time resolution artifact naming a namespace; reads no data itself, and the commands resolved over it are classified in their own right", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.CTERelationRef", + "reason" : "Leaf reference to a CTE definition; the definition's own plan appears under WithCTE in the same tree and is authorized there", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.CommandResult", + "reason" : "Holds rows already produced by an eagerly executed command; that command was authorized when it executed", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.LocalRelation", + "reason" : "Holds in-memory literal rows (VALUES lists, createDataFrame); reads no stored data", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.1" ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.NoopCommand", + "reason" : "Spark's placeholder for commands with nothing to do (e.g. IF EXISTS / IF NOT EXISTS variants when the object is absent); executes nothing", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.OneRowRelation", + "reason" : "The implicit single-row relation backing SELECT without FROM; reads no stored data", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.1" ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.Range", + "reason" : "Generates rows from a numeric range (e.g. spark.range); reads no stored data", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.1" ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.ShowNamespaces", + "reason" : "Enforced elsewhere: results are row-filtered per namespace by ObjectFilterPlaceHolder + FilterDataSourceV2Strategy; Spark eagerly executes the bare command in a nested QueryExecution whose unfiltered result the placeholder discards", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.ShowTables", + "reason" : "Enforced elsewhere: results are row-filtered per table by ObjectFilterPlaceHolder + FilterDataSourceV2Strategy; Spark eagerly executes the bare command in a nested QueryExecution whose unfiltered result the placeholder discards", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.execution.ExternalRDD", + "reason" : "Wraps a session-created RDD/local collection (e.g. spark.createDataset, and Delta's internal VACUUM plumbing); RDD-level access is outside the plugin's scope and is an existing, separate concern", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.execution.LogicalRDD", + "reason" : "Wraps a pre-existing RDD; RDD-level access is outside the plugin's scope and is an existing, separate concern", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.execution.columnar.InMemoryRelation", + "reason" : "Cached query results; the originating plan was authorized when the cache was populated", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.execution.command.DropTempViewCommand", + "reason" : "Operates only on session-local temporary views, which are deliberately not authz resources (their reads are authorized against the underlying tables); see KYUUBI #3426", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.execution.command.ResetCommand", + "reason" : "Resets session configuration only; sensitive configs are separately guarded by AuthzConfigurationChecker", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.execution.command.SetCommand", + "reason" : "Sets session configuration only; sensitive configs are separately guarded by AuthzConfigurationChecker", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +} ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json index 1145adbe07a..0aabde3ff2a 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json @@ -7,7 +7,8 @@ "comment" : "" } ], "functionDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.catalog.HiveTableRelation", "scanDescs" : [ { @@ -17,7 +18,8 @@ "comment" : "" } ], "functionDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.datasources.LogicalRelation", "scanDescs" : [ { @@ -32,7 +34,8 @@ "fieldExtractor" : "BaseRelationFileIndexURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation", "scanDescs" : [ { @@ -42,7 +45,8 @@ "comment" : "" } ], "functionDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hive.HiveGenericUDF", "scanDescs" : [ ], @@ -59,7 +63,8 @@ "isInput" : true, "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hive.HiveGenericUDTF", "scanDescs" : [ ], @@ -76,7 +81,8 @@ "isInput" : true, "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hive.HiveSimpleUDF", "scanDescs" : [ ], @@ -93,7 +99,8 @@ "isInput" : true, "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hive.HiveUDAFFunction", "scanDescs" : [ ], @@ -110,5 +117,6 @@ "isInput" : true, "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] } ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json index 50c1ee40de3..e10eb002556 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json @@ -18,7 +18,8 @@ "fieldExtractor" : "ResolvedTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AddPartitions", "tableDescs" : [ { @@ -34,7 +35,8 @@ } ], "opType" : "ALTERTABLE_ADDPARTS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AlterColumn", "tableDescs" : [ { @@ -55,7 +57,8 @@ "fieldExtractor" : "ResolvedTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AlterColumns", "tableDescs" : [ { @@ -97,7 +100,8 @@ "fieldExtractor" : "IdentifierURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AppendData", "tableDescs" : [ { @@ -127,7 +131,8 @@ "fieldExtractor" : "DataSourceV2RelationURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CacheTable", "tableDescs" : [ ], @@ -137,7 +142,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CacheTableAsSelect", "tableDescs" : [ ], @@ -147,7 +153,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CommentOnTable", "tableDescs" : [ { @@ -163,7 +170,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CreateTable", "tableDescs" : [ { @@ -218,7 +226,8 @@ "fieldExtractor" : "IdentifierURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CreateTableAsSelect", "tableDescs" : [ { @@ -272,7 +281,8 @@ "fieldExtractor" : "PropertiesLocationUriExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CreateV2Table", "tableDescs" : [ { @@ -302,7 +312,8 @@ "fieldExtractor" : "IdentifierURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DeleteFromTable", "tableDescs" : [ { @@ -323,7 +334,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DescribeRelation", "tableDescs" : [ { @@ -339,7 +351,8 @@ } ], "opType" : "DESCTABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropColumns", "tableDescs" : [ { @@ -360,7 +373,8 @@ "fieldExtractor" : "ResolvedTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropPartitions", "tableDescs" : [ { @@ -376,7 +390,8 @@ } ], "opType" : "ALTERTABLE_DROPPARTS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropTable", "tableDescs" : [ { @@ -402,7 +417,8 @@ } ], "opType" : "DROPTABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.MergeIntoTable", "tableDescs" : [ { @@ -427,7 +443,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.OverwriteByExpression", "tableDescs" : [ { @@ -457,7 +474,8 @@ "fieldExtractor" : "DataSourceV2RelationURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.OverwritePartitionsDynamic", "tableDescs" : [ { @@ -487,7 +505,8 @@ "fieldExtractor" : "DataSourceV2RelationURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.RefreshTable", "tableDescs" : [ { @@ -503,7 +522,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.RenameColumn", "tableDescs" : [ { @@ -524,7 +544,8 @@ "fieldExtractor" : "ResolvedTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.RenamePartitions", "tableDescs" : [ { @@ -540,7 +561,8 @@ } ], "opType" : "ALTERTABLE_RENAMEPART", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.RenameTable", "tableDescs" : [ { @@ -556,7 +578,8 @@ } ], "opType" : "ALTERTABLE_RENAME", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.RepairTable", "tableDescs" : [ { @@ -572,7 +595,8 @@ } ], "opType" : "MSCK", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ReplaceColumns", "tableDescs" : [ { @@ -593,7 +617,8 @@ "fieldExtractor" : "ResolvedTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ReplaceData", "tableDescs" : [ { @@ -618,7 +643,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ReplaceTable", "tableDescs" : [ { @@ -673,7 +699,8 @@ "fieldExtractor" : "IdentifierURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ReplaceTableAsSelect", "tableDescs" : [ { @@ -727,7 +754,8 @@ "fieldExtractor" : "PropertiesLocationUriExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.SetTableProperties", "tableDescs" : [ { @@ -748,7 +776,8 @@ "fieldExtractor" : "ResolvedTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ShowCreateTable", "tableDescs" : [ { @@ -764,7 +793,8 @@ } ], "opType" : "SHOW_CREATETABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ShowTableProperties", "tableDescs" : [ { @@ -780,7 +810,8 @@ } ], "opType" : "SHOW_TBLPROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.TruncatePartition", "tableDescs" : [ { @@ -796,7 +827,8 @@ } ], "opType" : "ALTERTABLE_DROPPARTS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.TruncateTable", "tableDescs" : [ { @@ -812,7 +844,8 @@ } ], "opType" : "TRUNCATETABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.UnsetTableProperties", "tableDescs" : [ { @@ -828,7 +861,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.UpdateTable", "tableDescs" : [ { @@ -849,7 +883,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AddArchivesCommand", "tableDescs" : [ ], @@ -860,7 +895,8 @@ "fieldExtractor" : "StringSeqURIExtractor", "isInput" : true, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AddFilesCommand", "tableDescs" : [ ], @@ -871,7 +907,8 @@ "fieldExtractor" : "StringSeqURIExtractor", "isInput" : true, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AddJarCommand", "tableDescs" : [ ], @@ -882,7 +919,8 @@ "fieldExtractor" : "StringURIExtractor", "isInput" : true, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AddJarsCommand", "tableDescs" : [ ], @@ -893,7 +931,8 @@ "fieldExtractor" : "StringSeqURIExtractor", "isInput" : true, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableAddColumnsCommand", "tableDescs" : [ { @@ -913,7 +952,8 @@ } ], "opType" : "ALTERTABLE_ADDCOLS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableAddPartitionCommand", "tableDescs" : [ { @@ -938,7 +978,8 @@ "fieldExtractor" : "PartitionLocsSeqURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableChangeColumnCommand", "tableDescs" : [ { @@ -958,7 +999,8 @@ } ], "opType" : "ALTERTABLE_REPLACECOLS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableDropPartitionCommand", "tableDescs" : [ { @@ -978,7 +1020,8 @@ } ], "opType" : "ALTERTABLE_DROPPARTS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableRenameCommand", "tableDescs" : [ { @@ -999,7 +1042,8 @@ } ], "opType" : "ALTERTABLE_RENAME", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableRenamePartitionCommand", "tableDescs" : [ { @@ -1019,7 +1063,8 @@ } ], "opType" : "ALTERTABLE_RENAMEPART", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableSerDePropertiesCommand", "tableDescs" : [ { @@ -1039,7 +1084,8 @@ } ], "opType" : "ALTERTABLE_SERDEPROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableSetLocationCommand", "tableDescs" : [ { @@ -1064,7 +1110,8 @@ "fieldExtractor" : "StringURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableSetPropertiesCommand", "tableDescs" : [ { @@ -1080,7 +1127,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableUnsetPropertiesCommand", "tableDescs" : [ { @@ -1096,7 +1144,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterViewAsCommand", "tableDescs" : [ { @@ -1121,7 +1170,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AnalyzeColumnCommand", "tableDescs" : [ { @@ -1165,7 +1215,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AnalyzePartitionCommand", "tableDescs" : [ { @@ -1195,7 +1246,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.AnalyzeTableCommand", "tableDescs" : [ { @@ -1221,7 +1273,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.CacheTableCommand", "tableDescs" : [ ], @@ -1231,7 +1284,8 @@ "fieldExtractor" : "LogicalPlanOptionQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand", "tableDescs" : [ { @@ -1256,7 +1310,8 @@ "fieldExtractor" : "CatalogTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateDataSourceTableCommand", "tableDescs" : [ { @@ -1277,7 +1332,8 @@ "fieldExtractor" : "CatalogTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateTableCommand", "tableDescs" : [ { @@ -1298,7 +1354,8 @@ "fieldExtractor" : "CatalogTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateTableLikeCommand", "tableDescs" : [ { @@ -1329,7 +1386,8 @@ "fieldExtractor" : "CatalogStorageFormatURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateViewCommand", "tableDescs" : [ { @@ -1358,7 +1416,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.DescribeColumnCommand", "tableDescs" : [ { @@ -1378,7 +1437,8 @@ } ], "opType" : "DESCTABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.DescribeTableCommand", "tableDescs" : [ { @@ -1398,7 +1458,8 @@ } ], "opType" : "DESCTABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.DropTableCommand", "tableDescs" : [ { @@ -1419,7 +1480,8 @@ } ], "opType" : "DROPTABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.InsertIntoDataSourceDirCommand", "tableDescs" : [ ], @@ -1430,7 +1492,8 @@ "fieldExtractor" : "CatalogStorageFormatURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.LoadDataCommand", "tableDescs" : [ { @@ -1460,7 +1523,8 @@ "fieldExtractor" : "StringURIExtractor", "isInput" : true, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.RefreshTableCommand", "tableDescs" : [ { @@ -1476,7 +1540,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.RepairTableCommand", "tableDescs" : [ { @@ -1492,7 +1557,8 @@ } ], "opType" : "MSCK", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowColumnsCommand", "tableDescs" : [ { @@ -1508,7 +1574,8 @@ } ], "opType" : "SHOWCOLUMNS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowCreateTableAsSerdeCommand", "tableDescs" : [ { @@ -1524,7 +1591,8 @@ } ], "opType" : "SHOW_CREATETABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowCreateTableCommand", "tableDescs" : [ { @@ -1540,7 +1608,8 @@ } ], "opType" : "SHOW_CREATETABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowPartitionsCommand", "tableDescs" : [ { @@ -1560,7 +1629,8 @@ } ], "opType" : "SHOWPARTITIONS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowTablePropertiesCommand", "tableDescs" : [ { @@ -1576,7 +1646,8 @@ } ], "opType" : "SHOW_TBLPROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.command.TruncateTableCommand", "tableDescs" : [ { @@ -1596,7 +1667,8 @@ } ], "opType" : "TRUNCATETABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.datasources.CreateTable", "tableDescs" : [ { @@ -1621,13 +1693,15 @@ "fieldExtractor" : "CatalogTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.datasources.CreateTempViewUsing", "tableDescs" : [ ], "opType" : "CREATEVIEW", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.datasources.InsertIntoDataSourceCommand", "tableDescs" : [ { @@ -1648,7 +1722,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand", "tableDescs" : [ { @@ -1673,7 +1748,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.datasources.RefreshTable", "tableDescs" : [ { @@ -1689,7 +1765,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand", "tableDescs" : [ ], @@ -1704,7 +1781,8 @@ "fieldExtractor" : "PropertiesPathUriExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hive.execution.CreateHiveTableAsSelectCommand", "tableDescs" : [ { @@ -1733,7 +1811,8 @@ "fieldExtractor" : "CatalogTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hive.execution.InsertIntoHiveDirCommand", "tableDescs" : [ ], @@ -1748,7 +1827,8 @@ "fieldExtractor" : "CatalogStorageFormatURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hive.execution.InsertIntoHiveTable", "tableDescs" : [ { @@ -1773,7 +1853,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hive.execution.OptimizedCreateHiveTableAsSelectCommand", "tableDescs" : [ { @@ -1802,7 +1883,8 @@ "fieldExtractor" : "CatalogTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AddPartitionField", "tableDescs" : [ { @@ -1818,7 +1900,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.Call", "tableDescs" : [ { @@ -1834,7 +1917,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceBranch", "tableDescs" : [ { @@ -1850,7 +1934,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceTag", "tableDescs" : [ { @@ -1866,7 +1951,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable", "tableDescs" : [ { @@ -1887,7 +1973,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropBranch", "tableDescs" : [ { @@ -1903,7 +1990,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields", "tableDescs" : [ { @@ -1919,7 +2007,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropPartitionField", "tableDescs" : [ { @@ -1935,7 +2024,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropTag", "tableDescs" : [ { @@ -1951,7 +2041,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable", "tableDescs" : [ { @@ -1976,7 +2067,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField", "tableDescs" : [ { @@ -1992,7 +2084,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields", "tableDescs" : [ { @@ -2008,7 +2101,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering", "tableDescs" : [ { @@ -2024,7 +2118,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.UnresolvedMergeIntoIcebergTable", "tableDescs" : [ { @@ -2049,7 +2144,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable", "tableDescs" : [ { @@ -2070,7 +2166,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.AlterHoodieTableAddColumnsCommand", "tableDescs" : [ { @@ -2090,7 +2187,8 @@ } ], "opType" : "ALTERTABLE_ADDCOLS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.AlterHoodieTableChangeColumnCommand", "tableDescs" : [ { @@ -2110,7 +2208,8 @@ } ], "opType" : "ALTERTABLE_REPLACECOLS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.AlterHoodieTableDropPartitionCommand", "tableDescs" : [ { @@ -2130,7 +2229,8 @@ } ], "opType" : "ALTERTABLE_DROPPARTS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.AlterHoodieTableRenameCommand", "tableDescs" : [ { @@ -2151,7 +2251,8 @@ } ], "opType" : "ALTERTABLE_RENAME", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.AlterTableCommand", "tableDescs" : [ { @@ -2167,7 +2268,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.CallProcedureHoodieCommand", "tableDescs" : [ { @@ -2213,7 +2315,8 @@ "fieldExtractor" : "HudiCallProcedureOutputUriExtractor", "isInput" : false, "comment" : "Hudi" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.CompactionHoodiePathCommand", "tableDescs" : [ ], @@ -2224,7 +2327,8 @@ "fieldExtractor" : "StringURIExtractor", "isInput" : false, "comment" : "Hudi" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.CompactionHoodieTableCommand", "tableDescs" : [ { @@ -2240,7 +2344,8 @@ } ], "opType" : "CREATETABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.CompactionShowHoodiePathCommand", "tableDescs" : [ ], @@ -2251,7 +2356,8 @@ "fieldExtractor" : "StringURIExtractor", "isInput" : true, "comment" : "Hudi" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.CompactionShowHoodieTableCommand", "tableDescs" : [ { @@ -2267,7 +2373,8 @@ } ], "opType" : "SHOW_TBLPROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableAsSelectCommand", "tableDescs" : [ { @@ -2287,7 +2394,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableCommand", "tableDescs" : [ { @@ -2303,7 +2411,8 @@ } ], "opType" : "CREATETABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableLikeCommand", "tableDescs" : [ { @@ -2329,7 +2438,8 @@ } ], "opType" : "CREATETABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.CreateIndexCommand", "tableDescs" : [ { @@ -2345,7 +2455,8 @@ } ], "opType" : "CREATEINDEX", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.DeleteHoodieTableCommand", "tableDescs" : [ { @@ -2381,7 +2492,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.DropHoodieTableCommand", "tableDescs" : [ { @@ -2402,7 +2514,8 @@ } ], "opType" : "DROPTABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.DropIndexCommand", "tableDescs" : [ { @@ -2418,7 +2531,8 @@ } ], "opType" : "DROPINDEX", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand", "tableDescs" : [ { @@ -2443,7 +2557,8 @@ "fieldExtractor" : "LogicalPlanQueryExtractor", "comment" : "" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.MergeIntoHoodieTableCommand", "tableDescs" : [ { @@ -2468,7 +2583,8 @@ "fieldExtractor" : "HudiMergeIntoSourceTableExtractor", "comment" : "Hudi" } ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.RefreshIndexCommand", "tableDescs" : [ { @@ -2484,7 +2600,8 @@ } ], "opType" : "ALTERINDEX_REBUILD", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.RepairHoodieTableCommand", "tableDescs" : [ { @@ -2500,7 +2617,8 @@ } ], "opType" : "MSCK", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.ShowHoodieTablePartitionsCommand", "tableDescs" : [ { @@ -2520,7 +2638,8 @@ } ], "opType" : "SHOWPARTITIONS", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.ShowIndexesCommand", "tableDescs" : [ { @@ -2536,7 +2655,8 @@ } ], "opType" : "SHOWINDEXES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.Spark31AlterTableCommand", "tableDescs" : [ { @@ -2552,7 +2672,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.TruncateHoodieTableCommand", "tableDescs" : [ { @@ -2572,7 +2693,8 @@ } ], "opType" : "TRUNCATETABLE", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.hudi.command.UpdateHoodieTableCommand", "tableDescs" : [ { @@ -2593,7 +2715,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "io.delta.tables.execution.VacuumTableCommand", "tableDescs" : [ { @@ -2634,7 +2757,8 @@ "fieldExtractor" : "StringURIExtractor", "isInput" : false, "comment" : "Delta" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.delta.commands.DeleteCommand", "tableDescs" : [ { @@ -2660,7 +2784,8 @@ "fieldExtractor" : "SubqueryAliasURIExtractor", "isInput" : false, "comment" : "Delta" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.delta.commands.MergeIntoCommand", "tableDescs" : [ { @@ -2690,7 +2815,8 @@ "fieldExtractor" : "SubqueryAliasURIExtractor", "isInput" : false, "comment" : "Delta" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.delta.commands.OptimizeTableCommand", "tableDescs" : [ { @@ -2731,7 +2857,8 @@ "fieldExtractor" : "StringURIExtractor", "isInput" : false, "comment" : "Delta" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.spark.sql.delta.commands.UpdateCommand", "tableDescs" : [ { @@ -2757,7 +2884,8 @@ "fieldExtractor" : "SubqueryAliasURIExtractor", "isInput" : false, "comment" : "Delta" - } ] + } ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.paimon.spark.catalyst.plans.logical.PaimonCallCommand", "tableDescs" : [ { @@ -2773,7 +2901,8 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.paimon.spark.commands.DeleteFromPaimonTableCommand", "tableDescs" : [ { @@ -2794,7 +2923,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.paimon.spark.commands.MergeIntoPaimonTable", "tableDescs" : [ { @@ -2825,7 +2955,8 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] }, { "classname" : "org.apache.paimon.spark.commands.UpdatePaimonTableCommand", "tableDescs" : [ { @@ -2846,5 +2977,6 @@ } ], "opType" : "QUERY", "queryDescs" : [ ], - "uriDescs" : [ ] + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] } ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ParanoidMode.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ParanoidMode.scala new file mode 100644 index 00000000000..325a0f218d5 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ParanoidMode.scala @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.LongAdder + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.slf4j.LoggerFactory + +/** + * Handling for plan nodes that fall outside the plugin's recognition machinery + * ("paranoid mode"). + * + * The privilege builder classifies plan nodes by pattern-matching against the command, + * scan and function spec files. Historically a node that fell through every match was + * silently treated as not-authz-relevant, i.e. the plugin failed open. This object + * centralizes the policy applied when such a node is encountered: + * + * - `allow`: legacy behavior, log at DEBUG only + * - `warn`: log at WARN once per (class name, violation kind) per JVM (default) + * - `deny`: throw [[AccessControlException]], failing the query closed + * + * configured via `spark.kyuubi.authz.unclassifiedNode.behavior`. + * + * Nodes that are genuinely not authz-relevant are declared in + * `known_harmless_spec.json`, each with a human-reviewed reason. + */ +object ParanoidMode { + + final private val LOG = LoggerFactory.getLogger(getClass) + + final val UNCLASSIFIED_NODE_BEHAVIOR_KEY = "spark.kyuubi.authz.unclassifiedNode.behavior" + + object Behavior extends Enumeration { + type Behavior = Value + val ALLOW: Value = Value("allow") + val WARN: Value = Value("warn") + val DENY: Value = Value("deny") + } + + /** The kinds of classification violations, used in log/error messages and dedup keys. */ + object ViolationKind extends Enumeration { + type ViolationKind = Value + + /** A `Command` with no entry in any command spec file (fail-open layer 1). */ + val UNCLASSIFIED_COMMAND: Value = Value("unclassified command") + + /** A leaf plan node not matched by any scan spec (fail-open layer 2). */ + val UNCLASSIFIED_LEAF: Value = Value("unclassified leaf node") + + /** A node matched by a scan spec but not resolved, so no privileges were extracted. */ + val UNRESOLVED_SCAN: Value = Value("unresolved scan node") + + /** + * A node whose class name has a command spec entry, encountered on the query path: + * the spec exists but the dispatch never reaches it (the CALL-on-Spark-4 shape). + */ + val UNREACHABLE_SPEC: Value = Value("command spec not reachable from dispatch") + + /** + * A node that executes during analysis (e.g. Spark 4's `ExecutableDuringAnalysis`): + * by the time authorization rules run it may already have produced side effects. + */ + val ANALYSIS_TIME_EXECUTION: Value = Value("node executes during analysis") + + /** + * A matched spec whose extractors all failed against the current plan shape + * (fail-open layer 3, typically Spark version drift). + */ + val EXTRACTION_FAILURE: Value = Value("spec matched but extraction failed") + } + + import Behavior._ + import ViolationKind.ViolationKind + + def behavior(spark: SparkSession): Behavior = { + val raw = spark.conf.getOption(UNCLASSIFIED_NODE_BEHAVIOR_KEY).getOrElse(WARN.toString) + Behavior.values.find(_.toString.equalsIgnoreCase(raw.trim)).getOrElse { + throw new IllegalArgumentException( + s"Invalid value '$raw' for $UNCLASSIFIED_NODE_BEHAVIOR_KEY," + + s" expected one of: ${Behavior.values.mkString(", ")}") + } + } + + // WARN-mode logging is deduplicated on (plan class name, violation kind) per JVM + private val warned = ConcurrentHashMap.newKeySet[(String, ViolationKind)]() + + // Violation counts by kind, kept regardless of behavior. Exposed for tests and as a + // cheap metric hook until a proper metrics source is wired up. + private[authz] val violationCounts = new ConcurrentHashMap[ViolationKind, LongAdder]() + + private[authz] def violationCount(kind: ViolationKind): Long = { + Option(violationCounts.get(kind)).map(_.sum()).getOrElse(0L) + } + + private[authz] def resetForTesting(): Unit = { + warned.clear() + violationCounts.clear() + } + + /** + * Report a plan node that the privilege builder could not classify, applying the + * configured behavior. In `deny` mode this throws and the query fails closed. + */ + def onViolation( + spark: SparkSession, + plan: LogicalPlan, + kind: ViolationKind, + detail: String = "", + cause: Option[Throwable] = None): Unit = { + violationCounts.computeIfAbsent(kind, _ => new LongAdder).increment() + + val classname = plan.getClass.getName + def message: String = { + val detailPart = if (detail.nonEmpty) s"; $detail" else "" + val causePart = cause.map(e => s"; cause: $e").getOrElse("") + s"Plan node $classname is not covered by authorization: $kind$detailPart$causePart." + + s" Classify it with a command/scan spec, or add it to known_harmless_spec.json" + + s" with a reason if it is not authz-relevant." + } + + behavior(spark) match { + case ALLOW => + if (LOG.isDebugEnabled) { + LOG.debug(message) + } + case WARN => + if (warned.add((classname, kind))) { + LOG.warn(s"$message (Further occurrences of this class will not be logged." + + s" Set $UNCLASSIFIED_NODE_BEHAVIOR_KEY=deny to fail closed.)") + } + case DENY => + throw new AccessControlException( + s"$message ($UNCLASSIFIED_NODE_BEHAVIOR_KEY=deny)", + cause) + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala index e83367386df..65965104c36 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.execution.command.ExplainCommand import org.slf4j.LoggerFactory import org.apache.kyuubi.plugin.spark.authz.OperationType.OperationType +import org.apache.kyuubi.plugin.spark.authz.ParanoidMode.ViolationKind import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType._ import org.apache.kyuubi.plugin.spark.authz.rule.Authorization._ import org.apache.kyuubi.plugin.spark.authz.rule.rowfilter._ @@ -84,13 +85,23 @@ object PrivilegesBuilder { case p if p.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty => case scan if isKnownScan(scan) && scan.resolved => - val tables = getScanSpec(scan).tables(scan, spark) + val spec = getScanSpec(scan) + val (tables, tableFailures) = spec.tablesWithFailures(scan, spark) // If the the scan is table-based, we check privileges on the table we found // otherwise, we check privileges on the uri we found if (tables.nonEmpty) { tables.foreach(mergeProjection(_, scan)) } else { - getScanSpec(scan).uris(scan).foreach(privilegeObjects += PrivilegeObject(_)) + val (uris, uriFailures) = spec.urisWithFailures(scan) + uris.foreach(privilegeObjects += PrivilegeObject(_)) + if (uris.isEmpty && (tableFailures.nonEmpty || uriFailures.nonEmpty)) { + ParanoidMode.onViolation( + spark, + scan, + ViolationKind.EXTRACTION_FAILURE, + "scan spec matched but no table or uri could be extracted", + (tableFailures ++ uriFailures).headOption) + } } case u if u.nodeName == "UnresolvedRelation" => @@ -100,6 +111,7 @@ object PrivilegesBuilder { privilegeObjects += PrivilegeObject(table) case p => + checkUnclassifiedQueryNode(p, spark) for (child <- p.children) { // If current plan's references don't have relation to it's input, have two cases // 1. `MapInPandas`, `ScriptTransformation` @@ -113,6 +125,11 @@ object PrivilegesBuilder { p.inputSet.map(_.toAttribute).toSeq, Nil, spark) + } else { + // The subtree is pruned for privilege building (a constant projection reads + // no columns), but it still executes, so classification must still see it: + // an unclassified node must not hide under `SELECT FROM ...`. + sweepClassificationOnly(child, spark) } } else { buildQuery( @@ -132,6 +149,100 @@ object PrivilegesBuilder { } } + /** + * Classification checks for nodes on the query path. Ordinary non-leaf operators + * (Project, Filter, Join, ...) recurse freely (privileges are carried by leaves and + * commands) but a node matching any of the shapes below would otherwise be silently + * treated as not-authz-relevant, and [[ParanoidMode]] decides how loud that is. + * + * If the class has an allowlist entry that simply is not verified for the running Spark + * version, say so: "re-review the entry" is far more actionable than "unclassified". + */ + private def unverifiedAllowlistDetail(classname: String): String = { + KNOWN_HARMLESS_NODES.get(classname).map { spec => + s"""its known_harmless_spec.json entry is verified for + | Spark ${spec.verifiedSparkVersions.mkString(", ")} + | but not for $SPARK_RUNTIME_MAJOR_MINOR - + | re-review the entry for this version""".stripMargin + }.getOrElse("") + } + + private def checkUnclassifiedQueryNode(p: LogicalPlan, spark: SparkSession): Unit = { + if (isKnownHarmless(p)) { + return + } + val detail = unverifiedAllowlistDetail(p.getClass.getName) + if (hasCommandSpec(p.getClass.getName)) { + // The spec exists but dispatch never consulted it — the class kept its name but + // changed supertype, like CALL between Spark 3 (Iceberg's Command) and Spark 4. + ParanoidMode.onViolation(spark, p, ViolationKind.UNREACHABLE_SPEC, detail) + } else if (executesDuringAnalysis(p)) { + ParanoidMode.onViolation(spark, p, ViolationKind.ANALYSIS_TIME_EXECUTION, detail) + } else if (isKnownScan(p)) { + // a known scan only reaches the generic arm when unresolved, contributing nothing + ParanoidMode.onViolation(spark, p, ViolationKind.UNRESOLVED_SCAN, detail) + } else if (p.children.isEmpty) { + ParanoidMode.onViolation(spark, p, ViolationKind.UNCLASSIFIED_LEAF, detail) + } + } + + /** + * Walk a subtree that privilege building skips, applying only the classification checks. + * Contributes no privilege objects; nodes are classified and traversal pruned exactly as + * [[buildQuery]] would (no descent below checked, scan, or unresolved-relation nodes). + */ + private def sweepClassificationOnly(plan: LogicalPlan, spark: SparkSession): Unit = { + plan match { + case p if p.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty => + case scan if isKnownScan(scan) && scan.resolved => + case u if u.nodeName == "UnresolvedRelation" => + case p => + checkUnclassifiedQueryNode(p, spark) + p.children.foreach(sweepClassificationOnly(_, spark)) + } + } + + /** + * Tracks descriptor outcomes across all families (table/database/uri/query/function) of + * one matched command spec. Individual descriptors failing is expected version variance: + * specs are written so an object "wins at least once" across Spark versions and command + * shapes (e.g. table descs legitimately all fail for a path-based procedure whose uri + * descs succeed). What must not pass silently is the drift shape where *no* descriptor of + * the command completes at all: the spec matches by name but can no longer extract + * anything from this Spark's plan shape (fail-open layer 3). + */ + private class DescOutcomes(plan: LogicalPlan, spark: SparkSession) { + private var succeeded = 0 + private val failures = ArrayBuffer[Exception]() + + def run[D <: Descriptor](descs: Seq[D])(run: D => Unit): Unit = { + descs.foreach { d => + try { + run(d) + succeeded += 1 + } catch { + // an authorization decision bubbling up from nested privilege building is a + // verdict, not an extraction failure — never swallow it + case e: AccessControlException => throw e + case e: Exception => + LOG.debug(d.error(plan, e)) + failures += e + } + } + } + + def reportIfAllFailed(): Unit = { + if (succeeded == 0 && failures.nonEmpty) { + ParanoidMode.onViolation( + spark, + plan, + ViolationKind.EXTRACTION_FAILURE, + "no descriptor of the matched command spec completed extraction", + failures.headOption) + } + } + } + /** * Build PrivilegeObjects from Spark LogicalPlan * @param plan a Spark LogicalPlan used to generate Spark PrivilegeObjects @@ -145,109 +256,104 @@ object PrivilegesBuilder { spark: SparkSession): OperationType = { def getTablePriv(tableDesc: TableDesc): Seq[PrivilegeObject] = { - try { - val maybeTable = tableDesc.extract(plan, spark) - maybeTable match { - case Some(table) => - val newTable = if (tableDesc.setCurrentDatabaseIfMissing) { - setCurrentDBIfNecessary(table, spark) - } else { - table - } - if (tableDesc.tableTypeDesc.exists(_.skip(plan))) { - Nil - } else { - val actionType = tableDesc.actionTypeDesc.map(_.extract(plan)).getOrElse(OTHER) - val columnNames = tableDesc.columnDesc.map(_.extract(plan)).getOrElse(Nil) - Seq(PrivilegeObject(newTable, columnNames, actionType)) - } - case None => Nil - } - } catch { - case e: Exception => - LOG.debug(tableDesc.error(plan, e)) - Nil + val maybeTable = tableDesc.extract(plan, spark) + maybeTable match { + case Some(table) => + val newTable = if (tableDesc.setCurrentDatabaseIfMissing) { + setCurrentDBIfNecessary(table, spark) + } else { + table + } + if (tableDesc.tableTypeDesc.exists(_.skip(plan))) { + Nil + } else { + val actionType = tableDesc.actionTypeDesc.map(_.extract(plan)).getOrElse(OTHER) + val columnNames = tableDesc.columnDesc.map(_.extract(plan)).getOrElse(Nil) + Seq(PrivilegeObject(newTable, columnNames, actionType)) + } + case None => Nil } } plan.getClass.getName match { case classname if DB_COMMAND_SPECS.contains(classname) => val desc = DB_COMMAND_SPECS(classname) - desc.databaseDescs.foreach { databaseDesc => - try { - val database = databaseDesc.extract(plan) - if (databaseDesc.isInput) { - inputObjs += PrivilegeObject(database) - } else { - outputObjs += PrivilegeObject(database) - } - } catch { - case e: Exception => - LOG.debug(databaseDesc.error(plan, e)) + val outcomes = new DescOutcomes(plan, spark) + outcomes.run(desc.databaseDescs) { databaseDesc => + val database = databaseDesc.extract(plan) + if (databaseDesc.isInput) { + inputObjs += PrivilegeObject(database) + } else { + outputObjs += PrivilegeObject(database) } } - desc.uriDescs.foreach { ud => - try { - val uris = ud.extract(plan, spark) - if (ud.isInput) { - inputObjs ++= uris.map(PrivilegeObject(_)) - } else { - outputObjs ++= uris.map(PrivilegeObject(_)) - } - } catch { - case e: Exception => - LOG.debug(ud.error(plan, e)) + outcomes.run(desc.uriDescs) { ud => + val uris = ud.extract(plan, spark) + if (ud.isInput) { + inputObjs ++= uris.map(PrivilegeObject(_)) + } else { + outputObjs ++= uris.map(PrivilegeObject(_)) } } + outcomes.reportIfAllFailed() desc.operationType case classname if TABLE_COMMAND_SPECS.contains(classname) => val spec = TABLE_COMMAND_SPECS(classname) - spec.tableDescs.foreach { td => + val outcomes = new DescOutcomes(plan, spark) + outcomes.run(spec.tableDescs) { td => if (td.isInput) { inputObjs ++= getTablePriv(td) } else { outputObjs ++= getTablePriv(td) } } - spec.uriDescs.foreach { ud => - try { - val uris = ud.extract(plan, spark) - if (ud.isInput) { - inputObjs ++= uris.map(PrivilegeObject(_)) - } else { - outputObjs ++= uris.map(PrivilegeObject(_)) - } - } catch { - case e: Exception => - LOG.debug(ud.error(plan, e)) + outcomes.run(spec.uriDescs) { ud => + val uris = ud.extract(plan, spark) + if (ud.isInput) { + inputObjs ++= uris.map(PrivilegeObject(_)) + } else { + outputObjs ++= uris.map(PrivilegeObject(_)) } } - spec.queries(plan).foreach { p => + // extract inside the tracked run (extraction failures are layer 3), but recurse + // into the extracted queries outside it, so their violations surface as themselves + val queries = ArrayBuffer[LogicalPlan]() + outcomes.run(spec.queryDescs) { qd => + queries ++= qd.extract(plan) + } + outcomes.reportIfAllFailed() + queries.foreach { p => buildQuery(Project(p.output, p), inputObjs, spark = spark) } spec.operationType case classname if FUNCTION_COMMAND_SPECS.contains(classname) => val spec = FUNCTION_COMMAND_SPECS(classname) - spec.functionDescs.foreach { fd => - try { - val function = fd.extract(plan) - if (!fd.functionTypeDesc.exists(_.skip(plan, spark))) { - if (fd.isInput) { - inputObjs += PrivilegeObject(function) - } else { - outputObjs += PrivilegeObject(function) - } + val outcomes = new DescOutcomes(plan, spark) + outcomes.run(spec.functionDescs) { fd => + val function = fd.extract(plan) + if (!fd.functionTypeDesc.exists(_.skip(plan, spark))) { + if (fd.isInput) { + inputObjs += PrivilegeObject(function) + } else { + outputObjs += PrivilegeObject(function) } - } catch { - case e: Exception => - LOG.debug(fd.error(plan, e)) } } + outcomes.reportIfAllFailed() spec.operationType - case _ => OperationType.QUERY + case classname => + // fail-open layer 1: a command with no spec produces zero access requests + if (!isKnownHarmlessClassname(classname)) { + ParanoidMode.onViolation( + spark, + plan, + ViolationKind.UNCLASSIFIED_COMMAND, + unverifiedAllowlistDetail(classname)) + } + OperationType.QUERY } } @@ -316,11 +422,16 @@ object PrivilegesBuilder { OperationType.EXPLAIN case _ if isExplainCommandChild(spark) => OperationType.EXPLAIN + // RunnableCommand case cmd: Command => buildCommand(cmd, inputObjs, outputObjs, spark) - // Spark 4.0 made some v2 commands (e.g. `Call`) no longer extend `Command`; dispatch - // them via the spec table as long as the className is recognized. - case cmd if isKnownTableCommand(cmd) => buildCommand(cmd, inputObjs, outputObjs, spark) + + // A node with a command spec that is not a Command on this Spark version: the class + // kept its name but changed supertype (e.g. CALL, a Command via Iceberg on Spark 3 + // but an ExecutableDuringAnalysis UnaryNode on Spark 4). Route it to its spec instead + // of letting it fall through to the query path where the spec is unreachable. + case cmd if hasCommandSpec(cmd.getClass.getName) => + buildCommand(cmd, inputObjs, outputObjs, spark) // Queries case _ => buildQuery(Project(plan0.output, plan0), inputObjs, spark = spark) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala index 2e6e1b6c1f8..3ca38c9f02b 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi.plugin.spark.authz.serde +import scala.collection.mutable.ArrayBuffer + import com.fasterxml.jackson.annotation.JsonIgnore import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Expression @@ -35,11 +37,43 @@ import org.apache.kyuubi.plugin.spark.authz.OperationType.OperationType * - the classname of a command which this spec point to * - the [[OperationType]] of this command which finally maps to an access privilege */ +private[serde] object SparkVersionAudit { + + /** + * Validates a spec's audited-Spark-versions list. Versions are exact `major.minor` + * pairs, deliberately enumerated rather than expressed as ranges: range boundaries + * invite misreadings ("less than 4.0, exclusive" read as inclusive), while an explicit + * list has no boundary to misread, and a new Spark minor is unaudited by default until + * a human adds it. + */ + def validate(classname: String, versions: Seq[String]): Unit = { + versions.foreach { v => + require( + v.matches("""\d+\.\d+"""), + s"""spec for $classname: verified Spark version '$v' must be an + | exact major.minor pair (e.g. "3.5"): explicit enumeration, + | no ranges or wildcards""".stripMargin) + } + } +} + trait CommandSpec extends { @JsonIgnore final protected val LOG = LoggerFactory.getLogger(getClass) def classname: String def opType: String + + /** + * The exact Spark `major.minor` versions this spec was authored or re-reviewed against. + * For command and scan specs this is ADVISORY metadata: the spec still engages on other + * versions (a spec imposes checks, so staying active on an unaudited version is the + * safe direction: drift is caught by the extraction-failure checks and the build-time + * coverage suite). Contrast [[HarmlessNodeSpec.verifiedSparkVersions]], where an + * unverified version makes the entry inert, because an allowlist entry grants silence. + * Empty means not yet audited per-version. + */ + def verifiedSparkVersions: Seq[String] + final def operationType: OperationType = OperationType.withName(opType) } @@ -58,7 +92,10 @@ case class DatabaseCommandSpec( classname: String, databaseDescs: Seq[DatabaseDesc], opType: String = OperationType.QUERY.toString, - uriDescs: Seq[UriDesc] = Nil) extends CommandSpec {} + uriDescs: Seq[UriDesc] = Nil, + verifiedSparkVersions: Seq[String] = Nil) extends CommandSpec { + SparkVersionAudit.validate(classname, verifiedSparkVersions) +} /** * A specification describe a function command @@ -70,7 +107,10 @@ case class DatabaseCommandSpec( case class FunctionCommandSpec( classname: String, functionDescs: Seq[FunctionDesc], - opType: String) extends CommandSpec + opType: String, + verifiedSparkVersions: Seq[String] = Nil) extends CommandSpec { + SparkVersionAudit.validate(classname, verifiedSparkVersions) +} /** * A specification describe a table command @@ -85,7 +125,10 @@ case class TableCommandSpec( tableDescs: Seq[TableDesc], opType: String = OperationType.QUERY.toString, queryDescs: Seq[QueryDesc] = Nil, - uriDescs: Seq[UriDesc] = Nil) extends CommandSpec { + uriDescs: Seq[UriDesc] = Nil, + verifiedSparkVersions: Seq[String] = Nil) extends CommandSpec { + SparkVersionAudit.validate(classname, verifiedSparkVersions) + def queries: LogicalPlan => Seq[LogicalPlan] = plan => { queryDescs.flatMap { qd => try { @@ -107,37 +150,99 @@ case class TableCommandSpec( */ case class DeniedPlanNodeSpec(classname: String, message: String) +/** + * A specification declaring that a plan node class is not authorization-relevant, so its + * appearance during privilege building is not a classification violation (see + * [[org.apache.kyuubi.plugin.spark.authz.ParanoidMode]]). + * + * The "known" and "harmless" assertions are only as good as the review behind them, and a + * class is free to change shape under the same fully qualified name in the next Spark + * release (exactly what CALL did between Spark 3 and 4). Each entry therefore names the + * Spark minor versions it has been reviewed against: as an explicit enumeration, not a + * range: ranges invite boundary misreadings ("less than 4.0, exclusive" read as + * inclusive), while a list has no boundary to misread, and a new Spark minor is unverified + * by default until a human adds it. + * + * @param classname the fully qualified plan node classname + * @param reason why this node is harmless: a required, human-reviewed justification + * @param verifiedSparkVersions the exact Spark `major.minor` versions the reason was + * reviewed against; on any other version the entry is inert + * and the node counts as unclassified + */ +case class HarmlessNodeSpec( + classname: String, + reason: String, + verifiedSparkVersions: Seq[String]) { + require(classname.nonEmpty, "harmless node spec requires a classname") + require( + reason.trim.nonEmpty, + s"harmless node spec for $classname requires a reason: each allowlist entry must be" + + s" a reviewed decision, not a reflexive silencing") + require( + verifiedSparkVersions.nonEmpty, + s"harmless node spec for $classname requires at least one verified Spark version:" + + s" 'harmless' is an assertion about a class on a specific Spark version") + SparkVersionAudit.validate(classname, verifiedSparkVersions) + + def appliesTo(sparkMajorMinor: String): Boolean = { + verifiedSparkVersions.contains(sparkMajorMinor) + } +} + case class ScanSpec( classname: String, scanDescs: Seq[ScanDesc], functionDescs: Seq[FunctionDesc] = Seq.empty, - uriDescs: Seq[UriDesc] = Seq.empty) extends CommandSpec { + uriDescs: Seq[UriDesc] = Seq.empty, + verifiedSparkVersions: Seq[String] = Nil) extends CommandSpec { + SparkVersionAudit.validate(classname, verifiedSparkVersions) + override def opType: String = OperationType.QUERY.toString def tables: (LogicalPlan, SparkSession) => Seq[Table] = (plan, spark) => { - scanDescs.flatMap { td => - try { - td.extract(plan, spark) - } catch { - case e: Exception => - LOG.debug(td.error(plan, e)) - None + tablesWithFailures(plan, spark)._1 + } + + /** + * Like [[tables]], but also returns the extraction failures so the caller can tell a scan + * that legitimately has no table from one whose extractors all broke (fail-open layer 3). + */ + def tablesWithFailures: (LogicalPlan, SparkSession) => (Seq[Table], Seq[Throwable]) = + (plan, spark) => { + val failures = ArrayBuffer[Throwable]() + val tables = scanDescs.flatMap { td => + try { + td.extract(plan, spark) + } catch { + case e: Exception => + LOG.debug(td.error(plan, e)) + failures += e + None + } } + (tables, failures) } - } def uris: LogicalPlan => Seq[Uri] = plan => { - uriDescs.flatMap { ud => + urisWithFailures(plan)._1 + } + + /** Like [[uris]], but also returns the extraction failures. */ + def urisWithFailures: LogicalPlan => (Seq[Uri], Seq[Throwable]) = plan => { + val failures = ArrayBuffer[Throwable]() + val uris = uriDescs.flatMap { ud => try { ud.extract(plan) } catch { case e: Exception => LOG.debug(ud.error(plan, e)) + failures += e None } } + (uris, failures) } - def functions: (Expression) => Seq[Function] = (expr) => { + def functions: Expression => Seq[Function] = expr => { functionDescs.flatMap { fd => try { Some(fd.extract(expr)) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala index f9a8c38ff5c..ba1863e00ad 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala @@ -35,6 +35,7 @@ import org.apache.kyuubi.plugin.spark.authz.serde.QueryExtractor.queryExtractors import org.apache.kyuubi.plugin.spark.authz.serde.TableExtractor.tableExtractors import org.apache.kyuubi.plugin.spark.authz.serde.TableTypeExtractor.tableTypeExtractors import org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor.uriExtractors +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils.SPARK_RUNTIME_MAJOR_MINOR import org.apache.kyuubi.util.reflect.ReflectUtils._ package object serde { @@ -87,6 +88,8 @@ package object serde { SCAN_SPECS.contains(r.getClass.getName) } + final lazy val SCAN_SPEC_CLASSNAMES: Set[String] = SCAN_SPECS.keySet + def getScanSpec(r: AnyRef): ScanSpec = { SCAN_SPECS(r.getClass.getName) } @@ -106,6 +109,37 @@ package object serde { FUNCTION_SPECS(r.getClass.getName) } + /** + * Whether the classname has an entry in any command spec file. Note that spec presence + * alone is not coverage: the node must also be routable to the command dispatch, see + * [[org.apache.kyuubi.plugin.spark.authz.ParanoidMode.ViolationKind.UNREACHABLE_SPEC]]. + */ + def hasCommandSpec(classname: String): Boolean = { + TABLE_COMMAND_SPECS.contains(classname) || + DB_COMMAND_SPECS.contains(classname) || + FUNCTION_COMMAND_SPECS.contains(classname) + } + + final lazy val KNOWN_HARMLESS_NODES: Map[String, HarmlessNodeSpec] = { + val is = getClass.getClassLoader.getResourceAsStream("known_harmless_spec.json") + mapper.readValue(is, new TypeReference[Array[HarmlessNodeSpec]] {}) + .map(e => (e.classname, e)).toMap + } + + /** + * An allowlist entry only classifies a node on the Spark minor versions its "harmless" + * assertion was reviewed against: a class is free to change shape under the same name + * in the next release, so on an unverified version the entry is inert and the node + * counts as unclassified (fail closed). + */ + def isKnownHarmless(r: AnyRef): Boolean = { + isKnownHarmlessClassname(r.getClass.getName) + } + + def isKnownHarmlessClassname(classname: String): Boolean = { + KNOWN_HARMLESS_NODES.get(classname).exists(_.appliesTo(SPARK_RUNTIME_MAJOR_MINOR)) + } + def operationType(plan: LogicalPlan): OperationType = { val effective = unwrapCommandResult(plan) val classname = effective.getClass.getName diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala index f9b4af74c5f..b44e38ae6f3 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala @@ -91,7 +91,30 @@ private[authz] object AuthZUtils { } } + /** + * Whether the plan node executes during analysis (Spark 4's `ExecutableDuringAnalysis`, + * e.g. `CALL`), i.e. potentially before any authorization rule runs. Resolved reflectively: + * the trait does not exist on all supported Spark versions, and the set of such "dangerous" + * supertypes is open, so this must never be the only line of defense. + */ + lazy private val executableDuringAnalysisClass: Option[Class[_]] = { + try { + Some(Class.forName("org.apache.spark.sql.catalyst.plans.logical.ExecutableDuringAnalysis")) + } catch { + case _: ClassNotFoundException => None + } + } + + def executesDuringAnalysis(plan: LogicalPlan): Boolean = { + executableDuringAnalysisClass.exists(_.isInstance(plan)) + } + lazy val SPARK_RUNTIME_VERSION: SemanticVersion = SemanticVersion(SPARK_VERSION) + + /** The running Spark's exact `major.minor` pair, the granularity spec audits key on. */ + lazy val SPARK_RUNTIME_MAJOR_MINOR: String = + s"${SPARK_RUNTIME_VERSION.majorVersion}.${SPARK_RUNTIME_VERSION.minorVersion}" + lazy val isSparkV34OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "3.4" lazy val isSparkV35OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "3.5" lazy val isSparkV40OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "4.0" diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_3.5.txt b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_3.5.txt new file mode 100644 index 00000000000..989fac0f9ac --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_3.5.txt @@ -0,0 +1,136 @@ +org.apache.paimon.spark.catalyst.plans.logical.IncrementalQuery +org.apache.paimon.spark.commands.PaimonAnalyzeTableColumnCommand +org.apache.paimon.spark.commands.PaimonDynamicPartitionOverwriteCommand +org.apache.paimon.spark.commands.PaimonTruncateTableCommand +org.apache.paimon.spark.commands.WriteIntoPaimonTable +org.apache.spark.sql.catalyst.TimeTravel +org.apache.spark.sql.catalyst.analysis.RelationTimeTravel +org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier +org.apache.spark.sql.catalyst.analysis.ResolvedInlineTable +org.apache.spark.sql.catalyst.analysis.ResolvedNonPersistentFunc +org.apache.spark.sql.catalyst.analysis.ResolvedPersistentFunc +org.apache.spark.sql.catalyst.analysis.ResolvedPersistentView +org.apache.spark.sql.catalyst.analysis.ResolvedTable +org.apache.spark.sql.catalyst.analysis.ResolvedTempView +org.apache.spark.sql.catalyst.analysis.UnresolvedFunctionName +org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier +org.apache.spark.sql.catalyst.analysis.UnresolvedInlineTable +org.apache.spark.sql.catalyst.analysis.UnresolvedNamespace +org.apache.spark.sql.catalyst.analysis.UnresolvedTable +org.apache.spark.sql.catalyst.analysis.UnresolvedTableOrView +org.apache.spark.sql.catalyst.analysis.UnresolvedTableValuedFunction +org.apache.spark.sql.catalyst.analysis.UnresolvedView +org.apache.spark.sql.catalyst.catalog.TemporaryViewRelation +org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation +org.apache.spark.sql.catalyst.encoders.DummyExpressionHolder +org.apache.spark.sql.catalyst.plans.logcal.HoodieFileSystemViewTableValuedFunction +org.apache.spark.sql.catalyst.plans.logcal.HoodieFileSystemViewTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logcal.HoodieMetadataTableValuedFunction +org.apache.spark.sql.catalyst.plans.logcal.HoodieQuery +org.apache.spark.sql.catalyst.plans.logcal.HoodieTableChanges +org.apache.spark.sql.catalyst.plans.logcal.HoodieTableChangesByPath +org.apache.spark.sql.catalyst.plans.logcal.HoodieTimelineTableValuedFunction +org.apache.spark.sql.catalyst.plans.logcal.HoodieTimelineTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logical.AlterColumnSyncIdentity +org.apache.spark.sql.catalyst.plans.logical.AlterTableAddConstraint +org.apache.spark.sql.catalyst.plans.logical.AlterTableDropConstraint +org.apache.spark.sql.catalyst.plans.logical.AlterTableDropFeature +org.apache.spark.sql.catalyst.plans.logical.AlterViewAs +org.apache.spark.sql.catalyst.plans.logical.AnalyzeColumn +org.apache.spark.sql.catalyst.plans.logical.AnalyzeTable +org.apache.spark.sql.catalyst.plans.logical.AnalyzeTables +org.apache.spark.sql.catalyst.plans.logical.CallCommand +org.apache.spark.sql.catalyst.plans.logical.CompactionPath +org.apache.spark.sql.catalyst.plans.logical.CompactionShowOnPath +org.apache.spark.sql.catalyst.plans.logical.CompactionShowOnTable +org.apache.spark.sql.catalyst.plans.logical.CompactionTable +org.apache.spark.sql.catalyst.plans.logical.CreateFunction +org.apache.spark.sql.catalyst.plans.logical.CreateIndex +org.apache.spark.sql.catalyst.plans.logical.CreateView +org.apache.spark.sql.catalyst.plans.logical.DeleteFromTableWithFilters +org.apache.spark.sql.catalyst.plans.logical.DeltaMergeInto +org.apache.spark.sql.catalyst.plans.logical.DescribeColumn +org.apache.spark.sql.catalyst.plans.logical.DescribeFunction +org.apache.spark.sql.catalyst.plans.logical.DropFunction +org.apache.spark.sql.catalyst.plans.logical.DropIndex +org.apache.spark.sql.catalyst.plans.logical.DropView +org.apache.spark.sql.catalyst.plans.logical.LoadData +org.apache.spark.sql.catalyst.plans.logical.RecoverPartitions +org.apache.spark.sql.catalyst.plans.logical.RefreshFunction +org.apache.spark.sql.catalyst.plans.logical.RefreshIndex +org.apache.spark.sql.catalyst.plans.logical.SetTableLocation +org.apache.spark.sql.catalyst.plans.logical.SetTableSerDeProperties +org.apache.spark.sql.catalyst.plans.logical.SetViewProperties +org.apache.spark.sql.catalyst.plans.logical.ShowColumns +org.apache.spark.sql.catalyst.plans.logical.ShowFunctions +org.apache.spark.sql.catalyst.plans.logical.ShowIndexes +org.apache.spark.sql.catalyst.plans.logical.ShowPartitions +org.apache.spark.sql.catalyst.plans.logical.ShowTableExtended +org.apache.spark.sql.catalyst.plans.logical.ShowViews +org.apache.spark.sql.catalyst.plans.logical.UncacheTable +org.apache.spark.sql.catalyst.plans.logical.UnsetViewProperties +org.apache.spark.sql.catalyst.plans.logical.WriteDelta +org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView +org.apache.spark.sql.catalyst.plans.logical.views.DropIcebergView +org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View +org.apache.spark.sql.catalyst.plans.logical.views.ShowIcebergViews +org.apache.spark.sql.catalyst.streaming.StreamingRelationV2 +org.apache.spark.sql.delta.CDCNameBased +org.apache.spark.sql.delta.CDCPathBased +org.apache.spark.sql.delta.DeltaDynamicPartitionOverwriteCommand +org.apache.spark.sql.delta.ResolvedPathBasedNonDeltaTable +org.apache.spark.sql.delta.UnresolvedPathBasedDeltaTable +org.apache.spark.sql.delta.UnresolvedPathBasedDeltaTableRelation +org.apache.spark.sql.delta.UnresolvedPathBasedTable +org.apache.spark.sql.delta.commands.AlterTableAddColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableAddConstraintDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableChangeColumnDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableClusterByDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropConstraintDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropFeatureDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableReplaceColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableSetLocationDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableSetPropertiesDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableUnsetPropertiesDeltaCommand +org.apache.spark.sql.delta.commands.CloneTableCommand +org.apache.spark.sql.delta.commands.ConvertToDeltaCommand +org.apache.spark.sql.delta.commands.CreateDeltaTableCommand +org.apache.spark.sql.delta.commands.DeltaGenerateCommand +org.apache.spark.sql.delta.commands.DeltaReorgTable +org.apache.spark.sql.delta.commands.DeltaReorgTableCommand +org.apache.spark.sql.delta.commands.DescribeDeltaDetailCommand +org.apache.spark.sql.delta.commands.DescribeDeltaHistoryCommand +org.apache.spark.sql.delta.commands.RestoreTableCommand +org.apache.spark.sql.delta.commands.ShowDeltaTableColumnsCommand +org.apache.spark.sql.delta.commands.WriteIntoDelta +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillCommand +org.apache.spark.sql.delta.constraints.ExpressionLogicalPlanWrapper +org.apache.spark.sql.delta.skipping.clustering.temp.AlterTableClusterBy +org.apache.spark.sql.delta.skipping.clustering.temp.ClusterByPlan +org.apache.spark.sql.execution.adaptive.LogicalQueryStage +org.apache.spark.sql.execution.command.ClearCacheCommand$ +org.apache.spark.sql.execution.command.DescribeQueryCommand +org.apache.spark.sql.execution.command.ExplainCommand +org.apache.spark.sql.execution.command.ExternalCommandExecutor +org.apache.spark.sql.execution.command.ListArchivesCommand +org.apache.spark.sql.execution.command.ListFilesCommand +org.apache.spark.sql.execution.command.ListJarsCommand +org.apache.spark.sql.execution.command.SetCatalogCommand +org.apache.spark.sql.execution.command.ShowCatalogsCommand +org.apache.spark.sql.execution.command.ShowCurrentNamespaceCommand +org.apache.spark.sql.execution.command.ShowFunctionsCommand +org.apache.spark.sql.execution.command.ShowTablesCommand +org.apache.spark.sql.execution.command.ShowViewsCommand +org.apache.spark.sql.execution.command.StreamingExplainCommand +org.apache.spark.sql.execution.datasources.RefreshResource +org.apache.spark.sql.execution.datasources.SparkExpressionConverter$DummyRelation +org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation +org.apache.spark.sql.execution.datasources.v2.ScanBuilderHolder +org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation +org.apache.spark.sql.execution.streaming.OffsetHolder +org.apache.spark.sql.execution.streaming.StreamingExecutionRelation +org.apache.spark.sql.execution.streaming.StreamingRelation +org.apache.spark.sql.execution.streaming.sources.MemoryPlan +org.apache.spark.sql.hudi.command.AlterHoodieTableAddPartitionCommand +org.apache.spark.sql.hudi.command.ShowHoodieCreateTableCommand diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt new file mode 100644 index 00000000000..530fa82d4c5 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt @@ -0,0 +1,181 @@ +org.apache.spark.sql.catalyst.TimeTravel +org.apache.spark.sql.catalyst.analysis.CurrentNamespace$ +org.apache.spark.sql.catalyst.analysis.RelationTimeTravel +org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier +org.apache.spark.sql.catalyst.analysis.ResolvedInlineTable +org.apache.spark.sql.catalyst.analysis.ResolvedNamespace +org.apache.spark.sql.catalyst.analysis.ResolvedNonPersistentFunc +org.apache.spark.sql.catalyst.analysis.ResolvedPersistentFunc +org.apache.spark.sql.catalyst.analysis.ResolvedPersistentView +org.apache.spark.sql.catalyst.analysis.ResolvedProcedure +org.apache.spark.sql.catalyst.analysis.ResolvedTable +org.apache.spark.sql.catalyst.analysis.ResolvedTempView +org.apache.spark.sql.catalyst.analysis.SQLTableFunction +org.apache.spark.sql.catalyst.analysis.UnresolvedExecuteImmediate +org.apache.spark.sql.catalyst.analysis.UnresolvedFunctionName +org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier +org.apache.spark.sql.catalyst.analysis.UnresolvedInlineTable +org.apache.spark.sql.catalyst.analysis.UnresolvedNamespace +org.apache.spark.sql.catalyst.analysis.UnresolvedProcedure +org.apache.spark.sql.catalyst.analysis.UnresolvedTable +org.apache.spark.sql.catalyst.analysis.UnresolvedTableOrView +org.apache.spark.sql.catalyst.analysis.UnresolvedTableValuedFunction +org.apache.spark.sql.catalyst.analysis.UnresolvedView +org.apache.spark.sql.catalyst.analysis.V2TableReference +org.apache.spark.sql.catalyst.analysis.resolver.UnresolvedCteRelationRef +org.apache.spark.sql.catalyst.catalog.TemporaryViewRelation +org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation +org.apache.spark.sql.catalyst.encoders.DummyExpressionHolder +org.apache.spark.sql.catalyst.plans.logcal.HoodieFileSystemViewTableValuedFunction +org.apache.spark.sql.catalyst.plans.logcal.HoodieFileSystemViewTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logcal.HoodieMetadataTableValuedFunction +org.apache.spark.sql.catalyst.plans.logcal.HoodieQuery +org.apache.spark.sql.catalyst.plans.logcal.HoodieTableChanges +org.apache.spark.sql.catalyst.plans.logcal.HoodieTableChangesByPath +org.apache.spark.sql.catalyst.plans.logcal.HoodieTimelineTableValuedFunction +org.apache.spark.sql.catalyst.plans.logcal.HoodieTimelineTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logical.AddCheckConstraint +org.apache.spark.sql.catalyst.plans.logical.AddConstraint +org.apache.spark.sql.catalyst.plans.logical.AlterColumnSyncIdentity +org.apache.spark.sql.catalyst.plans.logical.AlterColumns +org.apache.spark.sql.catalyst.plans.logical.AlterTableAddConstraint +org.apache.spark.sql.catalyst.plans.logical.AlterTableClusterBy +org.apache.spark.sql.catalyst.plans.logical.AlterTableCollation +org.apache.spark.sql.catalyst.plans.logical.AlterTableDropConstraint +org.apache.spark.sql.catalyst.plans.logical.AlterTableDropFeature +org.apache.spark.sql.catalyst.plans.logical.AlterViewAs +org.apache.spark.sql.catalyst.plans.logical.AlterViewSchemaBinding +org.apache.spark.sql.catalyst.plans.logical.AnalyzeColumn +org.apache.spark.sql.catalyst.plans.logical.AnalyzeTable +org.apache.spark.sql.catalyst.plans.logical.AnalyzeTables +org.apache.spark.sql.catalyst.plans.logical.CTERelationRef +org.apache.spark.sql.catalyst.plans.logical.CallCommand +org.apache.spark.sql.catalyst.plans.logical.CommandResult +org.apache.spark.sql.catalyst.plans.logical.CompactionPath +org.apache.spark.sql.catalyst.plans.logical.CompactionShowOnPath +org.apache.spark.sql.catalyst.plans.logical.CompactionShowOnTable +org.apache.spark.sql.catalyst.plans.logical.CompactionTable +org.apache.spark.sql.catalyst.plans.logical.CompoundBody +org.apache.spark.sql.catalyst.plans.logical.CreateFlowCommand +org.apache.spark.sql.catalyst.plans.logical.CreateFunction +org.apache.spark.sql.catalyst.plans.logical.CreateIndex +org.apache.spark.sql.catalyst.plans.logical.CreateMaterializedViewAsSelect +org.apache.spark.sql.catalyst.plans.logical.CreateStreamingTable +org.apache.spark.sql.catalyst.plans.logical.CreateStreamingTableAsSelect +org.apache.spark.sql.catalyst.plans.logical.CreateUserDefinedFunction +org.apache.spark.sql.catalyst.plans.logical.CreateVariable +org.apache.spark.sql.catalyst.plans.logical.CreateView +org.apache.spark.sql.catalyst.plans.logical.DeleteFromTableWithFilters +org.apache.spark.sql.catalyst.plans.logical.DeltaMergeInto +org.apache.spark.sql.catalyst.plans.logical.DescribeColumn +org.apache.spark.sql.catalyst.plans.logical.DescribeFunction +org.apache.spark.sql.catalyst.plans.logical.DropConstraint +org.apache.spark.sql.catalyst.plans.logical.DropFunction +org.apache.spark.sql.catalyst.plans.logical.DropIndex +org.apache.spark.sql.catalyst.plans.logical.DropVariable +org.apache.spark.sql.catalyst.plans.logical.DropView +org.apache.spark.sql.catalyst.plans.logical.EmptyRelation +org.apache.spark.sql.catalyst.plans.logical.LoadData +org.apache.spark.sql.catalyst.plans.logical.NoopCommand +org.apache.spark.sql.catalyst.plans.logical.PythonWorkerLogs +org.apache.spark.sql.catalyst.plans.logical.RecoverPartitions +org.apache.spark.sql.catalyst.plans.logical.RefreshFunction +org.apache.spark.sql.catalyst.plans.logical.RefreshIndex +org.apache.spark.sql.catalyst.plans.logical.SetTableLocation +org.apache.spark.sql.catalyst.plans.logical.SetTableSerDeProperties +org.apache.spark.sql.catalyst.plans.logical.SetVariable +org.apache.spark.sql.catalyst.plans.logical.SetViewProperties +org.apache.spark.sql.catalyst.plans.logical.ShowColumns +org.apache.spark.sql.catalyst.plans.logical.ShowFunctions +org.apache.spark.sql.catalyst.plans.logical.ShowIndexes +org.apache.spark.sql.catalyst.plans.logical.ShowPartitions +org.apache.spark.sql.catalyst.plans.logical.ShowTablePartition +org.apache.spark.sql.catalyst.plans.logical.ShowTables +org.apache.spark.sql.catalyst.plans.logical.ShowTablesExtended +org.apache.spark.sql.catalyst.plans.logical.ShowViews +org.apache.spark.sql.catalyst.plans.logical.Transpose +org.apache.spark.sql.catalyst.plans.logical.UncacheTable +org.apache.spark.sql.catalyst.plans.logical.UnionLoopRef +org.apache.spark.sql.catalyst.plans.logical.UnresolvedDataSource +org.apache.spark.sql.catalyst.plans.logical.UnsetViewProperties +org.apache.spark.sql.catalyst.plans.logical.WriteDelta +org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView +org.apache.spark.sql.catalyst.plans.logical.views.DropIcebergView +org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View +org.apache.spark.sql.catalyst.plans.logical.views.ShowIcebergViews +org.apache.spark.sql.catalyst.streaming.StreamingRelationV2 +org.apache.spark.sql.delta.CDCNameBased +org.apache.spark.sql.delta.CDCPathBased +org.apache.spark.sql.delta.DeltaDynamicPartitionOverwriteCommand +org.apache.spark.sql.delta.ResolvedPathBasedNonDeltaTable +org.apache.spark.sql.delta.UnresolvedPathBasedDeltaTable +org.apache.spark.sql.delta.UnresolvedPathBasedDeltaTableRelation +org.apache.spark.sql.delta.UnresolvedPathBasedTable +org.apache.spark.sql.delta.commands.AlterTableAddColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableAddConstraintDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableChangeColumnDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableClusterByDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropConstraintDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropFeatureDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableReplaceColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableSetLocationDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableSetPropertiesDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableUnsetPropertiesDeltaCommand +org.apache.spark.sql.delta.commands.CloneTableCommand +org.apache.spark.sql.delta.commands.ConvertToDeltaCommand +org.apache.spark.sql.delta.commands.CreateDeltaTableCommand +org.apache.spark.sql.delta.commands.DeltaGenerateCommand +org.apache.spark.sql.delta.commands.DeltaReorgTable +org.apache.spark.sql.delta.commands.DeltaReorgTableCommand +org.apache.spark.sql.delta.commands.DescribeDeltaDetailCommand +org.apache.spark.sql.delta.commands.DescribeDeltaHistoryCommand +org.apache.spark.sql.delta.commands.RestoreTableCommand +org.apache.spark.sql.delta.commands.ShowDeltaTableColumnsCommand +org.apache.spark.sql.delta.commands.WriteIntoDelta +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillCommand +org.apache.spark.sql.delta.constraints.ExpressionLogicalPlanWrapper +org.apache.spark.sql.delta.skipping.clustering.temp.AlterTableClusterBy +org.apache.spark.sql.delta.skipping.clustering.temp.ClusterByPlan +org.apache.spark.sql.execution.ExternalRDD +org.apache.spark.sql.execution.LogicalRDD +org.apache.spark.sql.execution.adaptive.LogicalQueryStage +org.apache.spark.sql.execution.columnar.InMemoryRelation +org.apache.spark.sql.execution.command.AlterViewSchemaBindingCommand +org.apache.spark.sql.execution.command.ClearCacheCommand$ +org.apache.spark.sql.execution.command.CreateSQLFunctionCommand +org.apache.spark.sql.execution.command.DescribeProcedureCommand +org.apache.spark.sql.execution.command.DescribeQueryCommand +org.apache.spark.sql.execution.command.DescribeRelationJsonCommand +org.apache.spark.sql.execution.command.DropTempViewCommand +org.apache.spark.sql.execution.command.ExplainCommand +org.apache.spark.sql.execution.command.ExternalCommandExecutor +org.apache.spark.sql.execution.command.ListArchivesCommand +org.apache.spark.sql.execution.command.ListFilesCommand +org.apache.spark.sql.execution.command.ListJarsCommand +org.apache.spark.sql.execution.command.ResetCommand +org.apache.spark.sql.execution.command.SaveAsV1TableCommand +org.apache.spark.sql.execution.command.SetCatalogCommand +org.apache.spark.sql.execution.command.SetCommand +org.apache.spark.sql.execution.command.SetNamespaceCollationCommand +org.apache.spark.sql.execution.command.ShowCatalogsCommand +org.apache.spark.sql.execution.command.ShowCurrentNamespaceCommand +org.apache.spark.sql.execution.command.ShowFunctionsCommand +org.apache.spark.sql.execution.command.ShowNamespacesCommand +org.apache.spark.sql.execution.command.ShowProceduresCommand +org.apache.spark.sql.execution.command.ShowTablesCommand +org.apache.spark.sql.execution.command.ShowViewsCommand +org.apache.spark.sql.execution.command.StreamingExplainCommand +org.apache.spark.sql.execution.command.UnsetNamespacePropertiesCommand +org.apache.spark.sql.execution.datasources.RefreshResource +org.apache.spark.sql.execution.datasources.SparkExpressionConverter$DummyRelation +org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation +org.apache.spark.sql.execution.datasources.v2.ScanBuilderHolder +org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation +org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2ScanRelation +org.apache.spark.sql.execution.streaming.runtime.OffsetHolder +org.apache.spark.sql.execution.streaming.runtime.StreamingExecutionRelation +org.apache.spark.sql.execution.streaming.runtime.StreamingRelation +org.apache.spark.sql.execution.streaming.sources.MemoryPlan +org.apache.spark.sql.hudi.command.AlterHoodieTableAddPartitionCommand +org.apache.spark.sql.hudi.command.ShowHoodieCreateTableCommand diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala new file mode 100644 index 00000000000..823149687b4 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Paths, StandardOpenOption} +import java.util.jar.JarFile + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.catalyst.plans.logical.{Command, LeafNode, LogicalPlan} +// scalastyle:off +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.kyuubi.plugin.spark.authz.serde._ +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils.SPARK_RUNTIME_MAJOR_MINOR +import org.apache.kyuubi.util.AssertionUtils._ +import org.apache.kyuubi.util.GoldenFileUtils._ + +/** + * Build-time (per Spark profile) coverage checks over the spec files. These catch + * classification drift when bumping Spark versions at PR time instead of at customer + * runtime. They complement, not replace, the runtime checks in [[ParanoidMode]]: this + * suite can only see classes on the build classpath, while third-party catalog plugins + * appear only in the user's environment. + */ +class ClassificationCoverageSuite extends AnyFunSuite { + // scalastyle:on + + private def loadable(classname: String): Option[Class[_]] = { + try { + Some(Class.forName(classname, false, getClass.getClassLoader)) + } catch { + // not every spec'd class exists on every Spark/catalog-plugin profile + case _: ClassNotFoundException | _: NoClassDefFoundError => None + } + } + + private lazy val executableDuringAnalysisClass: Option[Class[_]] = + loadable("org.apache.spark.sql.catalyst.plans.logical.ExecutableDuringAnalysis") + + private val allCommandSpecClassnames: Set[String] = + TABLE_COMMAND_SPECS.keySet ++ DB_COMMAND_SPECS.keySet ++ FUNCTION_COMMAND_SPECS.keySet + + test("every command spec entry present on this classpath is routable to buildCommand") { + // A spec whose class name matches a plan node that the dispatch never routes to + // buildCommand gives the appearance of coverage while enforcing nothing — the + // CALL-on-Spark-4 shape: same fully-qualified name, different supertype. + // Dispatch routes Commands and, as a fallback, any class with a command spec, so + // plain reachability holds by construction; what can still silently break is a + // spec'd node that executes *during analysis*, before any authorization rule runs. + // Known, tracked gaps. An entry here is an ACKNOWLEDGED VULNERABILITY on the affected + // Spark version, not a pass — it only keeps the build green while a fix is pending. + // Do not add to this list without a plan to close the gap (an analysis-time check rule, + // or blocking the operation outright on that Spark version). + val acknowledgedGaps = Set( + // On Spark 4.x, CALL is an ExecutableDuringAnalysis UnaryNode under the same class + // name Iceberg used for its Spark 3 Command: the stored procedure runs during + // analysis, before any authorization rule. See the paranoid-mode design doc, §2. + "org.apache.spark.sql.catalyst.plans.logical.Call") + + val tooLateToAuthorize = allCommandSpecClassnames.toSeq.sorted.flatMap { name => + loadable(name).flatMap { cls => + executableDuringAnalysisClass match { + case Some(eda) if eda.isAssignableFrom(cls) && !classOf[Command].isAssignableFrom(cls) => + Some(name) + case _ => None + } + } + } + assert( + tooLateToAuthorize.forall(acknowledgedGaps.contains), + s"\nThese spec'd plan nodes execute during analysis, before RuleAuthorization runs," + + s" so their command specs cannot enforce anything on this Spark version:" + + s"\n ${tooLateToAuthorize.filterNot(acknowledgedGaps.contains).mkString("\n ")}\n" + + s"Authorization for them must happen in an earlier (analysis-time) rule," + + s" or the operation must be blocked outright on this Spark version.") + } + + test("allowlisted classes present on this classpath are not Commands in disguise") { + // A node allowlisted as harmless could later gain authz-relevant behavior; catching + // an allowlist entry that is (or became) a Command forces a re-review on version bumps. + val suspicious = KNOWN_HARMLESS_NODES.keySet.toSeq.sorted.flatMap { name => + loadable(name).flatMap { cls => + // Allowlisting a Command is higher-stakes than allowlisting a leaf relation, so it + // takes a second, colocated review: the entry must also be exempted here. + val exempted = Set( + // mutate session conf only; sensitive configs guarded by AuthzConfigurationChecker + "org.apache.spark.sql.execution.command.SetCommand", + "org.apache.spark.sql.execution.command.ResetCommand", + // row-filtered by ObjectFilterPlaceHolder + FilterDataSourceV2Strategy + "org.apache.spark.sql.catalyst.plans.logical.ShowNamespaces", + "org.apache.spark.sql.catalyst.plans.logical.ShowTables", + // session-local temp views are deliberately not authz resources + "org.apache.spark.sql.execution.command.DropTempViewCommand", + // resolves to literally nothing to execute + "org.apache.spark.sql.catalyst.plans.logical.NoopCommand") + if (classOf[Command].isAssignableFrom(cls) && !exempted.contains(name)) { + Some(name) + } else { + None + } + } + } + assert( + suspicious.isEmpty, + s""" + |These allowlisted 'harmless' classes are Commands on this Spark version; + | re-review their known_harmless_spec.json entries: + | ${suspicious.mkString("\n ")}""".stripMargin) + } + + // --------------------------------------------------------------------------------------- + // Enumeration check: diff the classpath's plan-node population against our classification. + // --------------------------------------------------------------------------------------- + + /** Scan prefixes covering Spark itself and the catalog plugins that inject plan nodes. */ + private val scannedPackagePrefixes = Seq( + "org/apache/spark/sql/", + "org/apache/paimon/spark/") + + /** Classes whose presence identifies a jar that can contribute logical plan nodes. */ + private val jarAnchorClassnames = Seq( + "org.apache.spark.sql.catalyst.plans.logical.LogicalPlan", // spark-catalyst + "org.apache.spark.sql.execution.SparkPlan", // spark-sql core + "org.apache.spark.sql.hive.HiveSessionStateBuilder", // spark-hive + "org.apache.iceberg.spark.SparkCatalog", // iceberg runtime, if on this profile + "org.apache.spark.sql.delta.DeltaLog", // delta, if on this profile + "org.apache.spark.sql.hudi.command.CallProcedureHoodieCommand", // hudi, if on this profile + "org.apache.paimon.spark.SparkCatalog" // paimon, if on this profile + ) + + /** + * Concrete classes on this profile's classpath that carry authorization relevance by + * shape: every Command, every LeafNode, and everything that executes during analysis. + * These are exactly the shapes the runtime invariant refuses to let pass silently. + */ + private def enumerateRelevantPlanClasses(): Seq[String] = { + val loader = getClass.getClassLoader + val jars = jarAnchorClassnames.flatMap(loadable).flatMap { cls => + Option(cls.getProtectionDomain.getCodeSource).map(_.getLocation) + }.distinct.filter(_.getPath.endsWith(".jar")) + + val relevantSupertypes: Seq[Class[_]] = + Seq(classOf[Command], classOf[LeafNode]) ++ executableDuringAnalysisClass + + jars.flatMap { jarUrl => + val jar = new JarFile(Paths.get(jarUrl.toURI).toFile) + try { + jar.entries().asScala + .map(_.getName) + .filter(n => n.endsWith(".class") && scannedPackagePrefixes.exists(n.startsWith)) + // skip anonymous/synthetic classes; keep nested ones (plan nodes can be nested) + .filterNot(n => n.contains("$$") || n.matches(".*\\$\\d.*")) + .map(_.stripSuffix(".class").replace('/', '.')) + .flatMap { classname => + try { + val cls = Class.forName(classname, false, loader) + val concrete = + !cls.isInterface && !java.lang.reflect.Modifier.isAbstract(cls.getModifiers) + if (concrete && + classOf[LogicalPlan].isAssignableFrom(cls) && + relevantSupertypes.exists(_.isAssignableFrom(cls))) { + Some(classname) + } else { + None + } + } catch { + // optional dependencies of scanned classes may be absent at test time + case _: Throwable => None + } + }.toList // strict: the iterator must be exhausted before the jar closes + } finally { + jar.close() + } + }.distinct.sorted + } + + test("enumerate authz-relevant plan classes and diff against the classification") { + // Golden backlog file, one classname per line, per Spark minor version. The contract: + // a class NEW to this diff fails the build — classify it (command/scan spec), allowlist + // it with a reason, or consciously add it to the backlog via regeneration. A class that + // leaves the diff must also leave the backlog, so the backlog only ever shrinks by + // being triaged, never silently. + val backlogFilename = s"classification_backlog_spark_$SPARK_RUNTIME_MAJOR_MINOR.txt" + val backlogPath = Paths.get( + s"${getCurrentModuleHome(this)}/src/test/resources/$backlogFilename") + + val classified: Set[String] = allCommandSpecClassnames ++ + // an allowlist entry only classifies on the Spark minors it was reviewed against; + // on this profile the others belong in the backlog awaiting re-review + KNOWN_HARMLESS_NODES.filter(_._2.appliesTo(SPARK_RUNTIME_MAJOR_MINOR)).keySet ++ + // matched by nodeName rather than classname in buildQuery + Set("org.apache.spark.sql.catalyst.analysis.UnresolvedRelation") ++ + SCAN_SPEC_CLASSNAMES + + val unclassified = enumerateRelevantPlanClasses().filterNot(classified) + val generatedStr = unclassified.mkString("", "\n", "\n") + + if (sys.env.get("KYUUBI_UPDATE").contains("1")) { + // scalastyle:off println + println(s"writing ${unclassified.size} classnames to $backlogFilename") + // scalastyle:on println + Files.write( + backlogPath, + generatedStr.getBytes(StandardCharsets.UTF_8), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING) + } else { + withClue( + s"The set of unclassified authz-relevant plan classes on this classpath changed." + + s" For every NEW class: add a command/scan spec, or an entry in" + + s" known_harmless_spec.json with a reason; only leave it in the backlog as a" + + s" conscious decision. Regenerate with KYUUBI_UPDATE=1 (dev/gen/gen_ranger_spec_json.sh" + + s" regenerates the spec files; rerun this suite for the backlog).") { + assertFileContent( + backlogPath, + Seq(generatedStr), + "KYUUBI_UPDATE=1 build/mvn test -pl extensions/spark/kyuubi-spark-authz" + + " -DwildcardSuites=org.apache.kyuubi.plugin.spark.authz.ClassificationCoverageSuite", + splitFirstExpectedLine = true) + } + } + } + + test("no class has both a command spec and an allowlist entry") { + val both = allCommandSpecClassnames.intersect(KNOWN_HARMLESS_NODES.keySet) + assert( + both.isEmpty, + s""" + |Contradictory classification: both spec'd and allowlisted: + | ${both.toSeq.sorted.mkString("\n ")}""".stripMargin) + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ParanoidModeSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ParanoidModeSuite.scala new file mode 100644 index 00000000000..49d7d51d6c2 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ParanoidModeSuite.scala @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, Literal} +import org.apache.spark.sql.catalyst.plans.logical.{LeafCommand, LeafNode, LogicalPlan, Project} +import org.apache.spark.sql.types.IntegerType +// scalastyle:off +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.kyuubi.plugin.spark.authz.ParanoidMode.{Behavior, UNCLASSIFIED_NODE_BEHAVIOR_KEY, ViolationKind} +import org.apache.kyuubi.plugin.spark.authz.serde.{HarmlessNodeSpec, KNOWN_HARMLESS_NODES, ScanDesc, ScanSpec} + +/** A leaf relation the plugin has no classification for. */ +case class UnclassifiedTestRelation() extends LeafNode { + override def output: Seq[Attribute] = Seq(AttributeReference("id", IntegerType)()) +} + +/** A command the plugin has no classification for. */ +case class UnclassifiedTestCommand() extends LeafCommand + +class ParanoidModeSuite extends AnyFunSuite with BeforeAndAfterEach with BeforeAndAfterAll { + // scalastyle:on + + private lazy val spark: SparkSession = SparkSession.builder() + .master("local[1]") + .appName(getClass.getSimpleName) + .config("spark.ui.enabled", "false") + .getOrCreate() + + override def afterAll(): Unit = { + // suites share a JVM: a leaked default session would be picked up by the next + // suite's getOrCreate(), silently dropping its extensions + spark.stop() + SparkSession.clearActiveSession() + SparkSession.clearDefaultSession() + super.afterAll() + } + + override def afterEach(): Unit = { + spark.conf.unset(UNCLASSIFIED_NODE_BEHAVIOR_KEY) + ParanoidMode.resetForTesting() + super.afterEach() + } + + private def withBehavior(behavior: String)(f: => Unit): Unit = { + spark.conf.set(UNCLASSIFIED_NODE_BEHAVIOR_KEY, behavior) + try f + finally spark.conf.unset(UNCLASSIFIED_NODE_BEHAVIOR_KEY) + } + + private def build(plan: LogicalPlan): Unit = { + PrivilegesBuilder.build(plan, spark) + } + + test("default behavior is warn") { + assert(ParanoidMode.behavior(spark) === Behavior.WARN) + } + + test("behavior values are parsed case-insensitively") { + withBehavior("DENY") { + assert(ParanoidMode.behavior(spark) === Behavior.DENY) + } + withBehavior("Allow") { + assert(ParanoidMode.behavior(spark) === Behavior.ALLOW) + } + } + + test("invalid behavior value is rejected loudly") { + withBehavior("yolo") { + val e = intercept[IllegalArgumentException](ParanoidMode.behavior(spark)) + assert(e.getMessage.contains(UNCLASSIFIED_NODE_BEHAVIOR_KEY)) + assert(e.getMessage.contains("yolo")) + } + } + + test("deny: an unclassified leaf relation fails closed") { + withBehavior("deny") { + val e = intercept[AccessControlException](build(UnclassifiedTestRelation())) + assert(e.getMessage.contains(classOf[UnclassifiedTestRelation].getName)) + assert(e.getMessage.contains(UNCLASSIFIED_NODE_BEHAVIOR_KEY)) + assert(ParanoidMode.violationCount(ViolationKind.UNCLASSIFIED_LEAF) === 1) + } + } + + test("deny: an unclassified command fails closed") { + withBehavior("deny") { + val e = intercept[AccessControlException](build(UnclassifiedTestCommand())) + assert(e.getMessage.contains(classOf[UnclassifiedTestCommand].getName)) + assert(ParanoidMode.violationCount(ViolationKind.UNCLASSIFIED_COMMAND) === 1) + } + } + + test("deny: an unclassified node cannot hide under a constant projection") { + // A Project whose output has no relation to its input is pruned for privilege + // building, but the subtree still executes and must still be classified. + withBehavior("deny") { + val plan = Project( + Seq(Alias(Literal(1), "x")()), + UnclassifiedTestRelation()) + intercept[AccessControlException](build(plan)) + } + } + + test("warn: unclassified nodes pass but every occurrence is counted") { + withBehavior("warn") { + build(UnclassifiedTestRelation()) + build(UnclassifiedTestRelation()) + assert(ParanoidMode.violationCount(ViolationKind.UNCLASSIFIED_LEAF) === 2) + } + } + + test("allow: legacy behavior, unclassified nodes pass silently") { + withBehavior("allow") { + build(UnclassifiedTestRelation()) + build(UnclassifiedTestCommand()) + } + } + + test("deny: allowlisted nodes pass") { + withBehavior("deny") { + // OneRowRelation under a Project + build(spark.sql("SELECT 1").queryExecution.optimizedPlan) + // Range, LocalRelation + build(spark.range(3).queryExecution.optimizedPlan) + build(spark.sql("VALUES (1, 'a'), (2, 'b')").queryExecution.optimizedPlan) + assert(ParanoidMode.violationCount(ViolationKind.UNCLASSIFIED_LEAF) === 0) + } + } + + test("deny: ordinary multi-operator queries recurse freely") { + withBehavior("deny") { + val df = spark.range(10).filter("id > 1") + .join(spark.range(5), "id") + .groupBy("id").count() + build(df.queryExecution.optimizedPlan) + build(df.queryExecution.analyzed) + } + } + + test("scan spec extraction failures are surfaced, not swallowed") { + val spec = ScanSpec( + classOf[UnclassifiedTestRelation].getName, + Seq(ScanDesc("noSuchField", "LogicalRelationTableExtractor"))) + val (tables, failures) = spec.tablesWithFailures(UnclassifiedTestRelation(), spark) + assert(tables.isEmpty) + assert(failures.nonEmpty) + } + + test("allowlist entries require a reason") { + val e = intercept[IllegalArgumentException]( + HarmlessNodeSpec("some.Classname", " ", Seq("3.5"))) + assert(e.getMessage.contains("reason")) + intercept[IllegalArgumentException](HarmlessNodeSpec("", "a reason", Seq("3.5"))) + } + + test("allowlist entries require explicitly enumerated Spark versions, not ranges") { + val e = intercept[IllegalArgumentException]( + HarmlessNodeSpec("some.Classname", "a reason", Nil)) + assert(e.getMessage.contains("verified Spark version")) + // no range or wildcard syntax, exact major.minor pairs only + Seq("<4.0", "3.x", "3", "3.5.1", "3.5+").foreach { bad => + val ex = intercept[IllegalArgumentException]( + HarmlessNodeSpec("some.Classname", "a reason", Seq(bad))) + assert(ex.getMessage.contains(bad)) + } + } + + test("an allowlist entry only applies to the Spark minors it was reviewed against") { + val spec = HarmlessNodeSpec("some.Classname", "a reason", Seq("3.4", "3.5")) + assert(spec.appliesTo("3.5")) + assert(spec.appliesTo("3.4")) + // fail closed on anything not explicitly listed, in either direction + assert(!spec.appliesTo("3.3")) + assert(!spec.appliesTo("4.0")) + } + + test("the shipped allowlist is loadable and every entry carries a reason and versions") { + assert(KNOWN_HARMLESS_NODES.nonEmpty) + assert(KNOWN_HARMLESS_NODES.contains( + "org.apache.spark.sql.catalyst.plans.logical.LocalRelation")) + KNOWN_HARMLESS_NODES.values.foreach { spec => + assert(spec.reason.trim.nonEmpty, s"missing reason for ${spec.classname}") + assert( + spec.verifiedSparkVersions.nonEmpty, + s"missing verified Spark versions for ${spec.classname}") + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala index 644f65533b8..6ab51b6ba89 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala @@ -57,6 +57,10 @@ trait SparkSessionProvider { "spark.sql.warehouse.dir", Utils.createTempDir("spark-warehouse").toString) .config("spark.sql.extensions", sqlExtensions) + // All authz suites fail closed on unclassified plan nodes; fallout goes to a spec + // or to known_harmless_spec.json, never to relaxing this default. (A suite may + // still override it via extraSparkConf when testing the other behaviors.) + .config(ParanoidMode.UNCLASSIFIED_NODE_BEHAVIOR_KEY, "deny") .withExtensions(extension) .config(extraSparkConf) diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala index 78ef4a31e4d..1cff4a9709d 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala @@ -21,8 +21,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths, StandardOpenOption} import org.apache.kyuubi.KyuubiFunSuite -import org.apache.kyuubi.plugin.spark.authz.serde.{mapper, CommandSpec} -import org.apache.kyuubi.plugin.spark.authz.serde.CommandSpecs +import org.apache.kyuubi.plugin.spark.authz.serde._ import org.apache.kyuubi.util.AssertionUtils._ import org.apache.kyuubi.util.GoldenFileUtils._ @@ -47,6 +46,59 @@ class JsonSpecFileGenerator extends KyuubiFunSuite { Seq(TableCommands, IcebergCommands, HudiCommands, DeltaCommands, PaimonCommands)) writeCommandSpecJson("function", Seq(FunctionCommands)) writeCommandSpecJson("scan", Seq(Scans)) + writeHarmlessNodeSpecJson() + } + + def writeHarmlessNodeSpecJson(): Unit = { + val filename = "known_harmless_spec.json" + val filePath = Paths.get( + s"${getCurrentModuleHome(this)}/src/main/resources/$filename") + + val allSpecs = KnownHarmlessNodes.specs.sortBy(_.classname) + val duplicatedClassnames = allSpecs.groupBy(_.classname).values + .filter(_.size > 1).flatMap(specs => specs.map(_.classname)).toSet + withClue(s"Unexpected duplicated classnames: $duplicatedClassnames")( + assertResult(0)(duplicatedClassnames.size)) + val generatedStr = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(allSpecs) + + if (sys.env.get("KYUUBI_UPDATE").contains("1")) { + // scalastyle:off println + println(s"writing ${allSpecs.length} specs to $filename") + // scalastyle:on println + Files.write( + filePath, + generatedStr.getBytes(StandardCharsets.UTF_8), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING) + } else { + assertFileContent( + filePath, + Seq(generatedStr), + "dev/gen/gen_ranger_spec_json.sh", + splitFirstExpectedLine = true) + } + } + + // Every entry currently in the spec files predates the Spark 4 port, so this is the + // audited baseline for any spec that doesn't declare its own verified versions. Specs + // verified on other Spark minors should set verifiedSparkVersions explicitly at their + // definition site. Note the field is advisory for command/scan specs (they still engage + // on unaudited versions), unlike the allowlist where it gates. + private val defaultVerifiedSparkVersions = Seq("3.3", "3.4", "3.5") + + private def withDefaultVerifiedVersions[T <: CommandSpec](spec: T): T = { + val populated: CommandSpec = spec match { + case s: DatabaseCommandSpec if s.verifiedSparkVersions.isEmpty => + s.copy(verifiedSparkVersions = defaultVerifiedSparkVersions) + case s: TableCommandSpec if s.verifiedSparkVersions.isEmpty => + s.copy(verifiedSparkVersions = defaultVerifiedSparkVersions) + case s: FunctionCommandSpec if s.verifiedSparkVersions.isEmpty => + s.copy(verifiedSparkVersions = defaultVerifiedSparkVersions) + case s: ScanSpec if s.verifiedSparkVersions.isEmpty => + s.copy(verifiedSparkVersions = defaultVerifiedSparkVersions) + case s => s + } + populated.asInstanceOf[T] } def writeCommandSpecJson[T <: CommandSpec]( @@ -57,6 +109,7 @@ class JsonSpecFileGenerator extends KyuubiFunSuite { s"${getCurrentModuleHome(this)}/src/main/resources/$filename") val allSpecs = specsArr.flatMap(_.specs.sortBy(_.classname)) + .map(withDefaultVerifiedVersions) val duplicatedClassnames = allSpecs.groupBy(_.classname).values .filter(_.size > 1).flatMap(specs => specs.map(_.classname)).toSet withClue(s"Unexpected duplicated classnames: $duplicatedClassnames")( diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala new file mode 100644 index 00000000000..f3957a05a5e --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.gen + +import org.apache.kyuubi.plugin.spark.authz.serde.HarmlessNodeSpec + +/** + * The explicit allowlist of plan nodes that are not authorization-relevant, backing + * `known_harmless_spec.json`. Every entry must carry a reason (an allowlist entry is a + * reviewed security decision) and names the exact Spark minor versions the review was + * performed against. On any other Spark version the entry is inert and the node counts as + * unclassified: a class is free to change shape under the same fully qualified name in + * the next release (exactly what CALL did between Spark 3 and 4), so re-reviewing this + * list, entry by entry, is part of every Spark version bump. + */ +object KnownHarmlessNodes { + + // The Spark minors the plugin currently supports and tests per-profile; the baseline + // every entry was reviewed against when the allowlist was introduced. + private val spark3x = Seq("3.3", "3.4", "3.5") + + // Entries additionally exercised in deny mode under the spark-4.1 profile. 4.0 is + // deliberately absent until something verifies it: a version joins an entry's list by + // being tested or reviewed, never by interpolation. + private val spark3xAnd41 = spark3x :+ "4.1" + + val specs: Seq[HarmlessNodeSpec] = Seq( + HarmlessNodeSpec( + "org.apache.spark.sql.catalyst.analysis.ResolvedNamespace", + "Analysis-time resolution artifact naming a namespace; reads no data itself, and the" + + " commands resolved over it are classified in their own right", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.catalyst.plans.logical.CTERelationRef", + "Leaf reference to a CTE definition; the definition's own plan appears under" + + " WithCTE in the same tree and is authorized there", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.catalyst.plans.logical.CommandResult", + "Holds rows already produced by an eagerly executed command; that command was" + + " authorized when it executed", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.catalyst.plans.logical.LocalRelation", + "Holds in-memory literal rows (VALUES lists, createDataFrame); reads no stored data", + spark3xAnd41), + HarmlessNodeSpec( + "org.apache.spark.sql.catalyst.plans.logical.NoopCommand", + "Spark's placeholder for commands with nothing to do (e.g. IF EXISTS / IF NOT EXISTS" + + " variants when the object is absent); executes nothing", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.catalyst.plans.logical.OneRowRelation", + "The implicit single-row relation backing SELECT without FROM; reads no stored data", + spark3xAnd41), + HarmlessNodeSpec( + "org.apache.spark.sql.catalyst.plans.logical.Range", + "Generates rows from a numeric range (e.g. spark.range); reads no stored data", + spark3xAnd41), + HarmlessNodeSpec( + "org.apache.spark.sql.catalyst.plans.logical.ShowNamespaces", + "Enforced elsewhere: results are row-filtered per namespace by ObjectFilterPlaceHolder" + + " + FilterDataSourceV2Strategy; Spark eagerly executes the bare command in a nested" + + " QueryExecution whose unfiltered result the placeholder discards", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.catalyst.plans.logical.ShowTables", + "Enforced elsewhere: results are row-filtered per table by ObjectFilterPlaceHolder" + + " + FilterDataSourceV2Strategy; Spark eagerly executes the bare command in a nested" + + " QueryExecution whose unfiltered result the placeholder discards", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.execution.ExternalRDD", + "Wraps a session-created RDD/local collection (e.g. spark.createDataset, and Delta's" + + " internal VACUUM plumbing); RDD-level access is outside the plugin's scope and is" + + " an existing, separate concern", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.execution.LogicalRDD", + "Wraps a pre-existing RDD; RDD-level access is outside the plugin's scope and is an" + + " existing, separate concern", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.execution.columnar.InMemoryRelation", + "Cached query results; the originating plan was authorized when the cache was" + + " populated", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.execution.command.DropTempViewCommand", + "Operates only on session-local temporary views, which are deliberately not authz" + + " resources (their reads are authorized against the underlying tables); see" + + " KYUUBI #3426", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.execution.command.ResetCommand", + "Resets session configuration only; sensitive configs are separately guarded by" + + " AuthzConfigurationChecker", + spark3x), + HarmlessNodeSpec( + "org.apache.spark.sql.execution.command.SetCommand", + "Sets session configuration only; sensitive configs are separately guarded by" + + " AuthzConfigurationChecker", + spark3x)) +} From fb44a00ecff89223cf70cfead2da9710c4544d3a Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Wed, 15 Jul 2026 15:34:12 -0700 Subject: [PATCH 2/7] [KYUUBI #7593][AUTHZ] Sweep plan subtrees skipped by privilege building --- .rat-excludes | 1 + .../kyuubi-spark-authz/docs/paranoid-mode.md | 34 +++- .../main/resources/known_harmless_spec.json | 12 ++ .../spark/authz/serde/CommandSpec.scala | 6 +- .../plugin/spark/authz/util/AuthZUtils.scala | 2 +- .../classification_backlog_spark_4.1.txt | 3 + .../authz/ClassificationCoverageSuite.scala | 172 ++++++++++++------ .../spark/authz/gen/KnownHarmlessNodes.scala | 18 ++ 8 files changed, 181 insertions(+), 67 deletions(-) diff --git a/.rat-excludes b/.rat-excludes index f7a57ec3c9d..8b8ab71fb82 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -37,6 +37,7 @@ build/scala-*/** **/**/server_operation_logs/**/** **/**/engine_operation_logs/**/** **/*.output.schema +**/classification_backlog_spark_*.txt **/apache-kyuubi-*-bin*/** **/benchmarks/** **/org.apache.spark.status.AppHistoryServerPlugin diff --git a/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md b/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md index cd1e8538f72..de2166fec99 100644 --- a/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md +++ b/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md @@ -171,7 +171,7 @@ and is maintained by the same generator (`KnownHarmlessNodes` in the test tree, auditor can read, not a reflexive silencing. Each entry also names the exact Spark minor versions its review applies to (§4.5); on any other version the entry is inert. -Two patterns emerged during triage that future entries should be checked against: +Three patterns emerged during triage that future entries should be checked against: - **"Reads no stored data"** — `LocalRelation`, `OneRowRelation`, `Range`, `CTERelationRef`, `CommandResult`, session-conf commands. The straightforward kind. @@ -183,6 +183,10 @@ Two patterns emerged during triage that future entries should be checked against `RuleAuthorization` with the bare command as root and **must be allowed** — enforcement lives in the placeholder machinery, not in `PrivilegesBuilder`. Denying it breaks every SHOW query. This subtlety is load-bearing; do not "fix" it. +- **"This plugin's own machinery"** — the `FilteredShow*Command` wrappers the plugin + installs in place of the v1 SHOW commands. They are `Command`s by shape, handled by + dedicated dispatch arms in `PrivilegesBuilder.build`, with per-row access checks inside + the wrapper itself. Allowlisting a `Command` is higher-stakes than allowlisting a leaf relation, so it takes a second, colocated review: the entry must also be exempted by name in @@ -281,14 +285,25 @@ time instead of in production: 2. **Allowlist re-review.** Allowlisted classes that are (or became) `Command`s on this classpath fail unless explicitly exempted in the suite (§4.3), and no class may have both a spec and an allowlist entry. -3. **Enumeration.** Scan the jars that can contribute plan nodes (spark-catalyst, sql-core, - hive, plus whichever catalog plugins are on this profile's classpath), enumerate every - concrete class that is a `Command`, `LeafNode`, or `ExecutableDuringAnalysis`, and diff - against specs ∪ the allowlist entries verified for this profile's Spark minor (§4.5). - The unclassified remainder is pinned, one classname per line, in - `src/test/resources/classification_backlog_spark_.txt` (currently 136 entries - for 3.5, 181 for 4.1 — the 4.1 figure includes allowlist entries awaiting 4.x - re-review). A class **new** to the diff fails the build with an actionable +3. **Enumeration (total accounting).** Scan every code source that can contribute plan + nodes — the Spark jars, whichever catalog plugins are on this profile's classpath, and + this plugin's own classes directory (its markers and filtered-SHOW wrappers are plan + nodes too) — and enumerate **every concrete `LogicalPlan` descendant**. Each class + lands in exactly one bucket: + - *spec'd* — a command/scan spec (or nodeName match) builds privileges for it: + definitively authz-relevant; + - *allowlisted* — reviewed as harmless for this profile's Spark minor (§4.5); + - *pass-through* — neither `Command`, `LeafNode`, nor analysis-time-executable: + `buildQuery` recurses through it, and whatever carries relevance beneath it is + itself in the enumeration; + - *dirty laundry* — relevant by shape but neither spec'd nor allowlisted, pinned one + classname per line in `src/test/resources/classification_backlog_spark_.txt` + (currently 136 entries for 3.5, 181 for 4.1 — the 4.1 figure includes allowlist + entries awaiting 4.x re-review). + + A companion check keeps the allowlist honest from the other side: an entry whose class + is a pass-through shape fails the build, because nothing would ever consult it and it + would read as coverage. A class **new** to the diff fails the build with an actionable message: classify it, allowlist it with a reason, or consciously regenerate the backlog. A class that leaves the diff must also leave the backlog, so the backlog only ever shrinks by being triaged, never silently. @@ -337,3 +352,4 @@ provably complete. 3. This changes user-visible security posture, so it should land discuss-first: framed as "deny-by-default mode for unclassified plan nodes," with a search of existing issues and discussions for prior art before opening a new one. + diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json index fe698cbf48c..e640b97187f 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json @@ -1,4 +1,16 @@ [ { + "classname" : "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowColumnsCommand", + "reason" : "This plugin's own row-filtering replacement for ShowColumnsCommand (installed by RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a dedicated dispatch arm and every result row is checked for SHOWCOLUMNS access", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowFunctionsCommand", + "reason" : "This plugin's own row-filtering replacement for ShowFunctionsCommand (installed by RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a dedicated dispatch arm and every result row is checked for SHOWFUNCTIONS access", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowTablesCommand", + "reason" : "This plugin's own row-filtering replacement for ShowTablesCommand (installed by RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a dedicated dispatch arm and every result row is checked for SHOWTABLES access", + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { "classname" : "org.apache.spark.sql.catalyst.analysis.ResolvedNamespace", "reason" : "Analysis-time resolution artifact naming a namespace; reads no data itself, and the commands resolved over it are classified in their own right", "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala index 3ca38c9f02b..8bb32df51a6 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala @@ -219,7 +219,9 @@ case class ScanSpec( None } } - (tables, failures) + // .toSeq matters cross-build: on Scala 2.13 scala.Seq is immutable.Seq, which a + // mutable ArrayBuffer does not conform to + (tables, failures.toSeq) } def uris: LogicalPlan => Seq[Uri] = plan => { @@ -239,7 +241,7 @@ case class ScanSpec( None } } - (uris, failures) + (uris, failures.toSeq) } def functions: Expression => Seq[Function] = expr => { diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala index b44e38ae6f3..a4514697f5e 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala @@ -97,7 +97,7 @@ private[authz] object AuthZUtils { * the trait does not exist on all supported Spark versions, and the set of such "dangerous" * supertypes is open, so this must never be the only line of defense. */ - lazy private val executableDuringAnalysisClass: Option[Class[_]] = { + private lazy val executableDuringAnalysisClass: Option[Class[_]] = { try { Some(Class.forName("org.apache.spark.sql.catalyst.plans.logical.ExecutableDuringAnalysis")) } catch { diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt index 530fa82d4c5..9c2f603dfbc 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt @@ -1,3 +1,6 @@ +org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowColumnsCommand +org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowFunctionsCommand +org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowTablesCommand org.apache.spark.sql.catalyst.TimeTravel org.apache.spark.sql.catalyst.analysis.CurrentNamespace$ org.apache.spark.sql.catalyst.analysis.RelationTimeTravel diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala index 823149687b4..8ea96745971 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala @@ -22,6 +22,7 @@ import java.nio.file.{Files, Paths, StandardOpenOption} import java.util.jar.JarFile import scala.collection.JavaConverters._ +import scala.collection.mutable import org.apache.spark.sql.catalyst.plans.logical.{Command, LeafNode, LogicalPlan} // scalastyle:off @@ -109,7 +110,11 @@ class ClassificationCoverageSuite extends AnyFunSuite { // session-local temp views are deliberately not authz resources "org.apache.spark.sql.execution.command.DropTempViewCommand", // resolves to literally nothing to execute - "org.apache.spark.sql.catalyst.plans.logical.NoopCommand") + "org.apache.spark.sql.catalyst.plans.logical.NoopCommand", + // the plugin's own row-filtering wrappers, handled by dedicated dispatch arms + "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowTablesCommand", + "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowFunctionsCommand", + "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowColumnsCommand") if (classOf[Command].isAssignableFrom(cls) && !exempted.contains(name)) { Some(name) } else { @@ -129,87 +134,144 @@ class ClassificationCoverageSuite extends AnyFunSuite { // Enumeration check: diff the classpath's plan-node population against our classification. // --------------------------------------------------------------------------------------- - /** Scan prefixes covering Spark itself and the catalog plugins that inject plan nodes. */ + /** Scan prefixes covering Spark, catalog plugins, and this plugin's own plan nodes. */ private val scannedPackagePrefixes = Seq( "org/apache/spark/sql/", - "org/apache/paimon/spark/") + "org/apache/paimon/spark/", + "org/apache/kyuubi/plugin/spark/authz/") - /** Classes whose presence identifies a jar that can contribute logical plan nodes. */ - private val jarAnchorClassnames = Seq( + /** Classes whose presence identifies a code source that can contribute plan nodes. */ + private val codeSourceAnchorClassnames = Seq( "org.apache.spark.sql.catalyst.plans.logical.LogicalPlan", // spark-catalyst "org.apache.spark.sql.execution.SparkPlan", // spark-sql core "org.apache.spark.sql.hive.HiveSessionStateBuilder", // spark-hive "org.apache.iceberg.spark.SparkCatalog", // iceberg runtime, if on this profile "org.apache.spark.sql.delta.DeltaLog", // delta, if on this profile "org.apache.spark.sql.hudi.command.CallProcedureHoodieCommand", // hudi, if on this profile - "org.apache.paimon.spark.SparkCatalog" // paimon, if on this profile - ) + "org.apache.paimon.spark.SparkCatalog", // paimon, if on this profile + // this plugin's own plan nodes (markers, filtered SHOW wrappers) — a directory code + // source under target/, not a jar, and just as capable of hiding dirty laundry + "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.ObjectFilterPlaceHolder") - /** - * Concrete classes on this profile's classpath that carry authorization relevance by - * shape: every Command, every LeafNode, and everything that executes during analysis. - * These are exactly the shapes the runtime invariant refuses to let pass silently. - */ - private def enumerateRelevantPlanClasses(): Seq[String] = { + /** Relative `.class` entry names within one code source: a jar or a classes directory. */ + private def classEntriesIn(location: java.net.URL): Seq[String] = { + val path = Paths.get(location.toURI) + if (Files.isDirectory(path)) { + val stream = Files.walk(path) + try { + stream.iterator().asScala + .map(p => path.relativize(p).toString.replace(java.io.File.separatorChar, '/')) + .filter(_.endsWith(".class")) + .toList + } finally { + stream.close() + } + } else if (path.toString.endsWith(".jar")) { + val jar = new JarFile(path.toFile) + try { + // strict: the iterator must be exhausted before the jar closes + jar.entries().asScala.map(_.getName).filter(_.endsWith(".class")).toList + } finally { + jar.close() + } + } else { + Nil + } + } + + /** Every concrete `LogicalPlan` descendant on this profile's classpath. */ + private def enumeratePlanNodeClasses(): Seq[Class[_]] = { val loader = getClass.getClassLoader - val jars = jarAnchorClassnames.flatMap(loadable).flatMap { cls => + val locations = codeSourceAnchorClassnames.flatMap(loadable).flatMap { cls => Option(cls.getProtectionDomain.getCodeSource).map(_.getLocation) - }.distinct.filter(_.getPath.endsWith(".jar")) - - val relevantSupertypes: Seq[Class[_]] = - Seq(classOf[Command], classOf[LeafNode]) ++ executableDuringAnalysisClass + }.distinct - jars.flatMap { jarUrl => - val jar = new JarFile(Paths.get(jarUrl.toURI).toFile) - try { - jar.entries().asScala - .map(_.getName) - .filter(n => n.endsWith(".class") && scannedPackagePrefixes.exists(n.startsWith)) - // skip anonymous/synthetic classes; keep nested ones (plan nodes can be nested) - .filterNot(n => n.contains("$$") || n.matches(".*\\$\\d.*")) - .map(_.stripSuffix(".class").replace('/', '.')) - .flatMap { classname => - try { - val cls = Class.forName(classname, false, loader) - val concrete = - !cls.isInterface && !java.lang.reflect.Modifier.isAbstract(cls.getModifiers) - if (concrete && - classOf[LogicalPlan].isAssignableFrom(cls) && - relevantSupertypes.exists(_.isAssignableFrom(cls))) { - Some(classname) - } else { - None - } - } catch { - // optional dependencies of scanned classes may be absent at test time - case _: Throwable => None + val byName = mutable.SortedMap.empty[String, Class[_]] + locations.foreach { location => + classEntriesIn(location) + .filter(n => scannedPackagePrefixes.exists(n.startsWith)) + // skip anonymous/synthetic classes; keep nested ones (plan nodes can be nested) + .filterNot(n => n.contains("$$") || n.matches(".*\\$\\d.*")) + .map(_.stripSuffix(".class").replace('/', '.')) + .foreach { classname => + try { + val cls = Class.forName(classname, false, loader) + val concrete = + !cls.isInterface && !java.lang.reflect.Modifier.isAbstract(cls.getModifiers) + if (concrete && classOf[LogicalPlan].isAssignableFrom(cls)) { + byName.getOrElseUpdate(classname, cls) } - }.toList // strict: the iterator must be exhausted before the jar closes - } finally { - jar.close() + } catch { + // optional dependencies of scanned classes may be absent at test time + case _: Throwable => + } + } + } + byName.values.toSeq + } + + /** + * The shapes that can carry or hide authorization relevance on their own: every Command, + * every LeafNode, and everything that executes during analysis. These are exactly the + * shapes the runtime invariant refuses to let pass silently; anything else is a + * pass-through operator that [[PrivilegesBuilder.buildQuery]] recurses through, whose + * authz-relevant content is itself a plan node found by the same enumeration. + */ + private def isAuthzRelevantByShape(cls: Class[_]): Boolean = { + classOf[Command].isAssignableFrom(cls) || + classOf[LeafNode].isAssignableFrom(cls) || + executableDuringAnalysisClass.exists(_.isAssignableFrom(cls)) + } + + test("allowlist entries present on this classpath are shapes the classifier would flag") { + // An allowlist entry for a pass-through operator would be dead weight that reads as + // coverage: nothing ever consults it, because pass-through nodes recurse freely. + val pointless = KNOWN_HARMLESS_NODES.keySet.toSeq.sorted.flatMap { name => + loadable(name) match { + case Some(cls) if !isAuthzRelevantByShape(cls) => Some(name) + case _ => None } - }.distinct.sorted + } + assert( + pointless.isEmpty, + s"\nThese allowlisted classes are pass-through operators that are never consulted;" + + s" remove the entries:\n ${pointless.mkString("\n ")}") } - test("enumerate authz-relevant plan classes and diff against the classification") { - // Golden backlog file, one classname per line, per Spark minor version. The contract: - // a class NEW to this diff fails the build — classify it (command/scan spec), allowlist - // it with a reason, or consciously add it to the backlog via regeneration. A class that - // leaves the diff must also leave the backlog, so the backlog only ever shrinks by - // being triaged, never silently. + test("every plan node class on this classpath is accounted for") { + // Total accounting over every concrete LogicalPlan descendant the scan can see. Each + // class lands in exactly one bucket: + // 1. spec'd — a command/scan spec (or nodeName match) will build + // privileges for it: definitively authz-relevant; + // 2. allowlisted — reviewed as harmless for THIS Spark minor + // (an entry only counts on versions it was verified against); + // 3. pass-through — neither Command, LeafNode, nor analysis-time-executable: + // buildQuery recurses through it, and whatever carries + // relevance beneath it is itself in this enumeration; + // 4. dirty laundry — relevant by shape but neither spec'd nor allowlisted, + // pinned in the golden backlog file for this Spark minor. + // The contract on bucket 4: a class NEW to it fails the build — classify it, allowlist + // it with a reason, or consciously regenerate the backlog. A class that leaves the + // diff must also leave the backlog, so it only ever shrinks by being triaged. val backlogFilename = s"classification_backlog_spark_$SPARK_RUNTIME_MAJOR_MINOR.txt" val backlogPath = Paths.get( s"${getCurrentModuleHome(this)}/src/test/resources/$backlogFilename") val classified: Set[String] = allCommandSpecClassnames ++ - // an allowlist entry only classifies on the Spark minors it was reviewed against; - // on this profile the others belong in the backlog awaiting re-review KNOWN_HARMLESS_NODES.filter(_._2.appliesTo(SPARK_RUNTIME_MAJOR_MINOR)).keySet ++ // matched by nodeName rather than classname in buildQuery Set("org.apache.spark.sql.catalyst.analysis.UnresolvedRelation") ++ SCAN_SPEC_CLASSNAMES - val unclassified = enumerateRelevantPlanClasses().filterNot(classified) + val allPlanNodes = enumeratePlanNodeClasses() + val (relevantByShape, passThrough) = allPlanNodes.partition(isAuthzRelevantByShape) + val unclassified = relevantByShape.map(_.getName).filterNot(classified).sorted + + // the partition plus the golden diff below account for every enumerated class; log the + // shape of the population so coverage drift is visible in the test output + info(s"${allPlanNodes.size} plan node classes: ${relevantByShape.size} authz-relevant" + + s" by shape (${unclassified.size} in backlog), ${passThrough.size} pass-through") + val generatedStr = unclassified.mkString("", "\n", "\n") if (sys.env.get("KYUUBI_UPDATE").contains("1")) { diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala index f3957a05a5e..c72076aaaf4 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala @@ -40,6 +40,24 @@ object KnownHarmlessNodes { private val spark3xAnd41 = spark3x :+ "4.1" val specs: Seq[HarmlessNodeSpec] = Seq( + HarmlessNodeSpec( + "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowColumnsCommand", + "This plugin's own row-filtering replacement for ShowColumnsCommand (installed by" + + " RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a" + + " dedicated dispatch arm and every result row is checked for SHOWCOLUMNS access", + spark3x), + HarmlessNodeSpec( + "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowFunctionsCommand", + "This plugin's own row-filtering replacement for ShowFunctionsCommand (installed by" + + " RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a" + + " dedicated dispatch arm and every result row is checked for SHOWFUNCTIONS access", + spark3x), + HarmlessNodeSpec( + "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowTablesCommand", + "This plugin's own row-filtering replacement for ShowTablesCommand (installed by" + + " RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a" + + " dedicated dispatch arm and every result row is checked for SHOWTABLES access", + spark3x), HarmlessNodeSpec( "org.apache.spark.sql.catalyst.analysis.ResolvedNamespace", "Analysis-time resolution artifact naming a namespace; reads no data itself, and the" + From 33041c1b2bae0c8c0c1b94c43eec88a8b9bc2928 Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Thu, 16 Jul 2026 17:09:09 -0700 Subject: [PATCH 3/7] [KYUUBI #7593][AUTHZ] Fix Iceberg metadata table and MERGE INTO fail-opens Rebase fallout: - AlterColumns (new in upstream Spark 4 port) gets explicit verifiedSparkVersions 4.0/4.1/4.2 - the pre-Spark-4 default baseline cannot apply to it - Regenerate 3.5 backlog: Hudi 1.2.0 renamed plans.logcal -> plans.logical Found by running the existing suites in deny mode on upstream master: - Iceberg metadata tables (t.snapshots etc) were never authorized: their 4-part name() blew up StringTableExtractor and the MatchError vanished into the fail-open path. TableTableExtractor now reflectively unwraps BaseMetadataTable to its base table, so metadata reads are authorized as base-table reads; new regression test. - Iceberg MERGE INTO embeds an already-planned DataSourceV2ScanRelation the builder skipped silently; classified with its own scan spec. --- .../kyuubi-spark-authz/docs/paranoid-mode.md | 16 +++++++++ .../src/main/resources/scan_command_spec.json | 11 +++++++ .../main/resources/table_command_spec.json | 3 +- .../spark/authz/serde/tableExtractors.scala | 33 ++++++++++++++++++- .../classification_backlog_spark_3.5.txt | 17 +++++----- .../kyuubi/plugin/spark/authz/gen/Scans.scala | 15 +++++++++ .../spark/authz/gen/TableCommands.scala | 4 ++- ...bergCatalogRangerSparkExtensionSuite.scala | 19 +++++++++-- 8 files changed, 104 insertions(+), 14 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md b/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md index de2166fec99..dc8a5b333e6 100644 --- a/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md +++ b/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md @@ -42,6 +42,22 @@ enumeration check (§6) first ran, it counted **136 unclassified authz-relevant on the Spark 3.5 classpath and 170 on Spark 4.1** — the population that fail-open behavior had been hiding. +Merely running the existing test suite in deny mode then surfaced two live gaps on master +that green CI had been certifying for years: + +- **Iceberg metadata tables were never authorized.** `SELECT * FROM t.snapshots` produces a + `DataSourceV2Relation` whose table reports a four-part name; the extractor threw a + `MatchError` that vanished into the fail-open path, so metadata reads (snapshot history, + manifests, partition statistics) required no privilege at all. Deny mode turned the + swallowed exception into a violation, and the fix authorizes metadata reads as reads of + the base table. +- **Connector-planned scans were skipped.** Iceberg's MERGE INTO rewrite embeds an + already-planned `DataSourceV2ScanRelation` — the rewrite's own read of the target table — + a leaf the builder did not recognize and silently skipped. The named source and target + objects were still checked through the command spec, so every test kept passing; the + embedded scan is exactly the kind of node nobody thought to test. It is now classified + with its own scan spec. + A green test suite does not bound this risk: every test was written for a node type someone had already classified, so the suite certifies that the *previously known* sample still authorizes correctly and says nothing about nodes nobody thought to test. diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json index 0aabde3ff2a..177c5849733 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json @@ -47,6 +47,17 @@ "functionDescs" : [ ], "uriDescs" : [ ], "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation", + "scanDescs" : [ { + "fieldName" : "relation", + "fieldExtractor" : "DataSourceV2RelationTableExtractor", + "catalogDesc" : null, + "comment" : "" + } ], + "functionDescs" : [ ], + "uriDescs" : [ ], + "verifiedSparkVersions" : [ "3.5" ] }, { "classname" : "org.apache.spark.sql.hive.HiveGenericUDF", "scanDescs" : [ ], diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json index e10eb002556..82d0537b871 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json @@ -79,7 +79,8 @@ "fieldExtractor" : "ResolvedTableURIExtractor", "isInput" : false, "comment" : "" - } ] + } ], + "verifiedSparkVersions" : [ "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AlterTable", "tableDescs" : [ { diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala index 9729857a736..9676a719f33 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala @@ -167,6 +167,8 @@ class StringTableExtractor extends TableExtractor { case 1 => Table(None, None, tableNameArr(0), None) case 2 => Table(None, Some(tableNameArr(0)), tableNameArr(1), None) case 3 => Table(Some(tableNameArr(0)), Some(tableNameArr(1)), tableNameArr(2), None) + case n => throw new IllegalStateException( + s"table name '$v1' splits into $n parts; only 1-3 (catalog.db.table) are supported") } Option(maybeTable) } @@ -312,11 +314,40 @@ class SubqueryAliasTableExtractor extends TableExtractor { */ class TableTableExtractor extends TableExtractor { override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { - val tableName = invokeAs[String](v1, "name") + val tableName = invokeAs[String](TableTableExtractor.unwrapMetadataTable(v1), "name") lookupExtractor[StringTableExtractor].apply(spark, tableName) } } +object TableTableExtractor { + + /** + * An Iceberg metadata table (`t.snapshots`, `t.history`, ...) reports `name()` as + * `.`, which overflows the `catalog.db.table` form + * and names an object no policy can contain; reading table metadata is authorized + * as a read of the base table instead. Iceberg is an optional runtime dependency, + * so the metadata-table shape (a connector table whose `table()` payload extends + * `BaseMetadataTable`, whose own `table()` is the base table) is probed reflectively. + */ + private def unwrapMetadataTable(table: AnyRef): AnyRef = { + scala.util.Try(invokeAs[AnyRef](table, "table")) + .filter(isIcebergMetadataTable) + .flatMap(meta => scala.util.Try(invokeAs[AnyRef](meta, "table"))) + .getOrElse(table) + } + + private def isIcebergMetadataTable(o: AnyRef): Boolean = { + var clz: Class[_] = o.getClass + while (clz != null) { + if (clz.getName == "org.apache.iceberg.BaseMetadataTable") { + return true + } + clz = clz.getSuperclass + } + false + } +} + class HudiDataSourceV2RelationTableExtractor extends TableExtractor { override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { invokeAs[LogicalPlan](v1, "table") match { diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_3.5.txt b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_3.5.txt index 989fac0f9ac..425921de156 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_3.5.txt +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_3.5.txt @@ -23,14 +23,6 @@ org.apache.spark.sql.catalyst.analysis.UnresolvedView org.apache.spark.sql.catalyst.catalog.TemporaryViewRelation org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation org.apache.spark.sql.catalyst.encoders.DummyExpressionHolder -org.apache.spark.sql.catalyst.plans.logcal.HoodieFileSystemViewTableValuedFunction -org.apache.spark.sql.catalyst.plans.logcal.HoodieFileSystemViewTableValuedFunctionByPath -org.apache.spark.sql.catalyst.plans.logcal.HoodieMetadataTableValuedFunction -org.apache.spark.sql.catalyst.plans.logcal.HoodieQuery -org.apache.spark.sql.catalyst.plans.logcal.HoodieTableChanges -org.apache.spark.sql.catalyst.plans.logcal.HoodieTableChangesByPath -org.apache.spark.sql.catalyst.plans.logcal.HoodieTimelineTableValuedFunction -org.apache.spark.sql.catalyst.plans.logcal.HoodieTimelineTableValuedFunctionByPath org.apache.spark.sql.catalyst.plans.logical.AlterColumnSyncIdentity org.apache.spark.sql.catalyst.plans.logical.AlterTableAddConstraint org.apache.spark.sql.catalyst.plans.logical.AlterTableDropConstraint @@ -54,6 +46,14 @@ org.apache.spark.sql.catalyst.plans.logical.DescribeFunction org.apache.spark.sql.catalyst.plans.logical.DropFunction org.apache.spark.sql.catalyst.plans.logical.DropIndex org.apache.spark.sql.catalyst.plans.logical.DropView +org.apache.spark.sql.catalyst.plans.logical.HoodieFileSystemViewTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieFileSystemViewTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieMetadataTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieQuery +org.apache.spark.sql.catalyst.plans.logical.HoodieTableChanges +org.apache.spark.sql.catalyst.plans.logical.HoodieTableChangesByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieTimelineTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieTimelineTableValuedFunctionByPath org.apache.spark.sql.catalyst.plans.logical.LoadData org.apache.spark.sql.catalyst.plans.logical.RecoverPartitions org.apache.spark.sql.catalyst.plans.logical.RefreshFunction @@ -125,7 +125,6 @@ org.apache.spark.sql.execution.command.ShowViewsCommand org.apache.spark.sql.execution.command.StreamingExplainCommand org.apache.spark.sql.execution.datasources.RefreshResource org.apache.spark.sql.execution.datasources.SparkExpressionConverter$DummyRelation -org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation org.apache.spark.sql.execution.datasources.v2.ScanBuilderHolder org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation org.apache.spark.sql.execution.streaming.OffsetHolder diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala index ed17dc5dc43..c949a8f92f1 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala @@ -50,6 +50,20 @@ object Scans extends CommandSpecs[ScanSpec] { ScanSpec(r, Seq(tableDesc)) } + // Post-pushdown leaf produced by V2ScanRelationPushDown. Normally authorization runs + // before pushdown, but a connector's own rewrites can embed an already-planned scan + // (e.g. Iceberg MERGE INTO on Spark 3.5), and then this is the only node carrying the + // table. The wrapped v2 relation sits in the `relation` field, not a child, so the + // regular DataSourceV2Relation spec never sees it. + val DataSourceV2ScanRelation = { + val r = "org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation" + val tableDesc = + ScanDesc( + "relation", + classOf[DataSourceV2RelationTableExtractor]) + ScanSpec(r, Seq(tableDesc), verifiedSparkVersions = Seq("3.5")) + } + val PermanentViewMarker = { val r = "org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker" val tableDesc = @@ -84,6 +98,7 @@ object Scans extends CommandSpecs[ScanSpec] { HiveTableRelation, LogicalRelation, DataSourceV2Relation, + DataSourceV2ScanRelation, PermanentViewMarker, HiveSimpleUDF, HiveGenericUDF, diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala index 34acdfcc2b0..ff2666666d7 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala @@ -76,7 +76,9 @@ object TableCommands extends CommandSpecs[TableCommandSpec] { // is still carried in `child` as a `ResolvedTable`, so it reuses the AddColumns spec. val AlterColumns = { val cmd = "org.apache.spark.sql.catalyst.plans.logical.AlterColumns" - AddColumns.copy(classname = cmd) + // The node itself only exists on Spark 4.0+, so the pre-Spark-4 default baseline + // cannot apply to it. + AddColumns.copy(classname = cmd, verifiedSparkVersions = Seq("4.0", "4.1", "4.2")) } val DropColumns = { diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala index c4f99623a08..778fd0c410b 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala @@ -324,11 +324,26 @@ class IcebergCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite } private def getFirstSnapshot(table: String): Row = { - val existedSnapshots = - sql(s"SELECT * FROM $table.snapshots ORDER BY committed_at ASC LIMIT 1").collect() + // Reading the snapshots metadata table is authorized as a read of the base table + // (see TableTableExtractor), so this fixture query needs admin like the writes above. + val existedSnapshots = doAs( + admin, + sql(s"SELECT * FROM $table.snapshots ORDER BY committed_at ASC LIMIT 1").collect()) existedSnapshots(0) } + test("selecting an Iceberg metadata table requires select on the base table") { + val tableName = "table_metadata_select" + val table = s"$catalogV2.$namespace1.$tableName" + withCleanTmpResources(Seq((table, "table"))) { + prepareExampleIcebergTable(table, 1) + val selectSnapshots = s"SELECT committed_at FROM $table.snapshots" + interceptEndsWith[AccessControlException](doAs(someone, sql(selectSnapshots).collect()))( + s"does not have [select] privilege on [$namespace1/$tableName/committed_at]") + doAs(admin, sql(selectSnapshots).collect()) + } + } + test("CALL rollback_to_snapshot") { val tableName = "table_rollback_to_snapshot" val table = s"$catalogV2.$namespace1.$tableName" From cce61f821df613433295b1cdaaffd92135b8da77 Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Thu, 16 Jul 2026 17:35:09 -0700 Subject: [PATCH 4/7] [KYUUBI #7593][AUTHZ] Classify Spark 4 nodes found in deny mode Full module suite passes in deny mode under -Pspark-4.1 -Pscala-2.13 (642 tests). New Spark 4 classifications, both caught by deny-mode runs: - SaveAsV1TableCommand (SPARK-49246): DataFrameWriter.saveAsTable v1 path analyzes into this leaf wrapper which only plans the real CTAS inside a nested QueryExecution at run time; spec'd directly (CREATETABLE_AS_SELECT) so the write is authorized on the outer plan instead of relying on the nested pass - ShowNamespacesCommand: Spark 4.x's v1 SHOW DATABASES; allowlisted with the same 'enforced elsewhere' rationale as v2 ShowNamespaces (upstream's port already routes it through ObjectFilterPlaceHolder row filtering), plus the required second exemption in ClassificationCoverageSuite Allowlist entries re-reviewed and extended to 4.0/4.1/4.2 (full-suite deny runs per profile are the verification vehicle); refreshed 4.1 backlog against upstream Spark 4.1.2 + Iceberg 1.11/Hudi 1.2/Delta 4.3/Paimon 1.4 (195 entries) --- .../main/resources/known_harmless_spec.json | 40 +++++++----- .../main/resources/table_command_spec.json | 26 ++++++++ .../classification_backlog_spark_4.1.txt | 65 +++++++++++-------- .../authz/ClassificationCoverageSuite.scala | 2 + .../spark/authz/gen/KnownHarmlessNodes.scala | 58 ++++++++++------- .../spark/authz/gen/TableCommands.scala | 22 +++++++ 6 files changed, 143 insertions(+), 70 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json index e640b97187f..4fb56d21d10 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/known_harmless_spec.json @@ -1,73 +1,77 @@ [ { "classname" : "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowColumnsCommand", "reason" : "This plugin's own row-filtering replacement for ShowColumnsCommand (installed by RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a dedicated dispatch arm and every result row is checked for SHOWCOLUMNS access", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowFunctionsCommand", "reason" : "This plugin's own row-filtering replacement for ShowFunctionsCommand (installed by RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a dedicated dispatch arm and every result row is checked for SHOWFUNCTIONS access", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowTablesCommand", "reason" : "This plugin's own row-filtering replacement for ShowTablesCommand (installed by RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a dedicated dispatch arm and every result row is checked for SHOWTABLES access", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.catalyst.analysis.ResolvedNamespace", "reason" : "Analysis-time resolution artifact naming a namespace; reads no data itself, and the commands resolved over it are classified in their own right", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CTERelationRef", "reason" : "Leaf reference to a CTE definition; the definition's own plan appears under WithCTE in the same tree and is authorized there", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CommandResult", "reason" : "Holds rows already produced by an eagerly executed command; that command was authorized when it executed", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.LocalRelation", "reason" : "Holds in-memory literal rows (VALUES lists, createDataFrame); reads no stored data", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.1" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.NoopCommand", "reason" : "Spark's placeholder for commands with nothing to do (e.g. IF EXISTS / IF NOT EXISTS variants when the object is absent); executes nothing", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.OneRowRelation", "reason" : "The implicit single-row relation backing SELECT without FROM; reads no stored data", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.1" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.Range", "reason" : "Generates rows from a numeric range (e.g. spark.range); reads no stored data", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.1" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ShowNamespaces", "reason" : "Enforced elsewhere: results are row-filtered per namespace by ObjectFilterPlaceHolder + FilterDataSourceV2Strategy; Spark eagerly executes the bare command in a nested QueryExecution whose unfiltered result the placeholder discards", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ShowTables", "reason" : "Enforced elsewhere: results are row-filtered per table by ObjectFilterPlaceHolder + FilterDataSourceV2Strategy; Spark eagerly executes the bare command in a nested QueryExecution whose unfiltered result the placeholder discards", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.execution.ExternalRDD", "reason" : "Wraps a session-created RDD/local collection (e.g. spark.createDataset, and Delta's internal VACUUM plumbing); RDD-level access is outside the plugin's scope and is an existing, separate concern", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.execution.LogicalRDD", "reason" : "Wraps a pre-existing RDD; RDD-level access is outside the plugin's scope and is an existing, separate concern", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.execution.columnar.InMemoryRelation", "reason" : "Cached query results; the originating plan was authorized when the cache was populated", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.execution.command.DropTempViewCommand", "reason" : "Operates only on session-local temporary views, which are deliberately not authz resources (their reads are authorized against the underlying tables); see KYUUBI #3426", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.execution.command.ResetCommand", "reason" : "Resets session configuration only; sensitive configs are separately guarded by AuthzConfigurationChecker", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.execution.command.SetCommand", "reason" : "Sets session configuration only; sensitive configs are separately guarded by AuthzConfigurationChecker", - "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] + "verifiedSparkVersions" : [ "3.3", "3.4", "3.5", "4.0", "4.1", "4.2" ] +}, { + "classname" : "org.apache.spark.sql.execution.command.ShowNamespacesCommand", + "reason" : "Enforced elsewhere: Spark 4.x's v1 SHOW DATABASES/NAMESPACES command; results are row-filtered per namespace by ObjectFilterPlaceHolder + FilterDataSourceV2Strategy exactly like v2 ShowNamespaces, and the bare command Spark eagerly executes in a nested QueryExecution has its unfiltered result discarded by the placeholder", + "verifiedSparkVersions" : [ "4.0", "4.1", "4.2" ] } ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json index 82d0537b871..579dc932d95 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json @@ -1560,6 +1560,32 @@ "queryDescs" : [ ], "uriDescs" : [ ], "verifiedSparkVersions" : [ "3.3", "3.4", "3.5" ] +}, { + "classname" : "org.apache.spark.sql.execution.command.SaveAsV1TableCommand", + "tableDescs" : [ { + "fieldName" : "tableDesc", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : true, + "comment" : "" + } ], + "opType" : "CREATETABLE_AS_SELECT", + "queryDescs" : [ { + "fieldName" : "query", + "fieldExtractor" : "LogicalPlanQueryExtractor", + "comment" : "" + } ], + "uriDescs" : [ { + "fieldName" : "tableDesc", + "fieldExtractor" : "CatalogTableURIExtractor", + "isInput" : false, + "comment" : "" + } ], + "verifiedSparkVersions" : [ "4.1" ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowColumnsCommand", "tableDescs" : [ { diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt index 9c2f603dfbc..8acec4488f5 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.1.txt @@ -1,12 +1,32 @@ -org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowColumnsCommand -org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowFunctionsCommand -org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowTablesCommand +org.apache.paimon.spark.catalyst.analysis.ResolvedPaimonView +org.apache.paimon.spark.catalyst.plans.logical.CreateOrReplaceTagCommand +org.apache.paimon.spark.catalyst.plans.logical.CreatePaimonView +org.apache.paimon.spark.catalyst.plans.logical.DeleteTagCommand +org.apache.paimon.spark.catalyst.plans.logical.DropPaimonView +org.apache.paimon.spark.catalyst.plans.logical.FullTextSearchQuery +org.apache.paimon.spark.catalyst.plans.logical.IncrementalBetweenTimestamp +org.apache.paimon.spark.catalyst.plans.logical.IncrementalQuery +org.apache.paimon.spark.catalyst.plans.logical.IncrementalToAutoTag +org.apache.paimon.spark.catalyst.plans.logical.PaimonDropPartitions +org.apache.paimon.spark.catalyst.plans.logical.RenameTagCommand +org.apache.paimon.spark.catalyst.plans.logical.ResolvedIdentifier +org.apache.paimon.spark.catalyst.plans.logical.ShowPaimonViews +org.apache.paimon.spark.catalyst.plans.logical.ShowTagsCommand +org.apache.paimon.spark.catalyst.plans.logical.TruncatePaimonTableWithFilter +org.apache.paimon.spark.catalyst.plans.logical.VectorSearchQuery +org.apache.paimon.spark.commands.MergeIntoPaimonDataEvolutionTable +org.apache.paimon.spark.commands.PaimonAnalyzeTableColumnCommand +org.apache.paimon.spark.commands.PaimonDynamicPartitionOverwriteCommand +org.apache.paimon.spark.commands.PaimonShowColumnsCommand +org.apache.paimon.spark.commands.WriteIntoPaimonTable +org.apache.paimon.spark.execution.CreatePaimonV1FunctionCommand +org.apache.paimon.spark.execution.DescribePaimonV1FunctionCommand +org.apache.paimon.spark.execution.DropPaimonV1FunctionCommand org.apache.spark.sql.catalyst.TimeTravel org.apache.spark.sql.catalyst.analysis.CurrentNamespace$ org.apache.spark.sql.catalyst.analysis.RelationTimeTravel org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier org.apache.spark.sql.catalyst.analysis.ResolvedInlineTable -org.apache.spark.sql.catalyst.analysis.ResolvedNamespace org.apache.spark.sql.catalyst.analysis.ResolvedNonPersistentFunc org.apache.spark.sql.catalyst.analysis.ResolvedPersistentFunc org.apache.spark.sql.catalyst.analysis.ResolvedPersistentView @@ -29,18 +49,9 @@ org.apache.spark.sql.catalyst.analysis.resolver.UnresolvedCteRelationRef org.apache.spark.sql.catalyst.catalog.TemporaryViewRelation org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation org.apache.spark.sql.catalyst.encoders.DummyExpressionHolder -org.apache.spark.sql.catalyst.plans.logcal.HoodieFileSystemViewTableValuedFunction -org.apache.spark.sql.catalyst.plans.logcal.HoodieFileSystemViewTableValuedFunctionByPath -org.apache.spark.sql.catalyst.plans.logcal.HoodieMetadataTableValuedFunction -org.apache.spark.sql.catalyst.plans.logcal.HoodieQuery -org.apache.spark.sql.catalyst.plans.logcal.HoodieTableChanges -org.apache.spark.sql.catalyst.plans.logcal.HoodieTableChangesByPath -org.apache.spark.sql.catalyst.plans.logcal.HoodieTimelineTableValuedFunction -org.apache.spark.sql.catalyst.plans.logcal.HoodieTimelineTableValuedFunctionByPath org.apache.spark.sql.catalyst.plans.logical.AddCheckConstraint org.apache.spark.sql.catalyst.plans.logical.AddConstraint org.apache.spark.sql.catalyst.plans.logical.AlterColumnSyncIdentity -org.apache.spark.sql.catalyst.plans.logical.AlterColumns org.apache.spark.sql.catalyst.plans.logical.AlterTableAddConstraint org.apache.spark.sql.catalyst.plans.logical.AlterTableClusterBy org.apache.spark.sql.catalyst.plans.logical.AlterTableCollation @@ -51,9 +62,7 @@ org.apache.spark.sql.catalyst.plans.logical.AlterViewSchemaBinding org.apache.spark.sql.catalyst.plans.logical.AnalyzeColumn org.apache.spark.sql.catalyst.plans.logical.AnalyzeTable org.apache.spark.sql.catalyst.plans.logical.AnalyzeTables -org.apache.spark.sql.catalyst.plans.logical.CTERelationRef org.apache.spark.sql.catalyst.plans.logical.CallCommand -org.apache.spark.sql.catalyst.plans.logical.CommandResult org.apache.spark.sql.catalyst.plans.logical.CompactionPath org.apache.spark.sql.catalyst.plans.logical.CompactionShowOnPath org.apache.spark.sql.catalyst.plans.logical.CompactionShowOnTable @@ -78,8 +87,18 @@ org.apache.spark.sql.catalyst.plans.logical.DropIndex org.apache.spark.sql.catalyst.plans.logical.DropVariable org.apache.spark.sql.catalyst.plans.logical.DropView org.apache.spark.sql.catalyst.plans.logical.EmptyRelation +org.apache.spark.sql.catalyst.plans.logical.HoodieFileSystemViewTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieFileSystemViewTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieMetadataTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieQuery +org.apache.spark.sql.catalyst.plans.logical.HoodieShowIndexes +org.apache.spark.sql.catalyst.plans.logical.HoodieTableChanges +org.apache.spark.sql.catalyst.plans.logical.HoodieTableChangesByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieTimelineTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieTimelineTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieVectorSearchBatchTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieVectorSearchTableValuedFunction org.apache.spark.sql.catalyst.plans.logical.LoadData -org.apache.spark.sql.catalyst.plans.logical.NoopCommand org.apache.spark.sql.catalyst.plans.logical.PythonWorkerLogs org.apache.spark.sql.catalyst.plans.logical.RecoverPartitions org.apache.spark.sql.catalyst.plans.logical.RefreshFunction @@ -90,10 +109,8 @@ org.apache.spark.sql.catalyst.plans.logical.SetVariable org.apache.spark.sql.catalyst.plans.logical.SetViewProperties org.apache.spark.sql.catalyst.plans.logical.ShowColumns org.apache.spark.sql.catalyst.plans.logical.ShowFunctions -org.apache.spark.sql.catalyst.plans.logical.ShowIndexes org.apache.spark.sql.catalyst.plans.logical.ShowPartitions org.apache.spark.sql.catalyst.plans.logical.ShowTablePartition -org.apache.spark.sql.catalyst.plans.logical.ShowTables org.apache.spark.sql.catalyst.plans.logical.ShowTablesExtended org.apache.spark.sql.catalyst.plans.logical.ShowViews org.apache.spark.sql.catalyst.plans.logical.Transpose @@ -129,42 +146,37 @@ org.apache.spark.sql.delta.commands.CloneTableCommand org.apache.spark.sql.delta.commands.ConvertToDeltaCommand org.apache.spark.sql.delta.commands.CreateDeltaTableCommand org.apache.spark.sql.delta.commands.DeltaGenerateCommand +org.apache.spark.sql.delta.commands.DeltaInsertReplaceOnOrUsingCommand org.apache.spark.sql.delta.commands.DeltaReorgTable org.apache.spark.sql.delta.commands.DeltaReorgTableCommand org.apache.spark.sql.delta.commands.DescribeDeltaDetailCommand org.apache.spark.sql.delta.commands.DescribeDeltaHistoryCommand org.apache.spark.sql.delta.commands.RestoreTableCommand org.apache.spark.sql.delta.commands.ShowDeltaTableColumnsCommand +org.apache.spark.sql.delta.commands.TruncateDeltaTableCommand org.apache.spark.sql.delta.commands.WriteIntoDelta org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillCommand +org.apache.spark.sql.delta.commands.backfill.RowTrackingUnBackfillCommand org.apache.spark.sql.delta.constraints.ExpressionLogicalPlanWrapper org.apache.spark.sql.delta.skipping.clustering.temp.AlterTableClusterBy org.apache.spark.sql.delta.skipping.clustering.temp.ClusterByPlan -org.apache.spark.sql.execution.ExternalRDD -org.apache.spark.sql.execution.LogicalRDD org.apache.spark.sql.execution.adaptive.LogicalQueryStage -org.apache.spark.sql.execution.columnar.InMemoryRelation org.apache.spark.sql.execution.command.AlterViewSchemaBindingCommand org.apache.spark.sql.execution.command.ClearCacheCommand$ org.apache.spark.sql.execution.command.CreateSQLFunctionCommand org.apache.spark.sql.execution.command.DescribeProcedureCommand org.apache.spark.sql.execution.command.DescribeQueryCommand org.apache.spark.sql.execution.command.DescribeRelationJsonCommand -org.apache.spark.sql.execution.command.DropTempViewCommand org.apache.spark.sql.execution.command.ExplainCommand org.apache.spark.sql.execution.command.ExternalCommandExecutor org.apache.spark.sql.execution.command.ListArchivesCommand org.apache.spark.sql.execution.command.ListFilesCommand org.apache.spark.sql.execution.command.ListJarsCommand -org.apache.spark.sql.execution.command.ResetCommand -org.apache.spark.sql.execution.command.SaveAsV1TableCommand org.apache.spark.sql.execution.command.SetCatalogCommand -org.apache.spark.sql.execution.command.SetCommand org.apache.spark.sql.execution.command.SetNamespaceCollationCommand org.apache.spark.sql.execution.command.ShowCatalogsCommand org.apache.spark.sql.execution.command.ShowCurrentNamespaceCommand org.apache.spark.sql.execution.command.ShowFunctionsCommand -org.apache.spark.sql.execution.command.ShowNamespacesCommand org.apache.spark.sql.execution.command.ShowProceduresCommand org.apache.spark.sql.execution.command.ShowTablesCommand org.apache.spark.sql.execution.command.ShowViewsCommand @@ -172,7 +184,6 @@ org.apache.spark.sql.execution.command.StreamingExplainCommand org.apache.spark.sql.execution.command.UnsetNamespacePropertiesCommand org.apache.spark.sql.execution.datasources.RefreshResource org.apache.spark.sql.execution.datasources.SparkExpressionConverter$DummyRelation -org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation org.apache.spark.sql.execution.datasources.v2.ScanBuilderHolder org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2ScanRelation diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala index 8ea96745971..1212aaacd6d 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ClassificationCoverageSuite.scala @@ -107,6 +107,8 @@ class ClassificationCoverageSuite extends AnyFunSuite { // row-filtered by ObjectFilterPlaceHolder + FilterDataSourceV2Strategy "org.apache.spark.sql.catalyst.plans.logical.ShowNamespaces", "org.apache.spark.sql.catalyst.plans.logical.ShowTables", + // Spark 4.x's v1 SHOW DATABASES/NAMESPACES; row-filtered like ShowNamespaces + "org.apache.spark.sql.execution.command.ShowNamespacesCommand", // session-local temp views are deliberately not authz resources "org.apache.spark.sql.execution.command.DropTempViewCommand", // resolves to literally nothing to execute diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala index c72076aaaf4..8942ad35775 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/KnownHarmlessNodes.scala @@ -30,14 +30,15 @@ import org.apache.kyuubi.plugin.spark.authz.serde.HarmlessNodeSpec */ object KnownHarmlessNodes { - // The Spark minors the plugin currently supports and tests per-profile; the baseline - // every entry was reviewed against when the allowlist was introduced. - private val spark3x = Seq("3.3", "3.4", "3.5") + // The Spark minors the plugin currently supports and tests per-profile. Every entry + // below was reviewed against the 3.x baseline when the allowlist was introduced and + // re-reviewed (and the exercised ones re-run in deny mode, full module suite per + // profile) for the 4.x port. A version joins an entry's list by being tested or + // reviewed, never by interpolation. + private val spark3xAnd4x = Seq("3.3", "3.4", "3.5", "4.0", "4.1", "4.2") - // Entries additionally exercised in deny mode under the spark-4.1 profile. 4.0 is - // deliberately absent until something verifies it: a version joins an entry's list by - // being tested or reviewed, never by interpolation. - private val spark3xAnd41 = spark3x :+ "4.1" + // Nodes that only exist on Spark 4. + private val spark4x = Seq("4.0", "4.1", "4.2") val specs: Seq[HarmlessNodeSpec] = Seq( HarmlessNodeSpec( @@ -45,93 +46,100 @@ object KnownHarmlessNodes { "This plugin's own row-filtering replacement for ShowColumnsCommand (installed by" + " RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a" + " dedicated dispatch arm and every result row is checked for SHOWCOLUMNS access", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowFunctionsCommand", "This plugin's own row-filtering replacement for ShowFunctionsCommand (installed by" + " RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a" + " dedicated dispatch arm and every result row is checked for SHOWFUNCTIONS access", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.FilteredShowTablesCommand", "This plugin's own row-filtering replacement for ShowTablesCommand (installed by" + " RuleReplaceShowObjectCommands); PrivilegesBuilder.build handles it with a" + " dedicated dispatch arm and every result row is checked for SHOWTABLES access", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.catalyst.analysis.ResolvedNamespace", "Analysis-time resolution artifact naming a namespace; reads no data itself, and the" + " commands resolved over it are classified in their own right", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.catalyst.plans.logical.CTERelationRef", "Leaf reference to a CTE definition; the definition's own plan appears under" + " WithCTE in the same tree and is authorized there", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.catalyst.plans.logical.CommandResult", "Holds rows already produced by an eagerly executed command; that command was" + " authorized when it executed", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.catalyst.plans.logical.LocalRelation", "Holds in-memory literal rows (VALUES lists, createDataFrame); reads no stored data", - spark3xAnd41), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.catalyst.plans.logical.NoopCommand", "Spark's placeholder for commands with nothing to do (e.g. IF EXISTS / IF NOT EXISTS" + " variants when the object is absent); executes nothing", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.catalyst.plans.logical.OneRowRelation", "The implicit single-row relation backing SELECT without FROM; reads no stored data", - spark3xAnd41), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.catalyst.plans.logical.Range", "Generates rows from a numeric range (e.g. spark.range); reads no stored data", - spark3xAnd41), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.catalyst.plans.logical.ShowNamespaces", "Enforced elsewhere: results are row-filtered per namespace by ObjectFilterPlaceHolder" + " + FilterDataSourceV2Strategy; Spark eagerly executes the bare command in a nested" + " QueryExecution whose unfiltered result the placeholder discards", - spark3x), + spark3xAnd4x), + HarmlessNodeSpec( + "org.apache.spark.sql.execution.command.ShowNamespacesCommand", + "Enforced elsewhere: Spark 4.x's v1 SHOW DATABASES/NAMESPACES command; results are" + + " row-filtered per namespace by ObjectFilterPlaceHolder + FilterDataSourceV2Strategy" + + " exactly like v2 ShowNamespaces, and the bare command Spark eagerly executes in a" + + " nested QueryExecution has its unfiltered result discarded by the placeholder", + spark4x), HarmlessNodeSpec( "org.apache.spark.sql.catalyst.plans.logical.ShowTables", "Enforced elsewhere: results are row-filtered per table by ObjectFilterPlaceHolder" + " + FilterDataSourceV2Strategy; Spark eagerly executes the bare command in a nested" + " QueryExecution whose unfiltered result the placeholder discards", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.execution.ExternalRDD", "Wraps a session-created RDD/local collection (e.g. spark.createDataset, and Delta's" + " internal VACUUM plumbing); RDD-level access is outside the plugin's scope and is" + " an existing, separate concern", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.execution.LogicalRDD", "Wraps a pre-existing RDD; RDD-level access is outside the plugin's scope and is an" + " existing, separate concern", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.execution.columnar.InMemoryRelation", "Cached query results; the originating plan was authorized when the cache was" + " populated", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.execution.command.DropTempViewCommand", "Operates only on session-local temporary views, which are deliberately not authz" + " resources (their reads are authorized against the underlying tables); see" + " KYUUBI #3426", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.execution.command.ResetCommand", "Resets session configuration only; sensitive configs are separately guarded by" + " AuthzConfigurationChecker", - spark3x), + spark3xAnd4x), HarmlessNodeSpec( "org.apache.spark.sql.execution.command.SetCommand", "Sets session configuration only; sensitive configs are separately guarded by" + " AuthzConfigurationChecker", - spark3x)) + spark3xAnd4x)) } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala index ff2666666d7..76e958f800e 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala @@ -441,6 +441,27 @@ object TableCommands extends CommandSpecs[TableCommandSpec] { queryDescs = Seq(queryQueryDesc)) } + // SPARK-49246 (Spark 4.x): DataFrameWriter.saveAsTable on a v1 source analyzes into this + // leaf wrapper, which only plans the real Create*TableAsSelect command inside a nested + // QueryExecution when it runs. Spec it directly so the write is authorized on the outer + // plan rather than relying on the nested pass. + val SaveAsV1Table = { + val cmd = "org.apache.spark.sql.execution.command.SaveAsV1TableCommand" + val tableDesc = + TableDesc( + "tableDesc", + classOf[CatalogTableTableExtractor], + setCurrentDatabaseIfMissing = true) + val uriDesc = UriDesc("tableDesc", classOf[CatalogTableURIExtractor]) + TableCommandSpec( + cmd, + Seq(tableDesc), + CREATETABLE_AS_SELECT, + queryDescs = Seq(queryQueryDesc), + uriDescs = Seq(uriDesc), + verifiedSparkVersions = Seq("4.1")) + } + val CreateHiveTableAsSelect = { val cmd = "org.apache.spark.sql.hive.execution.CreateHiveTableAsSelectCommand" val columnDesc = ColumnDesc("outputColumnNames", classOf[StringSeqColumnExtractor]) @@ -723,6 +744,7 @@ object TableCommands extends CommandSpecs[TableCommandSpec] { CreateDataSourceTable.copy(classname = "org.apache.spark.sql.execution.command.CreateTableCommand"), CreateDataSourceTableAsSelect, + SaveAsV1Table, CreateHiveTableAsSelect, CreateHiveTableAsSelect.copy(classname = "org.apache.spark.sql.hive.execution.OptimizedCreateHiveTableAsSelectCommand"), From 3d21c17a7441af860334149acae506cd009a31aa Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Thu, 16 Jul 2026 17:42:42 -0700 Subject: [PATCH 5/7] [KYUUBI #7593][AUTHZ] Add Spark 4.0 classification backlog Full module suite passes in deny mode under -Pspark-4.0 -Pscala-2.13 (678 tests, first run - no new unclassified nodes beyond what 4.1 already surfaced). --- .../classification_backlog_spark_4.0.txt | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.0.txt diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.0.txt b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.0.txt new file mode 100644 index 00000000000..ad935c935eb --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.0.txt @@ -0,0 +1,182 @@ +org.apache.paimon.spark.catalyst.analysis.ResolvedPaimonView +org.apache.paimon.spark.catalyst.plans.logical.CreateOrReplaceTagCommand +org.apache.paimon.spark.catalyst.plans.logical.CreatePaimonView +org.apache.paimon.spark.catalyst.plans.logical.DeleteTagCommand +org.apache.paimon.spark.catalyst.plans.logical.DropPaimonView +org.apache.paimon.spark.catalyst.plans.logical.FullTextSearchQuery +org.apache.paimon.spark.catalyst.plans.logical.IncrementalBetweenTimestamp +org.apache.paimon.spark.catalyst.plans.logical.IncrementalQuery +org.apache.paimon.spark.catalyst.plans.logical.IncrementalToAutoTag +org.apache.paimon.spark.catalyst.plans.logical.PaimonDropPartitions +org.apache.paimon.spark.catalyst.plans.logical.RenameTagCommand +org.apache.paimon.spark.catalyst.plans.logical.ResolvedIdentifier +org.apache.paimon.spark.catalyst.plans.logical.ShowPaimonViews +org.apache.paimon.spark.catalyst.plans.logical.ShowTagsCommand +org.apache.paimon.spark.catalyst.plans.logical.TruncatePaimonTableWithFilter +org.apache.paimon.spark.catalyst.plans.logical.VectorSearchQuery +org.apache.paimon.spark.commands.MergeIntoPaimonDataEvolutionTable +org.apache.paimon.spark.commands.PaimonAnalyzeTableColumnCommand +org.apache.paimon.spark.commands.PaimonDynamicPartitionOverwriteCommand +org.apache.paimon.spark.commands.PaimonShowColumnsCommand +org.apache.paimon.spark.commands.WriteIntoPaimonTable +org.apache.paimon.spark.execution.CreatePaimonV1FunctionCommand +org.apache.paimon.spark.execution.DescribePaimonV1FunctionCommand +org.apache.paimon.spark.execution.DropPaimonV1FunctionCommand +org.apache.spark.sql.catalyst.TimeTravel +org.apache.spark.sql.catalyst.analysis.CurrentNamespace$ +org.apache.spark.sql.catalyst.analysis.ExecuteImmediateQuery +org.apache.spark.sql.catalyst.analysis.RelationTimeTravel +org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier +org.apache.spark.sql.catalyst.analysis.ResolvedInlineTable +org.apache.spark.sql.catalyst.analysis.ResolvedNonPersistentFunc +org.apache.spark.sql.catalyst.analysis.ResolvedPersistentFunc +org.apache.spark.sql.catalyst.analysis.ResolvedPersistentView +org.apache.spark.sql.catalyst.analysis.ResolvedProcedure +org.apache.spark.sql.catalyst.analysis.ResolvedTable +org.apache.spark.sql.catalyst.analysis.ResolvedTempView +org.apache.spark.sql.catalyst.analysis.SQLTableFunction +org.apache.spark.sql.catalyst.analysis.UnresolvedFunctionName +org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier +org.apache.spark.sql.catalyst.analysis.UnresolvedInlineTable +org.apache.spark.sql.catalyst.analysis.UnresolvedNamespace +org.apache.spark.sql.catalyst.analysis.UnresolvedProcedure +org.apache.spark.sql.catalyst.analysis.UnresolvedTable +org.apache.spark.sql.catalyst.analysis.UnresolvedTableOrView +org.apache.spark.sql.catalyst.analysis.UnresolvedTableValuedFunction +org.apache.spark.sql.catalyst.analysis.UnresolvedView +org.apache.spark.sql.catalyst.catalog.TemporaryViewRelation +org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation +org.apache.spark.sql.catalyst.encoders.DummyExpressionHolder +org.apache.spark.sql.catalyst.plans.logical.AlterColumnSyncIdentity +org.apache.spark.sql.catalyst.plans.logical.AlterTableAddConstraint +org.apache.spark.sql.catalyst.plans.logical.AlterTableClusterBy +org.apache.spark.sql.catalyst.plans.logical.AlterTableCollation +org.apache.spark.sql.catalyst.plans.logical.AlterTableDropConstraint +org.apache.spark.sql.catalyst.plans.logical.AlterTableDropFeature +org.apache.spark.sql.catalyst.plans.logical.AlterViewAs +org.apache.spark.sql.catalyst.plans.logical.AlterViewSchemaBinding +org.apache.spark.sql.catalyst.plans.logical.AnalyzeColumn +org.apache.spark.sql.catalyst.plans.logical.AnalyzeTable +org.apache.spark.sql.catalyst.plans.logical.AnalyzeTables +org.apache.spark.sql.catalyst.plans.logical.CallCommand +org.apache.spark.sql.catalyst.plans.logical.CompactionPath +org.apache.spark.sql.catalyst.plans.logical.CompactionShowOnPath +org.apache.spark.sql.catalyst.plans.logical.CompactionShowOnTable +org.apache.spark.sql.catalyst.plans.logical.CompactionTable +org.apache.spark.sql.catalyst.plans.logical.CompoundBody +org.apache.spark.sql.catalyst.plans.logical.CreateFunction +org.apache.spark.sql.catalyst.plans.logical.CreateIndex +org.apache.spark.sql.catalyst.plans.logical.CreateUserDefinedFunction +org.apache.spark.sql.catalyst.plans.logical.CreateVariable +org.apache.spark.sql.catalyst.plans.logical.CreateView +org.apache.spark.sql.catalyst.plans.logical.DeleteFromTableWithFilters +org.apache.spark.sql.catalyst.plans.logical.DeltaMergeInto +org.apache.spark.sql.catalyst.plans.logical.DescribeColumn +org.apache.spark.sql.catalyst.plans.logical.DescribeFunction +org.apache.spark.sql.catalyst.plans.logical.DropFunction +org.apache.spark.sql.catalyst.plans.logical.DropIndex +org.apache.spark.sql.catalyst.plans.logical.DropVariable +org.apache.spark.sql.catalyst.plans.logical.DropView +org.apache.spark.sql.catalyst.plans.logical.EmptyRelation +org.apache.spark.sql.catalyst.plans.logical.HoodieFileSystemViewTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieFileSystemViewTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieMetadataTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieQuery +org.apache.spark.sql.catalyst.plans.logical.HoodieShowIndexes +org.apache.spark.sql.catalyst.plans.logical.HoodieTableChanges +org.apache.spark.sql.catalyst.plans.logical.HoodieTableChangesByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieTimelineTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieTimelineTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieVectorSearchBatchTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieVectorSearchTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.LoadData +org.apache.spark.sql.catalyst.plans.logical.RecoverPartitions +org.apache.spark.sql.catalyst.plans.logical.RefreshFunction +org.apache.spark.sql.catalyst.plans.logical.RefreshIndex +org.apache.spark.sql.catalyst.plans.logical.SetTableLocation +org.apache.spark.sql.catalyst.plans.logical.SetTableSerDeProperties +org.apache.spark.sql.catalyst.plans.logical.SetVariable +org.apache.spark.sql.catalyst.plans.logical.SetViewProperties +org.apache.spark.sql.catalyst.plans.logical.ShowColumns +org.apache.spark.sql.catalyst.plans.logical.ShowFunctions +org.apache.spark.sql.catalyst.plans.logical.ShowPartitions +org.apache.spark.sql.catalyst.plans.logical.ShowTablePartition +org.apache.spark.sql.catalyst.plans.logical.ShowTablesExtended +org.apache.spark.sql.catalyst.plans.logical.ShowViews +org.apache.spark.sql.catalyst.plans.logical.Transpose +org.apache.spark.sql.catalyst.plans.logical.UncacheTable +org.apache.spark.sql.catalyst.plans.logical.UnionLoopRef +org.apache.spark.sql.catalyst.plans.logical.UnresolvedDataSource +org.apache.spark.sql.catalyst.plans.logical.UnsetViewProperties +org.apache.spark.sql.catalyst.plans.logical.WriteDelta +org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView +org.apache.spark.sql.catalyst.plans.logical.views.DropIcebergView +org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View +org.apache.spark.sql.catalyst.plans.logical.views.ShowIcebergViews +org.apache.spark.sql.catalyst.streaming.StreamingRelationV2 +org.apache.spark.sql.delta.CDCNameBased +org.apache.spark.sql.delta.CDCPathBased +org.apache.spark.sql.delta.DeltaDynamicPartitionOverwriteCommand +org.apache.spark.sql.delta.ResolvedPathBasedNonDeltaTable +org.apache.spark.sql.delta.UnresolvedPathBasedDeltaTable +org.apache.spark.sql.delta.UnresolvedPathBasedDeltaTableRelation +org.apache.spark.sql.delta.UnresolvedPathBasedTable +org.apache.spark.sql.delta.commands.AlterTableAddColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableAddConstraintDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableChangeColumnDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableClusterByDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropConstraintDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropFeatureDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableReplaceColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableSetLocationDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableSetPropertiesDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableUnsetPropertiesDeltaCommand +org.apache.spark.sql.delta.commands.CloneTableCommand +org.apache.spark.sql.delta.commands.ConvertToDeltaCommand +org.apache.spark.sql.delta.commands.CreateDeltaTableCommand +org.apache.spark.sql.delta.commands.DeltaGenerateCommand +org.apache.spark.sql.delta.commands.DeltaInsertReplaceOnOrUsingCommand +org.apache.spark.sql.delta.commands.DeltaReorgTable +org.apache.spark.sql.delta.commands.DeltaReorgTableCommand +org.apache.spark.sql.delta.commands.DescribeDeltaDetailCommand +org.apache.spark.sql.delta.commands.DescribeDeltaHistoryCommand +org.apache.spark.sql.delta.commands.RestoreTableCommand +org.apache.spark.sql.delta.commands.ShowDeltaTableColumnsCommand +org.apache.spark.sql.delta.commands.TruncateDeltaTableCommand +org.apache.spark.sql.delta.commands.WriteIntoDelta +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillCommand +org.apache.spark.sql.delta.commands.backfill.RowTrackingUnBackfillCommand +org.apache.spark.sql.delta.constraints.ExpressionLogicalPlanWrapper +org.apache.spark.sql.delta.skipping.clustering.temp.AlterTableClusterBy +org.apache.spark.sql.delta.skipping.clustering.temp.ClusterByPlan +org.apache.spark.sql.execution.adaptive.LogicalQueryStage +org.apache.spark.sql.execution.command.AlterViewSchemaBindingCommand +org.apache.spark.sql.execution.command.ClearCacheCommand$ +org.apache.spark.sql.execution.command.CreateSQLFunctionCommand +org.apache.spark.sql.execution.command.DescribeQueryCommand +org.apache.spark.sql.execution.command.DescribeRelationJsonCommand +org.apache.spark.sql.execution.command.ExplainCommand +org.apache.spark.sql.execution.command.ExternalCommandExecutor +org.apache.spark.sql.execution.command.ListArchivesCommand +org.apache.spark.sql.execution.command.ListFilesCommand +org.apache.spark.sql.execution.command.ListJarsCommand +org.apache.spark.sql.execution.command.SetCatalogCommand +org.apache.spark.sql.execution.command.ShowCatalogsCommand +org.apache.spark.sql.execution.command.ShowCurrentNamespaceCommand +org.apache.spark.sql.execution.command.ShowFunctionsCommand +org.apache.spark.sql.execution.command.ShowTablesCommand +org.apache.spark.sql.execution.command.ShowViewsCommand +org.apache.spark.sql.execution.command.StreamingExplainCommand +org.apache.spark.sql.execution.command.UnsetNamespacePropertiesCommand +org.apache.spark.sql.execution.datasources.RefreshResource +org.apache.spark.sql.execution.datasources.SparkExpressionConverter$DummyRelation +org.apache.spark.sql.execution.datasources.v2.ScanBuilderHolder +org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation +org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2ScanRelation +org.apache.spark.sql.execution.streaming.OffsetHolder +org.apache.spark.sql.execution.streaming.StreamingExecutionRelation +org.apache.spark.sql.execution.streaming.StreamingRelation +org.apache.spark.sql.execution.streaming.sources.MemoryPlan +org.apache.spark.sql.hudi.command.AlterHoodieTableAddPartitionCommand +org.apache.spark.sql.hudi.command.ShowHoodieCreateTableCommand From c0ab410db28dfdcd677fee1fe74a9991aba9388e Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Thu, 16 Jul 2026 17:56:30 -0700 Subject: [PATCH 6/7] [KYUUBI #7593][AUTHZ] Verify paranoid mode on Spark profiles 3.5-4.2 - Add 4.2 classification backlog (208 entries; connector jars stay on the 4.2 classpath even though their suites are tag-excluded there) - Extend verifiedSparkVersions now that the per-profile deny runs prove them: SaveAsV1TableCommand 4.0/4.1/4.2, DataSourceV2ScanRelation 3.5/4.0/4.1 (Iceberg is tag-excluded on 4.2, so no 4.2 claim) - Refresh backlog counts in the design doc (3.5=135, 4.0=182, 4.1=195, 4.2=208) Full module suite in deny mode: 3.5 677+1, 4.0 678, 4.1 642, 4.2 428 (CI tag exclusions) - all green. --- .../kyuubi-spark-authz/docs/paranoid-mode.md | 5 +- .../src/main/resources/scan_command_spec.json | 2 +- .../main/resources/table_command_spec.json | 2 +- .../classification_backlog_spark_4.2.txt | 208 ++++++++++++++++++ .../kyuubi/plugin/spark/authz/gen/Scans.scala | 4 +- .../spark/authz/gen/TableCommands.scala | 2 +- 6 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.2.txt diff --git a/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md b/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md index dc8a5b333e6..1b9a9ab6893 100644 --- a/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md +++ b/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md @@ -314,8 +314,9 @@ time instead of in production: itself in the enumeration; - *dirty laundry* — relevant by shape but neither spec'd nor allowlisted, pinned one classname per line in `src/test/resources/classification_backlog_spark_.txt` - (currently 136 entries for 3.5, 181 for 4.1 — the 4.1 figure includes allowlist - entries awaiting 4.x re-review). + (currently 135 entries for 3.5, 182 for 4.0, 195 for 4.1, and 208 for 4.2 — the 4.x + figures grow with each profile's connector versions, since third-party nodes were + never classified to begin with). A companion check keeps the allowlist honest from the other side: an entry whose class is a pass-through shape fails the build, because nothing would ever consult it and it diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json index 177c5849733..dbcf3794cb1 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json @@ -57,7 +57,7 @@ } ], "functionDescs" : [ ], "uriDescs" : [ ], - "verifiedSparkVersions" : [ "3.5" ] + "verifiedSparkVersions" : [ "3.5", "4.0", "4.1" ] }, { "classname" : "org.apache.spark.sql.hive.HiveGenericUDF", "scanDescs" : [ ], diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json index 579dc932d95..ffd89b728ba 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json @@ -1585,7 +1585,7 @@ "isInput" : false, "comment" : "" } ], - "verifiedSparkVersions" : [ "4.1" ] + "verifiedSparkVersions" : [ "4.0", "4.1", "4.2" ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowColumnsCommand", "tableDescs" : [ { diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.2.txt b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.2.txt new file mode 100644 index 00000000000..f13a2c4adcb --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/classification_backlog_spark_4.2.txt @@ -0,0 +1,208 @@ +org.apache.paimon.spark.catalyst.analysis.ResolvedPaimonView +org.apache.paimon.spark.catalyst.plans.logical.CreateOrReplaceTagCommand +org.apache.paimon.spark.catalyst.plans.logical.CreatePaimonView +org.apache.paimon.spark.catalyst.plans.logical.DeleteTagCommand +org.apache.paimon.spark.catalyst.plans.logical.DropPaimonView +org.apache.paimon.spark.catalyst.plans.logical.FullTextSearchQuery +org.apache.paimon.spark.catalyst.plans.logical.IncrementalBetweenTimestamp +org.apache.paimon.spark.catalyst.plans.logical.IncrementalQuery +org.apache.paimon.spark.catalyst.plans.logical.IncrementalToAutoTag +org.apache.paimon.spark.catalyst.plans.logical.PaimonDropPartitions +org.apache.paimon.spark.catalyst.plans.logical.RenameTagCommand +org.apache.paimon.spark.catalyst.plans.logical.ResolvedIdentifier +org.apache.paimon.spark.catalyst.plans.logical.ShowPaimonViews +org.apache.paimon.spark.catalyst.plans.logical.ShowTagsCommand +org.apache.paimon.spark.catalyst.plans.logical.TruncatePaimonTableWithFilter +org.apache.paimon.spark.catalyst.plans.logical.VectorSearchQuery +org.apache.paimon.spark.commands.MergeIntoPaimonDataEvolutionTable +org.apache.paimon.spark.commands.PaimonAnalyzeTableColumnCommand +org.apache.paimon.spark.commands.PaimonDynamicPartitionOverwriteCommand +org.apache.paimon.spark.commands.PaimonShowColumnsCommand +org.apache.paimon.spark.commands.WriteIntoPaimonTable +org.apache.paimon.spark.execution.CreatePaimonV1FunctionCommand +org.apache.paimon.spark.execution.DescribePaimonV1FunctionCommand +org.apache.paimon.spark.execution.DropPaimonV1FunctionCommand +org.apache.spark.sql.catalyst.TimeTravel +org.apache.spark.sql.catalyst.analysis.CurrentNamespace$ +org.apache.spark.sql.catalyst.analysis.RelationChanges +org.apache.spark.sql.catalyst.analysis.RelationTimeTravel +org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier +org.apache.spark.sql.catalyst.analysis.ResolvedInlineTable +org.apache.spark.sql.catalyst.analysis.ResolvedNonPersistentFunc +org.apache.spark.sql.catalyst.analysis.ResolvedPersistentFunc +org.apache.spark.sql.catalyst.analysis.ResolvedPersistentView +org.apache.spark.sql.catalyst.analysis.ResolvedProcedure +org.apache.spark.sql.catalyst.analysis.ResolvedTable +org.apache.spark.sql.catalyst.analysis.ResolvedTempView +org.apache.spark.sql.catalyst.analysis.SQLTableFunction +org.apache.spark.sql.catalyst.analysis.UnresolvedExecuteImmediate +org.apache.spark.sql.catalyst.analysis.UnresolvedFunctionName +org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier +org.apache.spark.sql.catalyst.analysis.UnresolvedInlineTable +org.apache.spark.sql.catalyst.analysis.UnresolvedNamespace +org.apache.spark.sql.catalyst.analysis.UnresolvedProcedure +org.apache.spark.sql.catalyst.analysis.UnresolvedTable +org.apache.spark.sql.catalyst.analysis.UnresolvedTableOrView +org.apache.spark.sql.catalyst.analysis.UnresolvedTableValuedFunction +org.apache.spark.sql.catalyst.analysis.UnresolvedView +org.apache.spark.sql.catalyst.analysis.V2TableReference +org.apache.spark.sql.catalyst.analysis.resolver.UnresolvedCteRelationRef +org.apache.spark.sql.catalyst.catalog.TemporaryViewRelation +org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation +org.apache.spark.sql.catalyst.encoders.DummyExpressionHolder +org.apache.spark.sql.catalyst.optimizer.NonGroupingAggregateReference +org.apache.spark.sql.catalyst.plans.logical.AddCheckConstraint +org.apache.spark.sql.catalyst.plans.logical.AddConstraint +org.apache.spark.sql.catalyst.plans.logical.AlterColumnSyncIdentity +org.apache.spark.sql.catalyst.plans.logical.AlterTableAddConstraint +org.apache.spark.sql.catalyst.plans.logical.AlterTableClusterBy +org.apache.spark.sql.catalyst.plans.logical.AlterTableCollation +org.apache.spark.sql.catalyst.plans.logical.AlterTableDropConstraint +org.apache.spark.sql.catalyst.plans.logical.AlterTableDropFeature +org.apache.spark.sql.catalyst.plans.logical.AlterViewAs +org.apache.spark.sql.catalyst.plans.logical.AlterViewSchemaBinding +org.apache.spark.sql.catalyst.plans.logical.AnalyzeColumn +org.apache.spark.sql.catalyst.plans.logical.AnalyzeTable +org.apache.spark.sql.catalyst.plans.logical.AnalyzeTables +org.apache.spark.sql.catalyst.plans.logical.CallCommand +org.apache.spark.sql.catalyst.plans.logical.CloseCursor +org.apache.spark.sql.catalyst.plans.logical.CompactionPath +org.apache.spark.sql.catalyst.plans.logical.CompactionShowOnPath +org.apache.spark.sql.catalyst.plans.logical.CompactionShowOnTable +org.apache.spark.sql.catalyst.plans.logical.CompactionTable +org.apache.spark.sql.catalyst.plans.logical.CompoundBody +org.apache.spark.sql.catalyst.plans.logical.CreateFlowCommand +org.apache.spark.sql.catalyst.plans.logical.CreateFunction +org.apache.spark.sql.catalyst.plans.logical.CreateIndex +org.apache.spark.sql.catalyst.plans.logical.CreateMaterializedViewAsSelect +org.apache.spark.sql.catalyst.plans.logical.CreateStreamingTable +org.apache.spark.sql.catalyst.plans.logical.CreateStreamingTableAsSelect +org.apache.spark.sql.catalyst.plans.logical.CreateTableLike +org.apache.spark.sql.catalyst.plans.logical.CreateUserDefinedFunction +org.apache.spark.sql.catalyst.plans.logical.CreateVariable +org.apache.spark.sql.catalyst.plans.logical.CreateView +org.apache.spark.sql.catalyst.plans.logical.DeclareCursor +org.apache.spark.sql.catalyst.plans.logical.DeleteFromTableWithFilters +org.apache.spark.sql.catalyst.plans.logical.DeltaMergeInto +org.apache.spark.sql.catalyst.plans.logical.DescribeColumn +org.apache.spark.sql.catalyst.plans.logical.DescribeFunction +org.apache.spark.sql.catalyst.plans.logical.DescribeTablePartition +org.apache.spark.sql.catalyst.plans.logical.DropConstraint +org.apache.spark.sql.catalyst.plans.logical.DropFunction +org.apache.spark.sql.catalyst.plans.logical.DropIndex +org.apache.spark.sql.catalyst.plans.logical.DropVariable +org.apache.spark.sql.catalyst.plans.logical.DropView +org.apache.spark.sql.catalyst.plans.logical.EmptyRelation +org.apache.spark.sql.catalyst.plans.logical.FetchCursor +org.apache.spark.sql.catalyst.plans.logical.HoodieFileSystemViewTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieFileSystemViewTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieMetadataTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieQuery +org.apache.spark.sql.catalyst.plans.logical.HoodieShowIndexes +org.apache.spark.sql.catalyst.plans.logical.HoodieTableChanges +org.apache.spark.sql.catalyst.plans.logical.HoodieTableChangesByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieTimelineTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieTimelineTableValuedFunctionByPath +org.apache.spark.sql.catalyst.plans.logical.HoodieVectorSearchBatchTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.HoodieVectorSearchTableValuedFunction +org.apache.spark.sql.catalyst.plans.logical.InsertOnlyMerge +org.apache.spark.sql.catalyst.plans.logical.LoadData +org.apache.spark.sql.catalyst.plans.logical.OpenCursor +org.apache.spark.sql.catalyst.plans.logical.PythonWorkerLogs +org.apache.spark.sql.catalyst.plans.logical.RecoverPartitions +org.apache.spark.sql.catalyst.plans.logical.RefreshFunction +org.apache.spark.sql.catalyst.plans.logical.RefreshIndex +org.apache.spark.sql.catalyst.plans.logical.SetTableLocation +org.apache.spark.sql.catalyst.plans.logical.SetTableSerDeProperties +org.apache.spark.sql.catalyst.plans.logical.SetVariable +org.apache.spark.sql.catalyst.plans.logical.SetViewProperties +org.apache.spark.sql.catalyst.plans.logical.ShowColumns +org.apache.spark.sql.catalyst.plans.logical.ShowFunctions +org.apache.spark.sql.catalyst.plans.logical.ShowPartitions +org.apache.spark.sql.catalyst.plans.logical.ShowTablePartition +org.apache.spark.sql.catalyst.plans.logical.ShowTablesExtended +org.apache.spark.sql.catalyst.plans.logical.ShowViews +org.apache.spark.sql.catalyst.plans.logical.Transpose +org.apache.spark.sql.catalyst.plans.logical.UncacheTable +org.apache.spark.sql.catalyst.plans.logical.UnionLoopRef +org.apache.spark.sql.catalyst.plans.logical.UnresolvedDataSource +org.apache.spark.sql.catalyst.plans.logical.UnsetViewProperties +org.apache.spark.sql.catalyst.plans.logical.WriteDelta +org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView +org.apache.spark.sql.catalyst.plans.logical.views.DropIcebergView +org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View +org.apache.spark.sql.catalyst.plans.logical.views.ShowIcebergViews +org.apache.spark.sql.catalyst.streaming.StreamingRelationV2 +org.apache.spark.sql.delta.CDCNameBased +org.apache.spark.sql.delta.CDCPathBased +org.apache.spark.sql.delta.DeltaDynamicPartitionOverwriteCommand +org.apache.spark.sql.delta.ResolvedPathBasedNonDeltaTable +org.apache.spark.sql.delta.UnresolvedPathBasedDeltaTable +org.apache.spark.sql.delta.UnresolvedPathBasedDeltaTableRelation +org.apache.spark.sql.delta.UnresolvedPathBasedTable +org.apache.spark.sql.delta.commands.AlterTableAddColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableAddConstraintDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableChangeColumnDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableClusterByDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropConstraintDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableDropFeatureDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableReplaceColumnsDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableSetLocationDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableSetPropertiesDeltaCommand +org.apache.spark.sql.delta.commands.AlterTableUnsetPropertiesDeltaCommand +org.apache.spark.sql.delta.commands.CloneTableCommand +org.apache.spark.sql.delta.commands.ConvertToDeltaCommand +org.apache.spark.sql.delta.commands.CreateDeltaTableCommand +org.apache.spark.sql.delta.commands.DeltaGenerateCommand +org.apache.spark.sql.delta.commands.DeltaInsertReplaceOnOrUsingCommand +org.apache.spark.sql.delta.commands.DeltaReorgTable +org.apache.spark.sql.delta.commands.DeltaReorgTableCommand +org.apache.spark.sql.delta.commands.DescribeDeltaDetailCommand +org.apache.spark.sql.delta.commands.DescribeDeltaHistoryCommand +org.apache.spark.sql.delta.commands.RestoreTableCommand +org.apache.spark.sql.delta.commands.ShowDeltaTableColumnsCommand +org.apache.spark.sql.delta.commands.TruncateDeltaTableCommand +org.apache.spark.sql.delta.commands.WriteIntoDelta +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillCommand +org.apache.spark.sql.delta.commands.backfill.RowTrackingUnBackfillCommand +org.apache.spark.sql.delta.constraints.ExpressionLogicalPlanWrapper +org.apache.spark.sql.delta.skipping.clustering.temp.AlterTableClusterBy +org.apache.spark.sql.delta.skipping.clustering.temp.ClusterByPlan +org.apache.spark.sql.execution.adaptive.LogicalQueryStage +org.apache.spark.sql.execution.command.AlterViewSchemaBindingCommand +org.apache.spark.sql.execution.command.ClearCacheCommand$ +org.apache.spark.sql.execution.command.CreateMetricViewCommand +org.apache.spark.sql.execution.command.CreateSQLFunctionCommand +org.apache.spark.sql.execution.command.DescribeProcedureCommand +org.apache.spark.sql.execution.command.DescribeQueryCommand +org.apache.spark.sql.execution.command.DescribeRelationJsonCommand +org.apache.spark.sql.execution.command.ExplainCommand +org.apache.spark.sql.execution.command.ExternalCommandExecutor +org.apache.spark.sql.execution.command.ListArchivesCommand +org.apache.spark.sql.execution.command.ListFilesCommand +org.apache.spark.sql.execution.command.ListJarsCommand +org.apache.spark.sql.execution.command.SetCatalogCommand +org.apache.spark.sql.execution.command.SetNamespaceCollationCommand +org.apache.spark.sql.execution.command.SetPathCommand +org.apache.spark.sql.execution.command.ShowCatalogsCommand +org.apache.spark.sql.execution.command.ShowCollationsCommand +org.apache.spark.sql.execution.command.ShowCurrentNamespaceCommand +org.apache.spark.sql.execution.command.ShowFunctionsCommand +org.apache.spark.sql.execution.command.ShowProceduresCommand +org.apache.spark.sql.execution.command.ShowTablesCommand +org.apache.spark.sql.execution.command.ShowViewsCommand +org.apache.spark.sql.execution.command.StreamingExplainCommand +org.apache.spark.sql.execution.command.UnsetNamespacePropertiesCommand +org.apache.spark.sql.execution.datasources.RefreshResource +org.apache.spark.sql.execution.datasources.SparkExpressionConverter$DummyRelation +org.apache.spark.sql.execution.datasources.v2.ScanBuilderHolder +org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation +org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2ScanRelation +org.apache.spark.sql.execution.streaming.runtime.OffsetHolder +org.apache.spark.sql.execution.streaming.runtime.StreamingExecutionRelation +org.apache.spark.sql.execution.streaming.runtime.StreamingRelation +org.apache.spark.sql.execution.streaming.sources.MemoryPlan +org.apache.spark.sql.hudi.command.AlterHoodieTableAddPartitionCommand +org.apache.spark.sql.hudi.command.ShowHoodieCreateTableCommand +org.apache.spark.sql.metricview.logical.CreateMetricView diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala index c949a8f92f1..687bb5284b3 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala @@ -61,7 +61,9 @@ object Scans extends CommandSpecs[ScanSpec] { ScanDesc( "relation", classOf[DataSourceV2RelationTableExtractor]) - ScanSpec(r, Seq(tableDesc), verifiedSparkVersions = Seq("3.5")) + // Exercised by the Iceberg MERGE INTO tests, which run on the 3.5/4.0/4.1 profiles + // (Iceberg is tag-excluded on 4.2). + ScanSpec(r, Seq(tableDesc), verifiedSparkVersions = Seq("3.5", "4.0", "4.1")) } val PermanentViewMarker = { diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala index 76e958f800e..f8a76889f7e 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala @@ -459,7 +459,7 @@ object TableCommands extends CommandSpecs[TableCommandSpec] { CREATETABLE_AS_SELECT, queryDescs = Seq(queryQueryDesc), uriDescs = Seq(uriDesc), - verifiedSparkVersions = Seq("4.1")) + verifiedSparkVersions = Seq("4.0", "4.1", "4.2")) } val CreateHiveTableAsSelect = { From e2068b7fda536c150a2664e70a0b4dfeaaee342d Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Tue, 21 Jul 2026 12:50:48 -0700 Subject: [PATCH 7/7] [KYUUBI #7593][AUTHZ] Record spec Spark-version provenance in a ledger --- .../kyuubi-spark-authz/docs/paranoid-mode.md | 11 ++ .../spec_verified_spark_versions.txt | 182 ++++++++++++++++++ .../authz/gen/JsonSpecFileGenerator.scala | 68 +++++-- 3 files changed, 245 insertions(+), 16 deletions(-) create mode 100644 extensions/spark/kyuubi-spark-authz/src/test/resources/spec_verified_spark_versions.txt diff --git a/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md b/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md index 1b9a9ab6893..d470fad9cd5 100644 --- a/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md +++ b/extensions/spark/kyuubi-spark-authz/docs/paranoid-mode.md @@ -272,6 +272,17 @@ The field's force differs by spec kind, deliberately: build-time tooling a place to grow (e.g. flagging specs engaged far outside their audited range). +Where the values come from differs as well. A new spec declares `verifiedSparkVersions` at +its definition site, naming the minors it was actually reviewed against. The command and +scan specs that predate the Spark 4 port instead take theirs from a frozen ledger, +`src/test/resources/spec_verified_spark_versions.txt`, which records the `3.3, 3.4, 3.5` +baseline those entries inherited wholesale rather than earned per minor; the ledger's header +explains why that distinction is preserved rather than laundered into as many individual +per-minor claims. The ledger is deliberately closed rather than a default: a spec that +neither declares its own versions nor appears there fails generation, so a newly added spec +cannot quietly inherit the baseline, and an entry left behind by a deleted spec fails the +same check rather than lingering as provenance for nothing. + The build-time enumeration (§6) respects the gate: on each profile, only allowlist entries verified for that profile's Spark minor count as classified, so porting to a new Spark version surfaces every entry awaiting re-review in that profile's backlog at PR time. diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/spec_verified_spark_versions.txt b/extensions/spark/kyuubi-spark-authz/src/test/resources/spec_verified_spark_versions.txt new file mode 100644 index 00000000000..804eca82469 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/spec_verified_spark_versions.txt @@ -0,0 +1,182 @@ +# Audited Spark-version provenance for command and scan specs that do not declare +# `verifiedSparkVersions` at their definition site. +# +# Format: [ ...] -- `#` starts a comment. +# A classname listed with no versions is a deliberate "no minor audited" record. +# +# Absence from this ledger is an error, not a default. A spec that neither declares +# its own versions nor appears here fails the generator, so a newly added spec can +# never silently inherit the pre-Spark-4-port baseline below. +# +# TODO(KYUUBI #7593): this ledger freezes an inherited claim rather than an audited +# one. The 3.3/3.4/3.5 entries were never reviewed per minor -- they are what a +# blanket back-fill stamped on every spec that predated the Spark 4 port, back when +# "everything here is pre-Spark-4" made that a safe approximation. The merge of the +# Spark 4.0/4.1/4.2 authz support falsified that premise, and rather than launder a +# wholesale default into 160 individual per-minor claims, the baseline is recorded +# here as-is and quarantined. Retiring this file means auditing each spec against +# the minors it actually engages on and declaring the result at its definition site. +# Until then, treat a 3.3/3.4/3.5 entry as "inherited, unreviewed", not as evidence. +# +# Note the field is advisory for command and scan specs -- they still engage on +# unaudited versions -- unlike the allowlist, where it gates. + +io.delta.tables.execution.VacuumTableCommand 3.3 3.4 3.5 +org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker 3.3 3.4 3.5 +org.apache.paimon.spark.catalyst.plans.logical.PaimonCallCommand 3.3 3.4 3.5 +org.apache.paimon.spark.commands.DeleteFromPaimonTableCommand 3.3 3.4 3.5 +org.apache.paimon.spark.commands.MergeIntoPaimonTable 3.3 3.4 3.5 +org.apache.paimon.spark.commands.UpdatePaimonTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.catalog.HiveTableRelation 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.AddColumns 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.AddPartitionField 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.AddPartitions 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.AlterColumn 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.AlterTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.AppendData 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.CacheTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.CacheTableAsSelect 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.Call 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.CommentOnNamespace 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.CommentOnTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.CreateNamespace 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceBranch 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceTag 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.CreateTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.CreateTableAsSelect 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.CreateV2Table 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DeleteFromTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DescribeNamespace 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DescribeRelation 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DropBranch 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DropColumns 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DropNamespace 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DropPartitionField 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DropPartitions 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DropTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.DropTag 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.MergeIntoTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.OverwriteByExpression 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.OverwritePartitionsDynamic 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.RefreshTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.RenameColumn 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.RenamePartitions 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.RenameTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.RepairTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.ReplaceColumns 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.ReplaceData 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.ReplaceTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.ReplaceTableAsSelect 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.SetCatalogAndNamespace 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.SetNamespaceLocation 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.SetNamespaceProperties 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.SetTableProperties 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.ShowCreateTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.ShowTableProperties 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.TruncatePartition 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.TruncateTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.UnresolvedMergeIntoIcebergTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.UnsetTableProperties 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable 3.3 3.4 3.5 +org.apache.spark.sql.catalyst.plans.logical.UpdateTable 3.3 3.4 3.5 +org.apache.spark.sql.delta.commands.DeleteCommand 3.3 3.4 3.5 +org.apache.spark.sql.delta.commands.MergeIntoCommand 3.3 3.4 3.5 +org.apache.spark.sql.delta.commands.OptimizeTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.delta.commands.UpdateCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AddArchivesCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AddFilesCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AddJarCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AddJarsCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterDatabasePropertiesCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterDatabaseSetLocationCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterTableAddColumnsCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterTableAddPartitionCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterTableChangeColumnCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterTableDropPartitionCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterTableRenameCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterTableRenamePartitionCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterTableSerDePropertiesCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterTableSetLocationCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterTableSetPropertiesCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterTableUnsetPropertiesCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AlterViewAsCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AnalyzeColumnCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AnalyzePartitionCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AnalyzeTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.AnalyzeTablesCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.CacheTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.CreateDataSourceTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.CreateDatabaseCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.CreateFunctionCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.CreateTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.CreateTableLikeCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.CreateViewCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.DescribeColumnCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.DescribeDatabaseCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.DescribeFunctionCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.DescribeTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.DropDatabaseCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.DropFunctionCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.DropTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.InsertIntoDataSourceDirCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.LoadDataCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.RefreshFunctionCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.RefreshTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.RepairTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.SetDatabaseCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.SetNamespaceCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.ShowColumnsCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.ShowCreateTableAsSerdeCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.ShowCreateTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.ShowPartitionsCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.ShowTablePropertiesCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.command.TruncateTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.datasources.CreateTable 3.3 3.4 3.5 +org.apache.spark.sql.execution.datasources.CreateTempViewUsing 3.3 3.4 3.5 +org.apache.spark.sql.execution.datasources.InsertIntoDataSourceCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.datasources.LogicalRelation 3.3 3.4 3.5 +org.apache.spark.sql.execution.datasources.RefreshTable 3.3 3.4 3.5 +org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand 3.3 3.4 3.5 +org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation 3.3 3.4 3.5 +org.apache.spark.sql.hive.HiveGenericUDF 3.3 3.4 3.5 +org.apache.spark.sql.hive.HiveGenericUDTF 3.3 3.4 3.5 +org.apache.spark.sql.hive.HiveSimpleUDF 3.3 3.4 3.5 +org.apache.spark.sql.hive.HiveUDAFFunction 3.3 3.4 3.5 +org.apache.spark.sql.hive.execution.CreateHiveTableAsSelectCommand 3.3 3.4 3.5 +org.apache.spark.sql.hive.execution.InsertIntoHiveDirCommand 3.3 3.4 3.5 +org.apache.spark.sql.hive.execution.InsertIntoHiveTable 3.3 3.4 3.5 +org.apache.spark.sql.hive.execution.OptimizedCreateHiveTableAsSelectCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.AlterHoodieTableAddColumnsCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.AlterHoodieTableChangeColumnCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.AlterHoodieTableDropPartitionCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.AlterHoodieTableRenameCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.AlterTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.CallProcedureHoodieCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.CompactionHoodiePathCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.CompactionHoodieTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.CompactionShowHoodiePathCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.CompactionShowHoodieTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.CreateHoodieTableAsSelectCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.CreateHoodieTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.CreateHoodieTableLikeCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.CreateIndexCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.DeleteHoodieTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.DropHoodieTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.DropIndexCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.MergeIntoHoodieTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.RefreshIndexCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.RepairHoodieTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.ShowHoodieTablePartitionsCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.ShowIndexesCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.Spark31AlterTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.TruncateHoodieTableCommand 3.3 3.4 3.5 +org.apache.spark.sql.hudi.command.UpdateHoodieTableCommand 3.3 3.4 3.5 diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala index 1cff4a9709d..3f454ae90d5 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala @@ -20,6 +20,9 @@ package org.apache.kyuubi.plugin.spark.authz.gen import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths, StandardOpenOption} +import scala.collection.mutable +import scala.io.Source + import org.apache.kyuubi.KyuubiFunSuite import org.apache.kyuubi.plugin.spark.authz.serde._ import org.apache.kyuubi.util.AssertionUtils._ @@ -47,6 +50,7 @@ class JsonSpecFileGenerator extends KyuubiFunSuite { writeCommandSpecJson("function", Seq(FunctionCommands)) writeCommandSpecJson("scan", Seq(Scans)) writeHarmlessNodeSpecJson() + assertLedgerHasNoStaleEntries() } def writeHarmlessNodeSpecJson(): Unit = { @@ -79,28 +83,60 @@ class JsonSpecFileGenerator extends KyuubiFunSuite { } } - // Every entry currently in the spec files predates the Spark 4 port, so this is the - // audited baseline for any spec that doesn't declare its own verified versions. Specs - // verified on other Spark minors should set verifiedSparkVersions explicitly at their - // definition site. Note the field is advisory for command/scan specs (they still engage - // on unaudited versions), unlike the allowlist where it gates. - private val defaultVerifiedSparkVersions = Seq("3.3", "3.4", "3.5") + private val verifiedVersionsLedgerFile = "spec_verified_spark_versions.txt" + + // Spark-version provenance for specs that do not declare verifiedSparkVersions at their + // definition site. There is deliberately no default: a spec that is neither declared nor + // listed fails generation, so a new spec cannot silently inherit the pre-Spark-4-port + // baseline the way it could when this was a blanket back-fill. See the ledger's header. + private lazy val verifiedVersionsLedger: Map[String, Seq[String]] = { + val ledgerPath = Paths.get( + s"${getCurrentModuleHome(this)}/src/test/resources/$verifiedVersionsLedgerFile") + val source = Source.fromFile(ledgerPath.toFile, StandardCharsets.UTF_8.name) + val entries = + try source.getLines().map(_.takeWhile(_ != '#').trim).filter(_.nonEmpty).toList + finally source.close() + entries.map { entry => + val fields = entry.split("\\s+").toSeq + fields.head -> fields.tail + }.toMap + } + + private val ledgerEntriesUsed = mutable.Set.empty[String] - private def withDefaultVerifiedVersions[T <: CommandSpec](spec: T): T = { + private def withVerifiedVersions[T <: CommandSpec](spec: T): T = { + if (spec.verifiedSparkVersions.nonEmpty) { + return spec + } + ledgerEntriesUsed += spec.classname + val versions = verifiedVersionsLedger.getOrElse( + spec.classname, + fail( + s"${spec.classname} declares no verifiedSparkVersions and is absent from" + + s" $verifiedVersionsLedgerFile. Set verifiedSparkVersions at the spec's" + + " definition site to the exact Spark major.minor versions it was reviewed" + + " against, using Seq.empty if none. Do not add it to the ledger: that file" + + " records the pre-Spark-4-port baseline and is not meant to grow.")) val populated: CommandSpec = spec match { - case s: DatabaseCommandSpec if s.verifiedSparkVersions.isEmpty => - s.copy(verifiedSparkVersions = defaultVerifiedSparkVersions) - case s: TableCommandSpec if s.verifiedSparkVersions.isEmpty => - s.copy(verifiedSparkVersions = defaultVerifiedSparkVersions) - case s: FunctionCommandSpec if s.verifiedSparkVersions.isEmpty => - s.copy(verifiedSparkVersions = defaultVerifiedSparkVersions) - case s: ScanSpec if s.verifiedSparkVersions.isEmpty => - s.copy(verifiedSparkVersions = defaultVerifiedSparkVersions) + case s: DatabaseCommandSpec => s.copy(verifiedSparkVersions = versions) + case s: TableCommandSpec => s.copy(verifiedSparkVersions = versions) + case s: FunctionCommandSpec => s.copy(verifiedSparkVersions = versions) + case s: ScanSpec => s.copy(verifiedSparkVersions = versions) case s => s } populated.asInstanceOf[T] } + // A ledger entry for a spec that no longer exists is dead weight that reads as + // provenance, so retire it along with its spec. + private def assertLedgerHasNoStaleEntries(): Unit = { + val staleEntries = verifiedVersionsLedger.keySet -- ledgerEntriesUsed + withClue( + s"$verifiedVersionsLedgerFile has entries for specs that no longer take their" + + s" versions from it, remove them: $staleEntries")( + assertResult(Set.empty[String])(staleEntries)) + } + def writeCommandSpecJson[T <: CommandSpec]( commandType: String, specsArr: Seq[CommandSpecs[T]]): Unit = { @@ -109,7 +145,7 @@ class JsonSpecFileGenerator extends KyuubiFunSuite { s"${getCurrentModuleHome(this)}/src/main/resources/$filename") val allSpecs = specsArr.flatMap(_.specs.sortBy(_.classname)) - .map(withDefaultVerifiedVersions) + .map(withVerifiedVersions) val duplicatedClassnames = allSpecs.groupBy(_.classname).values .filter(_.size > 1).flatMap(specs => specs.map(_.classname)).toSet withClue(s"Unexpected duplicated classnames: $duplicatedClassnames")(