From 9225150069f9b5e3db108e31f31700e268ee99f0 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 17:41:06 +0200 Subject: [PATCH 01/33] build: switch to smooth-operator branch and vendor the java-properties writer Patches operator-rs to the smooth-operator branch (matching trino/hdfs) as the foundation for the v2 config_overrides adoption later in this series, and vendors the Java-properties writer into config/writer (backed by the java-properties crate, Apache-2.0) so ConfigMap rendering no longer goes through product_config::writer. Repointed resource/configmap.rs. No behaviour change (18 tests pass). Regenerated Cargo.nix/crate-hashes.json. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 31 +++-- Cargo.nix | 116 +++++++++++++++--- Cargo.toml | 2 + crate-hashes.json | 18 +-- rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/config/mod.rs | 1 + rust/operator-binary/src/config/writer.rs | 78 ++++++++++++ .../operator-binary/src/resource/configmap.rs | 7 +- 8 files changed, 214 insertions(+), 40 deletions(-) create mode 100644 rust/operator-binary/src/config/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 5357bf1f..2f759557 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1473,6 +1473,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" dependencies = [ "jsonptr", + "schemars", "serde", "serde_json", "thiserror 1.0.69", @@ -1517,7 +1518,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "darling", "regex", @@ -2889,7 +2890,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "const-oid", "ecdsa", @@ -2920,6 +2921,7 @@ dependencies = [ "const_format", "futures", "indoc", + "java-properties", "product-config", "rstest", "serde", @@ -2935,7 +2937,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "base64", "clap", @@ -2971,12 +2973,13 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "url", + "uuid", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "darling", "proc-macro2", @@ -2987,7 +2990,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "jiff", "k8s-openapi", @@ -3004,7 +3007,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "axum", "clap", @@ -3028,7 +3031,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "kube", "schemars", @@ -3042,7 +3045,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "convert_case", "convert_case_extras", @@ -3060,7 +3063,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "arc-swap", "async-trait", @@ -3641,6 +3644,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" diff --git a/Cargo.nix b/Cargo.nix index ff2e1a37..0cb45c8f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4690,6 +4690,11 @@ rec { name = "jsonptr"; packageId = "jsonptr"; } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } { name = "serde"; packageId = "serde"; @@ -4705,6 +4710,10 @@ rec { } ]; devDependencies = [ + { + name = "schemars"; + packageId = "schemars"; + } { name = "serde_json"; packageId = "serde_json"; @@ -4716,7 +4725,7 @@ rec { "schemars" = [ "dep:schemars" ]; "utoipa" = [ "dep:utoipa" ]; }; - resolvedDefaultFeatures = [ "default" "diff" ]; + resolvedDefaultFeatures = [ "default" "diff" "schemars" ]; }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; @@ -4842,8 +4851,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "k8s_version"; @@ -9516,8 +9525,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_certs"; @@ -9649,6 +9658,10 @@ rec { name = "indoc"; packageId = "indoc"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "product-config"; packageId = "product-config"; @@ -9711,8 +9724,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_operator"; @@ -9770,6 +9783,7 @@ rec { { name = "json-patch"; packageId = "json-patch"; + features = [ "schemars" ]; } { name = "k8s-openapi"; @@ -9873,6 +9887,10 @@ rec { packageId = "url"; features = [ "serde" ]; } + { + name = "uuid"; + packageId = "uuid"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; @@ -9891,8 +9909,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; procMacro = true; @@ -9926,8 +9944,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_shared"; @@ -10007,8 +10025,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_telemetry"; @@ -10117,8 +10135,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_versioned"; @@ -10167,8 +10185,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; procMacro = true; @@ -10235,8 +10253,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_webhook"; @@ -12260,6 +12278,66 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "uuid" = rec { + crateName = "uuid"; + version = "1.23.2"; + edition = "2021"; + sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + authors = [ + "Ashley Mannix" + "Dylan DPC" + "Hunar Roop Kahlon" + ]; + dependencies = [ + { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + devDependencies = [ + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "atomic" = [ "dep:atomic" ]; + "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "default" = [ "std" ]; + "fast-rng" = [ "rng" "dep:rand" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "md5" = [ "dep:md-5" ]; + "rng" = [ "dep:getrandom" ]; + "rng-getrandom" = [ "rng" "dep:getrandom" "uuid-rng-internal-lib" "uuid-rng-internal-lib/getrandom" ]; + "rng-rand" = [ "rng" "dep:rand" "uuid-rng-internal-lib" "uuid-rng-internal-lib/rand" ]; + "serde" = [ "dep:serde_core" ]; + "sha1" = [ "dep:sha1_smol" ]; + "slog" = [ "dep:slog" ]; + "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; + "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; + "v1" = [ "atomic" ]; + "v3" = [ "md5" ]; + "v4" = [ "rng" ]; + "v5" = [ "sha1" ]; + "v6" = [ "atomic" ]; + "v7" = [ "rng" ]; + "zerocopy" = [ "dep:zerocopy" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "valuable" = rec { crateName = "valuable"; version = "0.1.1"; diff --git a/Cargo.toml b/Cargo.toml index 8620a9ef..2ce4ff81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ clap = "4.5" const_format = "0.2" futures = "0.3" indoc = "2.0" +java-properties = "2.0" rstest = "0.26" semver = "1.0" serde = { version = "1.0", features = ["derive"] } @@ -30,5 +31,6 @@ tokio = { version = "1.40", features = ["full"] } tracing = "0.1" [patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } diff --git a/crate-hashes.json b/crate-hashes.json index 86f2b840..5564a89e 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#k8s-version@0.1.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-certs@0.4.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-operator-derive@0.3.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-operator@0.111.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-shared@0.1.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-telemetry@0.6.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-versioned-macros@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-versioned@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-webhook@0.9.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index f2903572..23a9234b 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -13,6 +13,7 @@ product-config.workspace = true stackable-operator.workspace = true indoc.workspace = true +java-properties.workspace = true anyhow.workspace = true clap.workspace = true const_format.workspace = true diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index ae92b3c2..162c9c09 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -1,3 +1,4 @@ pub mod command; pub mod jvm; pub mod node_id_hasher; +pub mod writer; diff --git a/rust/operator-binary/src/config/writer.rs b/rust/operator-binary/src/config/writer.rs new file mode 100644 index 00000000..a74babf0 --- /dev/null +++ b/rust/operator-binary/src/config/writer.rs @@ -0,0 +1,78 @@ +//! Writer for Java `.properties` files. +//! +//! Vendored from the `product-config` crate's `writer` module so the operator no +//! longer depends on `product-config` for rendering. + +use std::io::Write; + +use java_properties::{PropertiesError, PropertiesWriter}; +use snafu::{ResultExt, Snafu}; + +#[derive(Debug, Snafu)] +pub enum PropertiesWriterError { + #[snafu(display("failed to create properties file"))] + Properties { source: PropertiesError }, + + #[snafu(display("failed to convert properties file byte array to UTF-8"))] + FromUtf8 { source: std::string::FromUtf8Error }, +} + +/// Creates a common Java properties file string in the format: +/// `property_1=value_1\nproperty_2=value_2\n`. +pub fn to_java_properties_string<'a, T>(properties: T) -> Result +where + T: Iterator)>, +{ + let mut output = Vec::new(); + write_java_properties(&mut output, properties)?; + String::from_utf8(output).context(FromUtf8Snafu) +} + +/// Writes Java properties to the given writer. A `None` value is written as an +/// empty value (`key=`). +fn write_java_properties<'a, W, T>(writer: W, properties: T) -> Result<(), PropertiesWriterError> +where + W: Write, + T: Iterator)>, +{ + let mut writer = PropertiesWriter::new(writer); + for (k, v) in properties { + let property_value = v.as_deref().unwrap_or_default(); + writer.write(k, property_value).context(PropertiesSnafu)?; + } + writer.flush().context(PropertiesSnafu)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + + fn props(pairs: &[(&str, Option<&str>)]) -> String { + let map: BTreeMap> = pairs + .iter() + .map(|(k, v)| (k.to_string(), v.map(str::to_string))) + .collect(); + to_java_properties_string(map.iter()).unwrap() + } + + #[test] + fn java_properties_renders_key_value() { + assert_eq!(props(&[("a", Some("1")), ("b", Some("2"))]), "a=1\nb=2\n"); + } + + #[test] + fn java_properties_renders_none_as_empty() { + assert_eq!(props(&[("none", None)]), "none=\n"); + } + + #[test] + fn java_properties_escapes_colon_in_value() { + assert_eq!( + props(&[("url", Some("file://this/location/file.abc"))]), + "url=file\\://this/location/file.abc\n" + ); + } +} diff --git a/rust/operator-binary/src/resource/configmap.rs b/rust/operator-binary/src/resource/configmap.rs index 473120b3..7359a0d2 100644 --- a/rust/operator-binary/src/resource/configmap.rs +++ b/rust/operator-binary/src/resource/configmap.rs @@ -4,7 +4,7 @@ use std::{ }; use indoc::formatdoc; -use product_config::{types::PropertyNameKind, writer::to_java_properties_string}; +use product_config::types::PropertyNameKind; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, @@ -14,6 +14,7 @@ use stackable_operator::{ }; use crate::{ + config::writer::to_java_properties_string, controller::KAFKA_CONTROLLER_NAME, crd::{ JVM_SECURITY_PROPERTIES_FILE, KafkaPodDescriptor, MetadataManager, @@ -48,7 +49,7 @@ pub enum Error { rolegroup ))] JvmSecurityProperties { - source: product_config::writer::PropertiesWriterError, + source: crate::config::writer::PropertiesWriterError, rolegroup: String, }, @@ -64,7 +65,7 @@ pub enum Error { #[snafu(display("failed to serialize config for {rolegroup}"))] SerializeConfig { - source: product_config::writer::PropertiesWriterError, + source: crate::config::writer::PropertiesWriterError, rolegroup: RoleGroupRef, }, From 75a05194f194c8061cbf4ebfdb4a00416e60b99a Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 18:04:31 +0200 Subject: [PATCH 02/33] refactor: extract per-file kafka .properties builders Splits server_properties_file into controller/build/properties/{broker,controller}_properties builders (base map + security settings + graceful-shutdown + user overrides), wired into resource/configmap.rs by role. The property assembly was moved verbatim, so the rendered broker.properties/controller.properties are unchanged (18 tests pass; byte parity to be confirmed via the kuttl ConfigMap snapshot). Override input stays BTreeMap; no product-config removed yet (later increment). Co-Authored-By: Claude Opus 4.8 --- rust/operator-binary/src/controller.rs | 1 + .../src/controller/build/mod.rs | 3 + .../build/properties/broker_properties.rs | 119 ++++++++ .../build/properties/controller_properties.rs | 76 ++++++ .../src/controller/build/properties/mod.rs | 35 +++ .../operator-binary/src/resource/configmap.rs | 255 +++--------------- 6 files changed, 273 insertions(+), 216 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/mod.rs create mode 100644 rust/operator-binary/src/controller/build/properties/broker_properties.rs create mode 100644 rust/operator-binary/src/controller/build/properties/controller_properties.rs create mode 100644 rust/operator-binary/src/controller/build/properties/mod.rs diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index a064c1ac..bebda106 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -26,6 +26,7 @@ use stackable_operator::{ }; use strum::{EnumDiscriminants, IntoStaticStr}; +pub(crate) mod build; mod dereference; mod validate; diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs new file mode 100644 index 00000000..2f51c3f6 --- /dev/null +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -0,0 +1,3 @@ +//! Builders that assemble Kubernetes resources for kafka rolegroups. + +pub mod properties; diff --git a/rust/operator-binary/src/controller/build/properties/broker_properties.rs b/rust/operator-binary/src/controller/build/properties/broker_properties.rs new file mode 100644 index 00000000..01bd0d1f --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/broker_properties.rs @@ -0,0 +1,119 @@ +use std::collections::BTreeMap; + +use snafu::OptionExt; + +use crate::{ + crd::{ + KafkaPodDescriptor, + listener::{KafkaListenerConfig, KafkaListenerName}, + role::{ + KAFKA_ADVERTISED_LISTENERS, KAFKA_BROKER_ID, + KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, + KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, KAFKA_PROCESS_ROLES, KafkaRole, + }, + security::KafkaTlsSecurity, + }, + operations::graceful_shutdown::graceful_shutdown_config_properties, +}; + +use super::{Error, NoKraftControllersFoundSnafu, kraft_controllers}; + +pub fn build( + kafka_security: &KafkaTlsSecurity, + listener_config: &KafkaListenerConfig, + pod_descriptors: &[KafkaPodDescriptor], + opa_connect_string: Option<&str>, + kraft_mode: bool, + disable_broker_id_generation: bool, + overrides: BTreeMap, +) -> Result, Error> { + let kraft_controllers = kraft_controllers(pod_descriptors); + + let mut result = BTreeMap::from([ + ( + KAFKA_LOG_DIRS.to_string(), + "/stackable/data/topicdata".to_string(), + ), + (KAFKA_LISTENERS.to_string(), listener_config.listeners()), + ( + KAFKA_ADVERTISED_LISTENERS.to_string(), + listener_config.advertised_listeners(), + ), + ( + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), + listener_config.listener_security_protocol_map(), + ), + ( + "inter.broker.listener.name".to_string(), + KafkaListenerName::Internal.to_string(), + ), + ]); + + if kraft_mode { + let kraft_controllers = kraft_controllers.context(NoKraftControllersFoundSnafu)?; + + // Running in KRaft mode + result.extend([ + ( + "broker.id.generation.enable".to_string(), + "false".to_string(), + ), + (KAFKA_NODE_ID.to_string(), "${env:REPLICA_ID}".to_string()), + ( + KAFKA_PROCESS_ROLES.to_string(), + KafkaRole::Broker.to_string(), + ), + ( + "controller.listener.names".to_string(), + KafkaListenerName::Controller.to_string(), + ), + ( + KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), + kraft_controllers.clone(), + ), + ]); + } else { + // Running with ZooKeeper enabled + result.extend([( + "zookeeper.connect".to_string(), + "${env:ZOOKEEPER}".to_string(), + )]); + // We are in zookeeper mode and the user has defined a broker id mapping + // so we disable automatic id generation. + // This check ensures that existing clusters running in ZooKeeper mode do not + // suddenly break after the introduction of this change. + if disable_broker_id_generation { + result.extend([ + ( + "broker.id.generation.enable".to_string(), + "false".to_string(), + ), + (KAFKA_BROKER_ID.to_string(), "${env:REPLICA_ID}".to_string()), + ]); + } + } + + // Enable OPA authorization + if opa_connect_string.is_some() { + result.extend([ + ( + "authorizer.class.name".to_string(), + "org.openpolicyagent.kafka.OpaAuthorizer".to_string(), + ), + ( + "opa.authorizer.metrics.enabled".to_string(), + "true".to_string(), + ), + ( + "opa.authorizer.url".to_string(), + opa_connect_string.unwrap_or_default().to_string(), + ), + ]); + } + + result.extend(kafka_security.broker_config_settings()); + result.extend(graceful_shutdown_config_properties()); + result.extend(overrides); + + Ok(result) +} diff --git a/rust/operator-binary/src/controller/build/properties/controller_properties.rs b/rust/operator-binary/src/controller/build/properties/controller_properties.rs new file mode 100644 index 00000000..f9862dd3 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/controller_properties.rs @@ -0,0 +1,76 @@ +use std::collections::BTreeMap; + +use snafu::OptionExt; + +use crate::{ + crd::{ + KafkaPodDescriptor, + listener::{KafkaListenerConfig, KafkaListenerName}, + role::{ + KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, + KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, KAFKA_PROCESS_ROLES, KafkaRole, + }, + security::KafkaTlsSecurity, + }, + operations::graceful_shutdown::graceful_shutdown_config_properties, +}; + +use super::{Error, NoKraftControllersFoundSnafu, kraft_controllers}; + +pub fn build( + kafka_security: &KafkaTlsSecurity, + listener_config: &KafkaListenerConfig, + pod_descriptors: &[KafkaPodDescriptor], + kraft_mode: bool, + overrides: BTreeMap, +) -> Result, Error> { + let kraft_controllers = kraft_controllers(pod_descriptors).context(NoKraftControllersFoundSnafu)?; + + let mut result = BTreeMap::from([ + ( + KAFKA_LOG_DIRS.to_string(), + "/stackable/data/kraft".to_string(), + ), + (KAFKA_PROCESS_ROLES.to_string(), KafkaRole::Controller.to_string()), + ( + "controller.listener.names".to_string(), + KafkaListenerName::Controller.to_string(), + ), + ( + KAFKA_NODE_ID.to_string(), + "${env:REPLICA_ID}".to_string(), + ), + ( + KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), + kraft_controllers.clone(), + ), + ( + KAFKA_LISTENERS.to_string(), + "CONTROLLER://${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}:${env:KAFKA_CLIENT_PORT}".to_string(), + ), + ( + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), + listener_config + .listener_security_protocol_map_for_controller()), + ]); + + result.insert( + "inter.broker.listener.name".to_string(), + KafkaListenerName::Internal.to_string(), + ); + + // The ZooKeeper connection is needed for migration from ZooKeeper to KRaft mode. + // It is not needed once the controller is fully running in KRaft mode. + if !kraft_mode { + result.insert( + "zookeeper.connect".to_string(), + "${env:ZOOKEEPER}".to_string(), + ); + } + + result.extend(kafka_security.controller_config_settings()); + result.extend(graceful_shutdown_config_properties()); + result.extend(overrides); + + Ok(result) +} diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs new file mode 100644 index 00000000..adcdcaae --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -0,0 +1,35 @@ +//! Property-file builders for Kafka rolegroup ConfigMaps. + +pub mod broker_properties; +pub mod controller_properties; + +use snafu::Snafu; + +use crate::crd::{KafkaPodDescriptor, role::KafkaRole}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("no Kraft controllers found to build"))] + NoKraftControllersFound, +} + +pub(crate) fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Option { + let result = pod_descriptors + .iter() + .filter(|pd| pd.role == KafkaRole::Controller.to_string()) + .map(|desc| { + format!( + "{fqdn}:{client_port}", + fqdn = desc.fqdn(), + client_port = desc.client_port + ) + }) + .collect::>() + .join(","); + + if result.is_empty() { + None + } else { + Some(result) + } +} diff --git a/rust/operator-binary/src/resource/configmap.rs b/rust/operator-binary/src/resource/configmap.rs index 7359a0d2..3f400e6c 100644 --- a/rust/operator-binary/src/resource/configmap.rs +++ b/rust/operator-binary/src/resource/configmap.rs @@ -1,11 +1,8 @@ -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, -}; +use std::collections::{BTreeMap, HashMap}; use indoc::formatdoc; use product_config::types::PropertyNameKind; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, commons::product_image_selection::ResolvedProductImage, @@ -19,16 +16,11 @@ use crate::{ crd::{ JVM_SECURITY_PROPERTIES_FILE, KafkaPodDescriptor, MetadataManager, STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, - listener::{KafkaListenerConfig, KafkaListenerName, node_address_cmd}, - role::{ - AnyConfig, KAFKA_ADVERTISED_LISTENERS, KAFKA_BROKER_ID, - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, - KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, KAFKA_PROCESS_ROLES, KafkaRole, - }, + listener::{KafkaListenerConfig, node_address_cmd}, + role::AnyConfig, security::KafkaTlsSecurity, v1alpha1, }, - operations::graceful_shutdown::graceful_shutdown_config_properties, product_logging::extend_role_group_config_map, utils::build_recommended_labels, }; @@ -69,13 +61,10 @@ pub enum Error { rolegroup: RoleGroupRef, }, - #[snafu(display("no Kraft controllers found to build"))] - NoKraftControllersFound, - - #[snafu(display("unknown Kafka role [{name}]"))] - UnknownKafkaRole { - source: strum::ParseError, - name: String, + #[snafu(display("failed to build properties for {rolegroup}"))] + BuildProperties { + source: crate::controller::build::properties::Error, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to build jaas configuration file for {rolegroup}"))] @@ -101,35 +90,40 @@ pub fn build_rolegroup_config_map( .effective_metadata_manager() .context(InvalidMetadataManagerSnafu)?; - let mut kafka_config = server_properties_file( - metadata_manager == MetadataManager::KRaft, - &rolegroup.role, - pod_descriptors, - listener_config, - opa_connect_string, - kafka - .spec - .cluster_config - .broker_id_pod_config_map_name - .is_some(), - )?; - - match merged_config { - AnyConfig::Broker(_) => kafka_config.extend(kafka_security.broker_config_settings()), + let overrides = rolegroup_config + .get(&PropertyNameKind::File(kafka_config_file_name.to_string())) + .cloned() + .unwrap_or_default(); + + let kafka_config = match merged_config { + AnyConfig::Broker(_) => { + crate::controller::build::properties::broker_properties::build( + kafka_security, + listener_config, + pod_descriptors, + opa_connect_string, + metadata_manager == MetadataManager::KRaft, + kafka + .spec + .cluster_config + .broker_id_pod_config_map_name + .is_some(), + overrides, + ) + } AnyConfig::Controller(_) => { - kafka_config.extend(kafka_security.controller_config_settings()) + crate::controller::build::properties::controller_properties::build( + kafka_security, + listener_config, + pod_descriptors, + metadata_manager == MetadataManager::KRaft, + overrides, + ) } } - - kafka_config.extend(graceful_shutdown_config_properties()); - - // Need to call this to get configOverrides :( - kafka_config.extend( - rolegroup_config - .get(&PropertyNameKind::File(kafka_config_file_name.to_string())) - .cloned() - .unwrap_or_default(), - ); + .with_context(|_| BuildPropertiesSnafu { + rolegroup: rolegroup.clone(), + })?; let kafka_config = kafka_config .into_iter() @@ -218,177 +212,6 @@ pub fn build_rolegroup_config_map( }) } -// Generate the content of both broker.properties and controller.properties files. -fn server_properties_file( - kraft_mode: bool, - role: &str, - pod_descriptors: &[KafkaPodDescriptor], - listener_config: &KafkaListenerConfig, - opa_connect_string: Option<&str>, - disable_broker_id_generation: bool, -) -> Result, Error> { - let kraft_controllers = kraft_controllers(pod_descriptors); - - let role = KafkaRole::from_str(role).context(UnknownKafkaRoleSnafu { - name: role.to_string(), - })?; - - match role { - KafkaRole::Controller => { - let kraft_controllers = kraft_controllers.context(NoKraftControllersFoundSnafu)?; - - let mut result = BTreeMap::from([ - ( - KAFKA_LOG_DIRS.to_string(), - "/stackable/data/kraft".to_string(), - ), - (KAFKA_PROCESS_ROLES.to_string(), role.to_string()), - ( - "controller.listener.names".to_string(), - KafkaListenerName::Controller.to_string(), - ), - ( - KAFKA_NODE_ID.to_string(), - "${env:REPLICA_ID}".to_string(), - ), - ( - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), - kraft_controllers.clone(), - ), - ( - KAFKA_LISTENERS.to_string(), - "CONTROLLER://${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}:${env:KAFKA_CLIENT_PORT}".to_string(), - ), - ( - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), - listener_config - .listener_security_protocol_map_for_controller()), - ]); - - result.insert( - "inter.broker.listener.name".to_string(), - KafkaListenerName::Internal.to_string(), - ); - - // The ZooKeeper connection is needed for migration from ZooKeeper to KRaft mode. - // It is not needed once the controller is fully running in KRaft mode. - if !kraft_mode { - result.insert( - "zookeeper.connect".to_string(), - "${env:ZOOKEEPER}".to_string(), - ); - } - Ok(result) - } - KafkaRole::Broker => { - let mut result = BTreeMap::from([ - ( - KAFKA_LOG_DIRS.to_string(), - "/stackable/data/topicdata".to_string(), - ), - (KAFKA_LISTENERS.to_string(), listener_config.listeners()), - ( - KAFKA_ADVERTISED_LISTENERS.to_string(), - listener_config.advertised_listeners(), - ), - ( - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), - listener_config.listener_security_protocol_map(), - ), - ( - "inter.broker.listener.name".to_string(), - KafkaListenerName::Internal.to_string(), - ), - ]); - - if kraft_mode { - let kraft_controllers = kraft_controllers.context(NoKraftControllersFoundSnafu)?; - - // Running in KRaft mode - result.extend([ - ( - "broker.id.generation.enable".to_string(), - "false".to_string(), - ), - (KAFKA_NODE_ID.to_string(), "${env:REPLICA_ID}".to_string()), - ( - KAFKA_PROCESS_ROLES.to_string(), - KafkaRole::Broker.to_string(), - ), - ( - "controller.listener.names".to_string(), - KafkaListenerName::Controller.to_string(), - ), - ( - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), - kraft_controllers.clone(), - ), - ]); - } else { - // Running with ZooKeeper enabled - result.extend([( - "zookeeper.connect".to_string(), - "${env:ZOOKEEPER}".to_string(), - )]); - // We are in zookeeper mode and the user has defined a broker id mapping - // so we disable automatic id generation. - // This check ensures that existing clusters running in ZooKeeper mode do not - // suddenly break after the introduction of this change. - if disable_broker_id_generation { - result.extend([ - ( - "broker.id.generation.enable".to_string(), - "false".to_string(), - ), - (KAFKA_BROKER_ID.to_string(), "${env:REPLICA_ID}".to_string()), - ]); - } - } - - // Enable OPA authorization - if opa_connect_string.is_some() { - result.extend([ - ( - "authorizer.class.name".to_string(), - "org.openpolicyagent.kafka.OpaAuthorizer".to_string(), - ), - ( - "opa.authorizer.metrics.enabled".to_string(), - "true".to_string(), - ), - ( - "opa.authorizer.url".to_string(), - opa_connect_string.unwrap_or_default().to_string(), - ), - ]); - } - - Ok(result) - } - } -} - -fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Option { - let result = pod_descriptors - .iter() - .filter(|pd| pd.role == KafkaRole::Controller.to_string()) - .map(|desc| { - format!( - "{fqdn}:{client_port}", - fqdn = desc.fqdn(), - client_port = desc.client_port - ) - }) - .collect::>() - .join(","); - - if result.is_empty() { - None - } else { - Some(result) - } -} - // Generate JAAS configuration file for Kerberos authentication // or an empty string if Kerberos is not enabled. // See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html From c067058b801b082e29f414b0ad755418608b9c27 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 18:05:03 +0200 Subject: [PATCH 03/33] formatting --- .../build/properties/broker_properties.rs | 9 +++--- .../build/properties/controller_properties.rs | 6 ++-- .../operator-binary/src/resource/configmap.rs | 28 +++++++++---------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/broker_properties.rs b/rust/operator-binary/src/controller/build/properties/broker_properties.rs index 01bd0d1f..5d43ab79 100644 --- a/rust/operator-binary/src/controller/build/properties/broker_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/broker_properties.rs @@ -2,22 +2,21 @@ use std::collections::BTreeMap; use snafu::OptionExt; +use super::{Error, NoKraftControllersFoundSnafu, kraft_controllers}; use crate::{ crd::{ KafkaPodDescriptor, listener::{KafkaListenerConfig, KafkaListenerName}, role::{ - KAFKA_ADVERTISED_LISTENERS, KAFKA_BROKER_ID, - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, - KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, KAFKA_PROCESS_ROLES, KafkaRole, + KAFKA_ADVERTISED_LISTENERS, KAFKA_BROKER_ID, KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, + KAFKA_PROCESS_ROLES, KafkaRole, }, security::KafkaTlsSecurity, }, operations::graceful_shutdown::graceful_shutdown_config_properties, }; -use super::{Error, NoKraftControllersFoundSnafu, kraft_controllers}; - pub fn build( kafka_security: &KafkaTlsSecurity, listener_config: &KafkaListenerConfig, diff --git a/rust/operator-binary/src/controller/build/properties/controller_properties.rs b/rust/operator-binary/src/controller/build/properties/controller_properties.rs index f9862dd3..6a8172a8 100644 --- a/rust/operator-binary/src/controller/build/properties/controller_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/controller_properties.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use snafu::OptionExt; +use super::{Error, NoKraftControllersFoundSnafu, kraft_controllers}; use crate::{ crd::{ KafkaPodDescriptor, @@ -15,8 +16,6 @@ use crate::{ operations::graceful_shutdown::graceful_shutdown_config_properties, }; -use super::{Error, NoKraftControllersFoundSnafu, kraft_controllers}; - pub fn build( kafka_security: &KafkaTlsSecurity, listener_config: &KafkaListenerConfig, @@ -24,7 +23,8 @@ pub fn build( kraft_mode: bool, overrides: BTreeMap, ) -> Result, Error> { - let kraft_controllers = kraft_controllers(pod_descriptors).context(NoKraftControllersFoundSnafu)?; + let kraft_controllers = + kraft_controllers(pod_descriptors).context(NoKraftControllersFoundSnafu)?; let mut result = BTreeMap::from([ ( diff --git a/rust/operator-binary/src/resource/configmap.rs b/rust/operator-binary/src/resource/configmap.rs index 3f400e6c..bba2bac2 100644 --- a/rust/operator-binary/src/resource/configmap.rs +++ b/rust/operator-binary/src/resource/configmap.rs @@ -96,21 +96,19 @@ pub fn build_rolegroup_config_map( .unwrap_or_default(); let kafka_config = match merged_config { - AnyConfig::Broker(_) => { - crate::controller::build::properties::broker_properties::build( - kafka_security, - listener_config, - pod_descriptors, - opa_connect_string, - metadata_manager == MetadataManager::KRaft, - kafka - .spec - .cluster_config - .broker_id_pod_config_map_name - .is_some(), - overrides, - ) - } + AnyConfig::Broker(_) => crate::controller::build::properties::broker_properties::build( + kafka_security, + listener_config, + pod_descriptors, + opa_connect_string, + metadata_manager == MetadataManager::KRaft, + kafka + .spec + .cluster_config + .broker_id_pod_config_map_name + .is_some(), + overrides, + ), AnyConfig::Controller(_) => { crate::controller::build::properties::controller_properties::build( kafka_security, From cb850dcf63bffc7a5335de6c81b83ba236e7f3ec Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 18:15:31 +0200 Subject: [PATCH 04/33] refactor: move rolegroup ConfigMap build into controller/build/config_map Relocates build_rolegroup_config_map (plus the jaas_config_file helper and its Error enum) from resource/configmap.rs to controller/build/config_map.rs, colocating the ConfigMap assembler with the per-file property builders and matching the controller/build/config_map.rs layout in hdfs/airflow. Pure move + repointed caller; no behaviour change (18 tests pass). Co-Authored-By: Claude Opus 4.8 --- rust/operator-binary/src/controller.rs | 5 ++--- .../configmap.rs => controller/build/config_map.rs} | 0 rust/operator-binary/src/controller/build/mod.rs | 1 + rust/operator-binary/src/resource/mod.rs | 1 - 4 files changed, 3 insertions(+), 4 deletions(-) rename rust/operator-binary/src/{resource/configmap.rs => controller/build/config_map.rs} (100%) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index bebda106..45006a5e 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -40,7 +40,6 @@ use crate::{ discovery::{self, build_discovery_configmap}, operations::pdb::add_pdbs, resource::{ - configmap::build_rolegroup_config_map, listener::build_broker_rolegroup_bootstrap_listener, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, statefulset::{build_broker_rolegroup_statefulset, build_controller_rolegroup_statefulset}, @@ -167,7 +166,7 @@ pub enum Error { #[snafu(display("failed to build configmap"))] BuildConfigMap { - source: crate::resource::configmap::Error, + source: crate::controller::build::config_map::Error, }, #[snafu(display("failed to build service"))] @@ -329,7 +328,7 @@ pub async fn reconcile_kafka( ) .context(BuildPodDescriptorsSnafu)?; - let rg_configmap = build_rolegroup_config_map( + let rg_configmap = build::config_map::build_rolegroup_config_map( kafka, &image, &kafka_security, diff --git a/rust/operator-binary/src/resource/configmap.rs b/rust/operator-binary/src/controller/build/config_map.rs similarity index 100% rename from rust/operator-binary/src/resource/configmap.rs rename to rust/operator-binary/src/controller/build/config_map.rs diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 2f51c3f6..b8c4c422 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -1,3 +1,4 @@ //! Builders that assemble Kubernetes resources for kafka rolegroups. +pub mod config_map; pub mod properties; diff --git a/rust/operator-binary/src/resource/mod.rs b/rust/operator-binary/src/resource/mod.rs index a79483f8..514d0adb 100644 --- a/rust/operator-binary/src/resource/mod.rs +++ b/rust/operator-binary/src/resource/mod.rs @@ -1,4 +1,3 @@ -pub mod configmap; pub mod listener; pub mod service; pub mod statefulset; From 3215d08ebdd082b3dfb4a79f799787b575c37c70 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 18:49:39 +0200 Subject: [PATCH 05/33] feat: remove product-config; merge config/env overrides directly in validate Replaces the product-config validation path with a ValidatedKafkaCluster that carries, per role group, the merged config plus the config-file, jvm-security, and env overrides resolved directly from the CRD (role <- role-group). Overrides now use stackable_operator::v2::config_overrides::KeyValueConfigOverrides (matching trino/hdfs); the v1 KeyValueOverridesProvider impls and the per-role Configuration impls are removed, and KAFKA_CLUSTER_ID injection moves into the override merge (collect_*_role_group_overrides). The dereferenced authorization config is folded into the validated cluster. Drops the product-config crate dependency (it remains transitive via stackable-operator). The CRD gains `nullable: true` on configOverrides values (v2 allows null to delete a key). Rendered .properties and env vars are unchanged (18 tests pass; byte parity to be confirmed via kuttl). Regenerated extra/crds.yaml and Cargo.nix. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 1 - Cargo.nix | 4 - Cargo.toml | 1 - extra/crds.yaml | 8 + rust/operator-binary/Cargo.toml | 1 - rust/operator-binary/src/controller.rs | 64 ++-- .../src/controller/build/config_map.rs | 22 +- .../src/controller/validate.rs | 353 +++++++++++++----- rust/operator-binary/src/crd/mod.rs | 30 +- rust/operator-binary/src/crd/role/broker.rs | 44 +-- .../src/crd/role/controller.rs | 44 +-- rust/operator-binary/src/crd/role/mod.rs | 2 + rust/operator-binary/src/main.rs | 8 +- .../src/resource/statefulset.rs | 22 +- 14 files changed, 314 insertions(+), 290 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f759557..87abc76f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2922,7 +2922,6 @@ dependencies = [ "futures", "indoc", "java-properties", - "product-config", "rstest", "serde", "serde_json", diff --git a/Cargo.nix b/Cargo.nix index 0cb45c8f..debf5eef 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -9662,10 +9662,6 @@ rec { name = "java-properties"; packageId = "java-properties"; } - { - name = "product-config"; - packageId = "product-config"; - } { name = "serde"; packageId = "serde"; diff --git a/Cargo.toml b/Cargo.toml index 2ce4ff81..74c770d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ edition = "2021" repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.111.1", features = ["crds", "webhook"] } anyhow = "1.0" diff --git a/extra/crds.yaml b/extra/crds.yaml index a2c3b7d9..9a83de14 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -541,6 +541,7 @@ spec: properties: broker.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -551,6 +552,7 @@ spec: type: object security.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1158,6 +1160,7 @@ spec: properties: broker.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1168,6 +1171,7 @@ spec: type: object security.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1786,6 +1790,7 @@ spec: properties: controller.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1796,6 +1801,7 @@ spec: type: object security.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -2235,6 +2241,7 @@ spec: properties: controller.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -2245,6 +2252,7 @@ spec: type: object security.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 23a9234b..077ccccd 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,6 @@ repository.workspace = true publish = false [dependencies] -product-config.workspace = true stackable-operator.workspace = true indoc.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 45006a5e..8f495905 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,9 +1,8 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha1::KafkaCluster`]. -use std::{str::FromStr, sync::Arc}; +use std::sync::Arc; use const_format::concatcp; -use product_config::ProductConfigManager; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, @@ -51,7 +50,6 @@ pub const KAFKA_FULL_CONTROLLER_NAME: &str = concatcp!(KAFKA_CONTROLLER_NAME, '. pub struct Ctx { pub client: stackable_operator::client::Client, - pub product_config: ProductConfigManager, pub operator_environment: OperatorEnvironmentOptions, } @@ -114,9 +112,6 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::role::Error }, - #[snafu(display("failed to patch service account"))] ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, @@ -156,9 +151,6 @@ pub enum Error { #[snafu(display("KafkaCluster object is misconfigured"))] MisconfiguredKafkaCluster { source: crd::Error }, - #[snafu(display("failed to parse role: {source}"))] - ParseRole { source: strum::ParseError }, - #[snafu(display("failed to build statefulset"))] BuildStatefulset { source: crate::resource::statefulset::Error, @@ -198,7 +190,6 @@ impl ReconcilerError for Error { Error::ApplyDiscoveryConfig { .. } => None, Error::DeleteOrphans { .. } => None, Error::CreateClusterResources { .. } => None, - Error::FailedToResolveConfig { .. } => None, Error::ApplyServiceAccount { .. } => None, Error::ApplyRoleBinding { .. } => None, Error::ApplyStatus { .. } => None, @@ -207,7 +198,6 @@ impl ReconcilerError for Error { Error::GetRequiredLabels { .. } => None, Error::InvalidKafkaCluster { .. } => None, Error::MisconfiguredKafkaCluster { .. } => None, - Error::ParseRole { .. } => None, Error::BuildStatefulset { .. } => None, Error::BuildConfigMap { .. } => None, Error::BuildService { .. } => None, @@ -238,18 +228,13 @@ pub async fn reconcile_kafka( .context(DereferenceSnafu)?; // validate (no client required) - let validate::ValidatedInputs { + let validate::ValidatedKafkaCluster { authorization_config, image, kafka_security, - role_config: validated_config, - } = validate::validate( - kafka, - dereferenced_objects, - &ctx.operator_environment, - &ctx.product_config, - ) - .context(ValidateClusterSnafu)?; + role_groups, + } = validate::validate(kafka, dereferenced_objects, &ctx.operator_environment) + .context(ValidateClusterSnafu)?; let opa_connect = authorization_config .as_ref() @@ -295,15 +280,9 @@ pub async fn reconcile_kafka( let mut bootstrap_listeners = Vec::::new(); - for (kafka_role_str, role_config) in &validated_config { - let kafka_role = KafkaRole::from_str(kafka_role_str).context(ParseRoleSnafu)?; - - for (rolegroup_name, rolegroup_config) in role_config.iter() { - let rolegroup_ref = kafka.rolegroup_ref(&kafka_role, rolegroup_name); - - let merged_config = kafka_role - .merged_config(kafka, &rolegroup_ref.role_group) - .context(FailedToResolveConfigSnafu)?; + for (kafka_role, rg_map) in &role_groups { + for (rolegroup_name, validated_rg) in rg_map { + let rolegroup_ref = kafka.rolegroup_ref(kafka_role, rolegroup_name); let rg_headless_service = build_rolegroup_headless_service(kafka, &image, &rolegroup_ref, &kafka_security) @@ -333,8 +312,9 @@ pub async fn reconcile_kafka( &image, &kafka_security, &rolegroup_ref, - rolegroup_config, - &merged_config, + validated_rg.config_file_overrides.clone(), + validated_rg.jvm_security_overrides.clone(), + &validated_rg.merged_config, &kafka_listeners, &pod_descriptors, opa_connect.as_deref(), @@ -344,37 +324,37 @@ pub async fn reconcile_kafka( let rg_statefulset = match kafka_role { KafkaRole::Broker => build_broker_rolegroup_statefulset( kafka, - &kafka_role, + kafka_role, &image, &rolegroup_ref, - rolegroup_config, + &validated_rg.env_overrides, &kafka_security, - &merged_config, + &validated_rg.merged_config, &rbac_sa, &client.kubernetes_cluster_info, ) .context(BuildStatefulsetSnafu)?, KafkaRole::Controller => build_controller_rolegroup_statefulset( kafka, - &kafka_role, + kafka_role, &image, &rolegroup_ref, - rolegroup_config, + &validated_rg.env_overrides, &kafka_security, - &merged_config, + &validated_rg.merged_config, &rbac_sa, &client.kubernetes_cluster_info, ) .context(BuildStatefulsetSnafu)?, }; - if let AnyConfig::Broker(broker_config) = merged_config { + if let AnyConfig::Broker(broker_config) = &validated_rg.merged_config { let rg_bootstrap_listener = build_broker_rolegroup_bootstrap_listener( kafka, &image, &kafka_security, &rolegroup_ref, - &broker_config, + broker_config, ) .context(BuildListenerSnafu)?; bootstrap_listeners.push( @@ -417,12 +397,12 @@ pub async fn reconcile_kafka( ); } - let role_config = kafka.role_config(&kafka_role); + let role_cfg = kafka.role_config(kafka_role); if let Some(GenericRoleConfig { pod_disruption_budget: pdb, - }) = role_config + }) = role_cfg { - add_pdbs(pdb, kafka, &kafka_role, client, &mut cluster_resources) + add_pdbs(pdb, kafka, kafka_role, client, &mut cluster_resources) .await .context(FailedToCreatePdbSnafu)?; } diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index bba2bac2..b018f449 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -1,7 +1,6 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use indoc::formatdoc; -use product_config::types::PropertyNameKind; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, @@ -78,7 +77,8 @@ pub fn build_rolegroup_config_map( resolved_product_image: &ResolvedProductImage, kafka_security: &KafkaTlsSecurity, rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, + config_file_overrides: BTreeMap, + jvm_security_overrides: BTreeMap, merged_config: &AnyConfig, listener_config: &KafkaListenerConfig, pod_descriptors: &[KafkaPodDescriptor], @@ -90,11 +90,6 @@ pub fn build_rolegroup_config_map( .effective_metadata_manager() .context(InvalidMetadataManagerSnafu)?; - let overrides = rolegroup_config - .get(&PropertyNameKind::File(kafka_config_file_name.to_string())) - .cloned() - .unwrap_or_default(); - let kafka_config = match merged_config { AnyConfig::Broker(_) => crate::controller::build::properties::broker_properties::build( kafka_security, @@ -107,7 +102,7 @@ pub fn build_rolegroup_config_map( .cluster_config .broker_id_pod_config_map_name .is_some(), - overrides, + config_file_overrides, ), AnyConfig::Controller(_) => { crate::controller::build::properties::controller_properties::build( @@ -115,7 +110,7 @@ pub fn build_rolegroup_config_map( listener_config, pod_descriptors, metadata_manager == MetadataManager::KRaft, - overrides, + config_file_overrides, ) } } @@ -128,12 +123,7 @@ pub fn build_rolegroup_config_map( .map(|(k, v)| (k, Some(v))) .collect::>(); - let jvm_sec_props: BTreeMap> = rolegroup_config - .get(&PropertyNameKind::File( - JVM_SECURITY_PROPERTIES_FILE.to_string(), - )) - .cloned() - .unwrap_or_default() + let jvm_sec_props: BTreeMap> = jvm_security_overrides .into_iter() .map(|(k, v)| (k, Some(v))) .collect(); diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index a3e441a0..72d6fa6a 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,28 +1,23 @@ //! The validate step in the KafkaCluster controller. //! //! Synchronously validates inputs that don't require a Kubernetes client. Produces -//! [`ValidatedInputs`], consumed by the rest of `reconcile_kafka`. +//! [`ValidatedKafkaCluster`], consumed by the rest of `reconcile_kafka`. -use std::collections::HashMap; +use std::collections::BTreeMap; -use product_config::{ProductConfigManager, types::PropertyNameKind}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, - product_config_utils::{ - ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, - validate_all_roles_and_groups_config, - }, }; use crate::{ controller::dereference::DereferencedObjects, crd::{ - self, CONTAINER_IMAGE_BASE_NAME, JVM_SECURITY_PROPERTIES_FILE, + self, CONTAINER_IMAGE_BASE_NAME, authentication::{self}, authorization::KafkaAuthorizationConfig, - role::{KafkaRole, broker::BROKER_PROPERTIES_FILE, controller::CONTROLLER_PROPERTIES_FILE}, + role::{AnyConfig, KafkaRole}, security::{self, KafkaTlsSecurity}, v1alpha1, }, @@ -44,25 +39,39 @@ pub enum Error { #[snafu(display("cluster object defines no '{role}' role"))] MissingKafkaRole { source: crd::Error, role: KafkaRole }, - #[snafu(display("failed to generate product config"))] - GenerateProductConfig { - source: stackable_operator::product_config_utils::Error, - }, - - #[snafu(display("invalid product config"))] - InvalidProductConfig { - source: stackable_operator::product_config_utils::Error, - }, + #[snafu(display("failed to resolve merged config for rolegroup"))] + ResolveMergedConfig { source: crate::crd::role::Error }, } type Result = std::result::Result; -/// Synchronous inputs the rest of `reconcile_kafka` needs after dereferencing. -pub struct ValidatedInputs { - pub authorization_config: Option, +/// The validated cluster. Carries everything the build steps need, resolved once +/// here so downstream code never re-derives it or touches the raw spec. +pub struct ValidatedKafkaCluster { pub image: ResolvedProductImage, pub kafka_security: KafkaTlsSecurity, - pub role_config: ValidatedRoleConfigByPropertyKind, + // DESIGN DECISION: the dereferenced authorization config is folded into the + // validated cluster (read from here downstream). The other dereferenced input, + // the authentication classes, is intentionally NOT stored: it is fully consumed + // here to build `kafka_security`. Alternative: also store the resolved auth + // classes — rejected because nothing downstream needs them beyond kafka_security. + pub authorization_config: Option, + pub role_groups: BTreeMap>, +} + +pub struct ValidatedRoleGroupConfig { + pub merged_config: AnyConfig, + // DESIGN DECISION: overrides are resolved into flat maps HERE rather than stored + // as the typed KeyValueConfigOverrides and resolved in the per-file builders (the + // hdfs-operator pattern). Reason: broker and controller use different override + // struct types (KafkaBrokerConfigOverrides vs KafkaControllerConfigOverrides), so a + // single typed field would require an enum. Resolving here keeps the build/properties + // builders taking plain `BTreeMap`. Alternative: an enum over the two + // override types threaded to builders that call resolved_overrides() — more types for + // no behavioural gain. + pub config_file_overrides: BTreeMap, + pub jvm_security_overrides: BTreeMap, + pub env_overrides: BTreeMap, } /// Validates the cluster spec and the dereferenced inputs. @@ -70,8 +79,7 @@ pub fn validate( kafka: &v1alpha1::KafkaCluster, dereferenced_objects: DereferencedObjects, operator_environment: &OperatorEnvironmentOptions, - product_config: &ProductConfigManager, -) -> Result { +) -> Result { let image = kafka .spec .image @@ -99,81 +107,242 @@ pub fn validate( .validate_authentication_methods() .context(FailedToValidateAuthenticationMethodSnafu)?; - let role_config = validated_product_config(kafka, &image.product_version, product_config)?; + // DESIGN DECISION: build the per-rolegroup config (merged config + resolved overrides) + // here, so reconcile reads a fully-typed ValidatedKafkaCluster instead of re-deriving + // merged_config in the loop and threading a product-config HashMap. Alternative: keep + // deriving merged_config in the reconcile loop — rejected; validation is the right place + // to prove every rolegroup resolves before any resource is built. + let mut role_groups: BTreeMap> = + BTreeMap::new(); - Ok(ValidatedInputs { - authorization_config: dereferenced_objects.authorization_config, + // Brokers always exist. + let broker_role = kafka + .broker_role() + .cloned() + .context(MissingKafkaRoleSnafu { + role: KafkaRole::Broker, + })?; + + let mut broker_groups: BTreeMap = BTreeMap::new(); + for rolegroup_name in broker_role.role_groups.keys() { + let merged_config = KafkaRole::Broker + .merged_config(kafka, rolegroup_name) + .context(ResolveMergedConfigSnafu)?; + let (config_file_overrides, jvm_security_overrides, env_overrides) = + collect_broker_role_group_overrides(kafka, &broker_role, rolegroup_name); + broker_groups.insert( + rolegroup_name.clone(), + ValidatedRoleGroupConfig { + merged_config, + config_file_overrides, + jvm_security_overrides, + env_overrides, + }, + ); + } + role_groups.insert(KafkaRole::Broker, broker_groups); + + // We need this guard because controller_role() returns an error if controllers is None, + // which would stop reconciliation for ZooKeeper-mode clusters. + if kafka.spec.controllers.is_some() { + let controller_role = kafka + .controller_role() + .cloned() + .context(MissingKafkaRoleSnafu { + role: KafkaRole::Controller, + })?; + + let mut controller_groups: BTreeMap = BTreeMap::new(); + for rolegroup_name in controller_role.role_groups.keys() { + let merged_config = KafkaRole::Controller + .merged_config(kafka, rolegroup_name) + .context(ResolveMergedConfigSnafu)?; + let (config_file_overrides, jvm_security_overrides, env_overrides) = + collect_controller_role_group_overrides(kafka, &controller_role, rolegroup_name); + controller_groups.insert( + rolegroup_name.clone(), + ValidatedRoleGroupConfig { + merged_config, + config_file_overrides, + jvm_security_overrides, + env_overrides, + }, + ); + } + role_groups.insert(KafkaRole::Controller, controller_groups); + } + + Ok(ValidatedKafkaCluster { image, kafka_security, - role_config, + authorization_config: dereferenced_objects.authorization_config, + role_groups, }) } -fn validated_product_config( +// DESIGN DECISION: role-group overrides are merged role-level first, then role-group +// extended on top so role-group wins — identical to the precedent product-config used. +// We read the v2 KeyValueConfigOverrides `.overrides` map and BTreeMap::extend rather +// than using its `Merge` impl, because plain extend reproduces the old behaviour exactly +// (last-writer-wins per key) and avoids depending on Merge semantics. Alternative: +// KeyValueConfigOverrides::merge — equivalent here but an unnecessary semantic dependency. +fn collect_broker_role_group_overrides( kafka: &v1alpha1::KafkaCluster, - product_version: &str, - product_config: &ProductConfigManager, -) -> Result { - let mut role_config = HashMap::new(); - - let broker_role = [( - KafkaRole::Broker.to_string(), - ( - vec![ - PropertyNameKind::File(BROKER_PROPERTIES_FILE.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - PropertyNameKind::Env, - ], - kafka - .broker_role() - .cloned() - .context(MissingKafkaRoleSnafu { - role: KafkaRole::Broker, - })? - .erase(), - ), - )] - .into(); - - let broker_role_config = - transform_all_roles_to_config(kafka, &broker_role).context(GenerateProductConfigSnafu)?; - - role_config.extend(broker_role_config); - - // We need this because controller_role() raises an error if non-existent, - // which would stop reconciliation. - if kafka.spec.controllers.is_some() { - let controller_role = [( - KafkaRole::Controller.to_string(), - ( - vec![ - PropertyNameKind::File(CONTROLLER_PROPERTIES_FILE.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - PropertyNameKind::Env, - ], - kafka - .controller_role() - .cloned() - .context(MissingKafkaRoleSnafu { - role: KafkaRole::Controller, - })? - .erase(), - ), - )] - .into(); - - let controller_role_config = transform_all_roles_to_config(kafka, &controller_role) - .context(GenerateProductConfigSnafu)?; - - role_config.extend(controller_role_config); + broker_role: &crate::crd::BrokerRole, + rolegroup_name: &str, +) -> ( + BTreeMap, + BTreeMap, + BTreeMap, +) { + // --- broker.properties overrides --- + let role_broker_overrides: BTreeMap> = broker_role + .config + .config_overrides + .broker_properties + .as_ref() + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let rg_broker_overrides: BTreeMap> = broker_role + .role_groups + .get(rolegroup_name) + .and_then(|rg| rg.config.config_overrides.broker_properties.as_ref()) + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let mut merged_broker = role_broker_overrides; + merged_broker.extend(rg_broker_overrides); + let config_file_overrides: BTreeMap = merged_broker + .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) + .collect(); + + // --- security.properties overrides --- + let role_security_overrides: BTreeMap> = broker_role + .config + .config_overrides + .security_properties + .as_ref() + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let rg_security_overrides: BTreeMap> = broker_role + .role_groups + .get(rolegroup_name) + .and_then(|rg| rg.config.config_overrides.security_properties.as_ref()) + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let mut merged_security = role_security_overrides; + merged_security.extend(rg_security_overrides); + let jvm_security_overrides: BTreeMap = merged_security + .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) + .collect(); + + // --- env overrides --- + // DESIGN DECISION: KAFKA_CLUSTER_ID is injected first, then the user env overrides + // (role then role-group) are extended on top, so a user override of the same key wins. + // This mirrors product-config's old merge of compute_env() output with user envOverrides. + // Alternative: inject after user overrides (operator wins) — rejected to preserve the + // previous precedence. + // + // KAFKA_CLUSTER_ID injection moved here from crd/role/broker.rs::Configuration::compute_env. + let mut env_overrides: BTreeMap = BTreeMap::new(); + if let Some(cluster_id) = kafka.cluster_id() { + env_overrides.insert("KAFKA_CLUSTER_ID".to_string(), cluster_id.to_string()); + } + let role_env: &std::collections::HashMap = &broker_role.config.env_overrides; + env_overrides.extend(role_env.iter().map(|(k, v)| (k.clone(), v.clone()))); + if let Some(rg) = broker_role.role_groups.get(rolegroup_name) { + env_overrides.extend( + rg.config + .env_overrides + .iter() + .map(|(k, v)| (k.clone(), v.clone())), + ); + } + + (config_file_overrides, jvm_security_overrides, env_overrides) +} + +// DESIGN DECISION: role-group overrides are merged role-level first, then role-group +// extended on top so role-group wins — identical to the precedent product-config used. +// We read the v2 KeyValueConfigOverrides `.overrides` map and BTreeMap::extend rather +// than using its `Merge` impl, because plain extend reproduces the old behaviour exactly +// (last-writer-wins per key) and avoids depending on Merge semantics. Alternative: +// KeyValueConfigOverrides::merge — equivalent here but an unnecessary semantic dependency. +fn collect_controller_role_group_overrides( + kafka: &v1alpha1::KafkaCluster, + controller_role: &crate::crd::ControllerRole, + rolegroup_name: &str, +) -> ( + BTreeMap, + BTreeMap, + BTreeMap, +) { + // --- controller.properties overrides --- + let role_controller_overrides: BTreeMap> = controller_role + .config + .config_overrides + .controller_properties + .as_ref() + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let rg_controller_overrides: BTreeMap> = controller_role + .role_groups + .get(rolegroup_name) + .and_then(|rg| rg.config.config_overrides.controller_properties.as_ref()) + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let mut merged_controller = role_controller_overrides; + merged_controller.extend(rg_controller_overrides); + let config_file_overrides: BTreeMap = merged_controller + .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) + .collect(); + + // --- security.properties overrides --- + let role_security_overrides: BTreeMap> = controller_role + .config + .config_overrides + .security_properties + .as_ref() + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let rg_security_overrides: BTreeMap> = controller_role + .role_groups + .get(rolegroup_name) + .and_then(|rg| rg.config.config_overrides.security_properties.as_ref()) + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let mut merged_security = role_security_overrides; + merged_security.extend(rg_security_overrides); + let jvm_security_overrides: BTreeMap = merged_security + .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) + .collect(); + + // --- env overrides --- + // DESIGN DECISION: KAFKA_CLUSTER_ID is injected first, then the user env overrides + // (role then role-group) are extended on top, so a user override of the same key wins. + // This mirrors product-config's old merge of compute_env() output with user envOverrides. + // Alternative: inject after user overrides (operator wins) — rejected to preserve the + // previous precedence. + // + // KAFKA_CLUSTER_ID injection moved here from crd/role/controller.rs::Configuration::compute_env. + let mut env_overrides: BTreeMap = BTreeMap::new(); + if let Some(cluster_id) = kafka.cluster_id() { + env_overrides.insert("KAFKA_CLUSTER_ID".to_string(), cluster_id.to_string()); + } + let role_env: &std::collections::HashMap = + &controller_role.config.env_overrides; + env_overrides.extend(role_env.iter().map(|(k, v)| (k.clone(), v.clone()))); + if let Some(rg) = controller_role.role_groups.get(rolegroup_name) { + env_overrides.extend( + rg.config + .env_overrides + .iter() + .map(|(k, v)| (k.clone(), v.clone())), + ); } - validate_all_roles_and_groups_config( - product_version, - &role_config, - product_config, - false, - false, - ) - .context(InvalidProductConfigSnafu) + (config_file_overrides, jvm_security_overrides, env_overrides) } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index d662de30..fa5b7498 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -16,13 +16,13 @@ use stackable_operator::{ cluster_operation::ClusterOperation, networking::DomainName, product_image_selection::ProductImage, }, - config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider}, deep_merger::ObjectOverrides, kube::{CustomResource, runtime::reflector::ObjectRef}, role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, utils::cluster_info::KubernetesClusterInfo, + v2::config_overrides::KeyValueConfigOverrides, versioned::versioned, }; use strum::{Display, EnumIter, EnumString}; @@ -240,6 +240,8 @@ pub mod versioned { pub broker_id_pod_config_map_name: Option, } + // Uses the v2 KeyValueConfigOverrides (Merge-capable, `nullable` values) to match + // trino/hdfs. Resolution into flat maps happens in controller/validate.rs. #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KafkaBrokerConfigOverrides { @@ -291,32 +293,6 @@ impl Default for v1alpha1::KafkaClusterConfig { } } -impl KeyValueOverridesProvider for v1alpha1::KafkaBrokerConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - let field = match file { - role::broker::BROKER_PROPERTIES_FILE => self.broker_properties.as_ref(), - JVM_SECURITY_PROPERTIES_FILE => self.security_properties.as_ref(), - _ => None, - }; - field - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default() - } -} - -impl KeyValueOverridesProvider for v1alpha1::KafkaControllerConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - let field = match file { - role::controller::CONTROLLER_PROPERTIES_FILE => self.controller_properties.as_ref(), - JVM_SECURITY_PROPERTIES_FILE => self.security_properties.as_ref(), - _ => None, - }; - field - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default() - } -} - impl HasStatusCondition for v1alpha1::KafkaCluster { fn conditions(&self) -> Vec { match &self.status { diff --git a/rust/operator-binary/src/crd/role/broker.rs b/rust/operator-binary/src/crd/role/broker.rs index 70ac85d0..674b9feb 100644 --- a/rust/operator-binary/src/crd/role/broker.rs +++ b/rust/operator-binary/src/crd/role/broker.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::resources::{ @@ -8,16 +6,12 @@ use stackable_operator::{ }, config::{fragment::Fragment, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, - product_config_utils::Configuration, product_logging::{self, spec::Logging}, schemars::{self, JsonSchema}, }; use strum::{Display, EnumIter}; -use crate::crd::{ - role::commons::{CommonConfig, Storage, StorageFragment}, - v1alpha1, -}; +use crate::crd::role::commons::{CommonConfig, Storage, StorageFragment}; pub const BROKER_PROPERTIES_FILE: &str = "broker.properties"; @@ -102,38 +96,4 @@ impl BrokerConfig { } } -impl Configuration for BrokerConfigFragment { - type Configurable = v1alpha1::KafkaCluster; - - fn compute_env( - &self, - resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - let mut result = BTreeMap::new(); - if let Some(cluster_id) = resource.cluster_id() { - result.insert("KAFKA_CLUSTER_ID".to_string(), Some(cluster_id.to_string())); - } - Ok(result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _resource: &Self::Configurable, - _role_name: &str, - _file: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } -} +// KAFKA_CLUSTER_ID injection moved to controller/validate.rs::collect_broker_role_group_overrides. diff --git a/rust/operator-binary/src/crd/role/controller.rs b/rust/operator-binary/src/crd/role/controller.rs index bf1468b6..27756ee1 100644 --- a/rust/operator-binary/src/crd/role/controller.rs +++ b/rust/operator-binary/src/crd/role/controller.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::resources::{ @@ -8,16 +6,12 @@ use stackable_operator::{ }, config::{fragment::Fragment, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, - product_config_utils::Configuration, product_logging::{self, spec::Logging}, schemars::{self, JsonSchema}, }; use strum::{Display, EnumIter}; -use crate::crd::{ - role::commons::{CommonConfig, Storage, StorageFragment}, - v1alpha1, -}; +use crate::crd::role::commons::{CommonConfig, Storage, StorageFragment}; pub const CONTROLLER_PROPERTIES_FILE: &str = "controller.properties"; @@ -92,38 +86,4 @@ impl ControllerConfig { } } -impl Configuration for ControllerConfigFragment { - type Configurable = v1alpha1::KafkaCluster; - - fn compute_env( - &self, - resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - let mut result = BTreeMap::new(); - if let Some(cluster_id) = resource.cluster_id() { - result.insert("KAFKA_CLUSTER_ID".to_string(), Some(cluster_id.to_string())); - } - Ok(result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _resource: &Self::Configurable, - _role_name: &str, - _file: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } -} +// KAFKA_CLUSTER_ID injection moved to controller/validate.rs::collect_controller_role_group_overrides. diff --git a/rust/operator-binary/src/crd/role/mod.rs b/rust/operator-binary/src/crd/role/mod.rs index e08474ed..6c7bac4f 100644 --- a/rust/operator-binary/src/crd/role/mod.rs +++ b/rust/operator-binary/src/crd/role/mod.rs @@ -98,7 +98,9 @@ pub enum Error { Eq, Hash, JsonSchema, + Ord, PartialEq, + PartialOrd, Serialize, EnumString, )] diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index c074f8af..25362c54 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -80,9 +80,9 @@ async fn main() -> anyhow::Result<()> { RunArguments { operator_environment, watch_namespace, - product_config, maintenance, common, + .. }, .. }) => { @@ -127,11 +127,6 @@ async fn main() -> anyhow::Result<()> { .run(sigterm_watcher.handle()) .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let product_config = product_config.load(&[ - "deploy/config-spec/properties.yaml", - "/etc/stackable/kafka-operator/config-spec/properties.yaml", - ])?; - let event_recorder = Arc::new(Recorder::new( client.as_kube_client(), Reporter { @@ -188,7 +183,6 @@ async fn main() -> anyhow::Result<()> { Arc::new(controller::Ctx { client: client.clone(), operator_environment, - product_config, }), ) // We can let the reporting happen in the background diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index 5cb262f0..7f232793 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -1,9 +1,5 @@ -use std::{ - collections::{BTreeMap, HashMap}, - ops::Deref, -}; +use std::{collections::BTreeMap, ops::Deref}; -use product_config::types::PropertyNameKind; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -171,7 +167,7 @@ pub fn build_broker_rolegroup_statefulset( kafka_role: &KafkaRole, resolved_product_image: &ResolvedProductImage, rolegroup_ref: &RoleGroupRef, - broker_config: &HashMap>, + env_overrides: &BTreeMap, kafka_security: &KafkaTlsSecurity, merged_config: &AnyConfig, service_account: &ServiceAccount, @@ -249,10 +245,8 @@ pub fn build_broker_rolegroup_statefulset( .context(AddKerberosConfigSnafu)?; } - let mut env = broker_config - .get(&PropertyNameKind::Env) - .into_iter() - .flatten() + let mut env = env_overrides + .iter() .map(|(k, v)| EnvVar { name: k.clone(), value: Some(v.clone()), @@ -581,7 +575,7 @@ pub fn build_controller_rolegroup_statefulset( kafka_role: &KafkaRole, resolved_product_image: &ResolvedProductImage, rolegroup_ref: &RoleGroupRef, - controller_config: &HashMap>, + env_overrides: &BTreeMap, kafka_security: &KafkaTlsSecurity, merged_config: &AnyConfig, service_account: &ServiceAccount, @@ -603,10 +597,8 @@ pub fn build_controller_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); - let mut env = controller_config - .get(&PropertyNameKind::Env) - .into_iter() - .flatten() + let mut env = env_overrides + .iter() .map(|(k, v)| EnvVar { name: k.clone(), value: Some(v.clone()), From b2183e3e97d329a246cd7d4a19bf9c4d895ccf75 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 18:54:43 +0200 Subject: [PATCH 06/33] docs: remove product-config CLI param + env var; gut config-spec; changelog Removes the --product-config section from the commandline reference and the PRODUCT_CONFIG section from the environment-variables reference (the flag/var is now a no-op via the shared RunArguments), reduces the product-config properties.yaml files to an empty shell (retained pending a later Helm config refactor), and notes the product-config removal in the changelog. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 4 + deploy/config-spec/properties.yaml | 151 +----------------- .../kafka-operator/configs/properties.yaml | 151 +----------------- .../reference/commandline-parameters.adoc | 13 -- .../reference/environment-variables.adoc | 26 --- 5 files changed, 8 insertions(+), 337 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 323d05c6..c4e21fb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ All notable changes to this project will be documented in this file. Previously, arbitrary file names were silently accepted and ignored ([#960]). - Bump `stackable-operator` to 0.111.1 and snafu to 0.9 ([#960], [#961]). - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#968]). +- Removed the product-config based configuration validation. Config and environment overrides are + now merged directly from the CRD into the validated cluster, the Java-properties writer is + vendored locally, and the `product-config` crate dependency is dropped. The `--product-config` + CLI flag is now a no-op ([#XXX]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#971]). [#953]: https://github.com/stackabletech/kafka-operator/pull/953 diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index b6f80cdd..9bd8c3b2 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -1,152 +1,5 @@ --- version: 0.1.0 spec: - units: - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" - - - unit: &unitUrl - name: "url" - regex: "^((https?|ftp|file)://)?[-a-zA-Z0-9+&@#}/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]" - examples: - - "https://www.stackable.de/blog/" - - - unit: &unitCapacity - name: "capacity" - regex: "^[1-9]\\d*$" - - - unit: &unitMilliseconds - name: "milliseconds" - regex: "^[1-9]\\d*$" - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for domain names that cannot be resolved." - description: "TTL for domain names that cannot be resolved." - - - property: &opaAuthorizerClassName - propertyNames: - - name: "authorizer.class.name" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "com.bisnode.kafka.authorization.OpaAuthorizer" - - fromVersion: "3.0.0" - value: "org.openpolicyagent.kafka.OpaAuthorizer" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer class name" - - - property: &opaAuthorizerUrl - propertyNames: - - name: "opa.authorizer.url" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - unit: *unitUrl - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer URL" - - - property: &opaAuthorizerInitialCacheCapacity - propertyNames: - - name: "opa.authorizer.cache.initial.capacity" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer initial cache capacity" - - - property: &opaAuthorizerMaxCacheSize - propertyNames: - - name: "opa.authorizer.cache.maximum.size" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA authorizer max cache size" - - - property: &opaAuthorizerCacheExpireAfterSeconds - propertyNames: - - name: "opa.authorizer.cache.expire.after.seconds" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "The number of seconds after which the OPA authorizer cache expires" + units: [] +properties: [] diff --git a/deploy/helm/kafka-operator/configs/properties.yaml b/deploy/helm/kafka-operator/configs/properties.yaml index b6f80cdd..9bd8c3b2 100644 --- a/deploy/helm/kafka-operator/configs/properties.yaml +++ b/deploy/helm/kafka-operator/configs/properties.yaml @@ -1,152 +1,5 @@ --- version: 0.1.0 spec: - units: - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" - - - unit: &unitUrl - name: "url" - regex: "^((https?|ftp|file)://)?[-a-zA-Z0-9+&@#}/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]" - examples: - - "https://www.stackable.de/blog/" - - - unit: &unitCapacity - name: "capacity" - regex: "^[1-9]\\d*$" - - - unit: &unitMilliseconds - name: "milliseconds" - regex: "^[1-9]\\d*$" - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for domain names that cannot be resolved." - description: "TTL for domain names that cannot be resolved." - - - property: &opaAuthorizerClassName - propertyNames: - - name: "authorizer.class.name" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "com.bisnode.kafka.authorization.OpaAuthorizer" - - fromVersion: "3.0.0" - value: "org.openpolicyagent.kafka.OpaAuthorizer" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer class name" - - - property: &opaAuthorizerUrl - propertyNames: - - name: "opa.authorizer.url" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - unit: *unitUrl - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer URL" - - - property: &opaAuthorizerInitialCacheCapacity - propertyNames: - - name: "opa.authorizer.cache.initial.capacity" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer initial cache capacity" - - - property: &opaAuthorizerMaxCacheSize - propertyNames: - - name: "opa.authorizer.cache.maximum.size" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA authorizer max cache size" - - - property: &opaAuthorizerCacheExpireAfterSeconds - propertyNames: - - name: "opa.authorizer.cache.expire.after.seconds" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "The number of seconds after which the OPA authorizer cache expires" + units: [] +properties: [] diff --git a/docs/modules/kafka/pages/reference/commandline-parameters.adoc b/docs/modules/kafka/pages/reference/commandline-parameters.adoc index 9059a960..3c66dc41 100644 --- a/docs/modules/kafka/pages/reference/commandline-parameters.adoc +++ b/docs/modules/kafka/pages/reference/commandline-parameters.adoc @@ -2,19 +2,6 @@ This operator accepts the following command line parameters: -== product-config - -*Default value*: `/etc/stackable/kafka-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values:* false - -[source] ----- -stackable-kafka-operator run --product-config /foo/bar/properties.yaml ----- - == watch-namespace *Default value*: All namespaces diff --git a/docs/modules/kafka/pages/reference/environment-variables.adoc b/docs/modules/kafka/pages/reference/environment-variables.adoc index cc7dd3a2..d2271300 100644 --- a/docs/modules/kafka/pages/reference/environment-variables.adoc +++ b/docs/modules/kafka/pages/reference/environment-variables.adoc @@ -33,32 +33,6 @@ docker run \ oci.stackable.tech/sdp/kafka-operator:0.0.0-dev ---- -== PRODUCT_CONFIG - -*Default value*: `/etc/stackable/kafka-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values*: false - -[source] ----- -export PRODUCT_CONFIG=/foo/bar/properties.yaml -stackable-kafka-operator run ----- - -or via docker: - ----- -docker run \ - --name kafka-operator \ - --network host \ - --env KUBECONFIG=/home/stackable/.kube/config \ - --env PRODUCT_CONFIG=/my/product/config.yaml \ - --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ - oci.stackable.tech/sdp/kafka-operator:0.0.0-dev ----- - == WATCH_NAMESPACE *Default value*: All namespaces From 5352a84a5110067959e04018fb8009206a27feae Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 5 Jun 2026 08:57:04 +0200 Subject: [PATCH 07/33] updated changelog --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4e21fb8..7d34355e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,11 +18,11 @@ All notable changes to this project will be documented in this file. Previously, arbitrary file names were silently accepted and ignored ([#960]). - Bump `stackable-operator` to 0.111.1 and snafu to 0.9 ([#960], [#961]). - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#968]). +- test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#971]). - Removed the product-config based configuration validation. Config and environment overrides are now merged directly from the CRD into the validated cluster, the Java-properties writer is vendored locally, and the `product-config` crate dependency is dropped. The `--product-config` - CLI flag is now a no-op ([#XXX]). -- test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#971]). + CLI flag is now a no-op ([#976]). [#953]: https://github.com/stackabletech/kafka-operator/pull/953 [#960]: https://github.com/stackabletech/kafka-operator/pull/960 @@ -30,6 +30,7 @@ All notable changes to this project will be documented in this file. [#968]: https://github.com/stackabletech/kafka-operator/pull/968 [#971]: https://github.com/stackabletech/kafka-operator/pull/971 [#973]: https://github.com/stackabletech/kafka-operator/pull/973 +[#976]: https://github.com/stackabletech/kafka-operator/pull/976 ## [26.3.0] - 2026-03-16 From cfd0efd1cd3855afe9f7939597af03efc3c8fb23 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 5 Jun 2026 17:13:02 +0200 Subject: [PATCH 08/33] refactor: consume the config-file writer from stackable-operator Replace the vendored Java-properties writer (rust/operator-binary/src/config/writer.rs) with stackable_operator::v2::config_file_writer (moved there via operator-rs #1217 on the smooth-operator branch). Kafka's copy was the java-only subset of the canonical hdfs writer; the upstream module's additional to_hadoop_xml simply goes unimported. Drop the now-unused java-properties dependency. No behaviour change; rendered .properties output is byte-identical by construction (same code, new home). Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 117 +++--- Cargo.nix | 341 ++++++++++-------- Cargo.toml | 1 - crate-hashes.json | 4 +- rust/operator-binary/Cargo.toml | 1 - rust/operator-binary/src/config/mod.rs | 1 - rust/operator-binary/src/config/writer.rs | 78 ---- .../src/controller/build/config_map.rs | 6 +- 8 files changed, 264 insertions(+), 285 deletions(-) delete mode 100644 rust/operator-binary/src/config/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 87abc76f..1390fe52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1518,11 +1518,11 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "regex", - "snafu 0.9.0", + "snafu 0.9.1", ] [[package]] @@ -1843,9 +1843,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -1857,9 +1857,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" dependencies = [ "opentelemetry", "tracing", @@ -1869,9 +1869,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", @@ -1882,9 +1882,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -1896,14 +1896,14 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1914,21 +1914,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", "tokio", @@ -2211,6 +2212,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.45" @@ -2350,9 +2360,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -2368,9 +2378,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -2813,11 +2820,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ - "snafu-derive 0.9.0", + "snafu-derive 0.9.1", ] [[package]] @@ -2845,9 +2852,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -2890,7 +2897,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "const-oid", "ecdsa", @@ -2902,7 +2909,7 @@ dependencies = [ "rsa", "sha2", "signature", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-shared", "tokio", "tokio-rustls", @@ -2921,12 +2928,11 @@ dependencies = [ "const_format", "futures", "indoc", - "java-properties", "rstest", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator", "strum", "tokio", @@ -2936,7 +2942,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "base64", "clap", @@ -2948,6 +2954,7 @@ dependencies = [ "futures", "http", "indexmap", + "java-properties", "jiff", "json-patch", "k8s-openapi", @@ -2960,7 +2967,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -2973,12 +2980,13 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "xml", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "proc-macro2", @@ -2988,8 +2996,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.1.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +version = "0.1.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "jiff", "k8s-openapi", @@ -2998,15 +3006,15 @@ dependencies = [ "semver", "serde", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "time", ] [[package]] name = "stackable-telemetry" -version = "0.6.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +version = "0.6.4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "axum", "clap", @@ -3017,7 +3025,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "pin-project", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "tokio", "tower", @@ -3030,21 +3038,21 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "kube", "schemars", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "convert_case", "convert_case_extras", @@ -3062,7 +3070,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "arc-swap", "async-trait", @@ -3078,7 +3086,7 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-certs", "stackable-shared", "stackable-telemetry", @@ -3414,6 +3422,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a875a902255423d34c1f20838ab374126db8eb41625b7947a1d54113b0b7399" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3525,9 +3544,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", @@ -3948,9 +3967,9 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" diff --git a/Cargo.nix b/Cargo.nix index debf5eef..321c6fe5 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4852,7 +4852,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "k8s_version"; @@ -4871,7 +4871,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } ]; features = { @@ -6052,9 +6052,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "18629xsj4rsyiby9aj511q6wcw6s9m09gx3ymw1yjcvix1mcsjxq"; + sha256 = "10ln14d1jgc8rvw97mblc9blzcgpg1bimim4d170b7ia4mijq55h"; dependencies = [ { name = "futures-core"; @@ -6091,24 +6091,24 @@ rec { ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "experimental_metrics_bound_instruments" = [ "metrics" ]; "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; "pin-project-lite" = [ "dep:pin-project-lite" ]; - "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "1hnwizzgfhpjfnvml638yy846py8hf2gl1n3p1igbk1srb2ilspg"; + sha256 = "0dyq4myan64sl8wly02jx0gb3jjz7575mn3w8rpphz0xvkq8001c"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6151,18 +6151,15 @@ rec { ]; features = { "experimental_metadata_attributes" = [ "dep:tracing-log" ]; - "experimental_use_tracing_span_context" = [ "tracing-opentelemetry" ]; "log" = [ "dep:log" ]; - "spec_unstable_logs_enabled" = [ "opentelemetry/spec_unstable_logs_enabled" ]; - "tracing-opentelemetry" = [ "dep:tracing-opentelemetry" ]; }; resolvedDefaultFeatures = [ "default" ]; }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0pc5nw1ds8v8w0nvyall39m92v8m1xl1p3vwvxk6nkhrffdd19np"; + sha256 = "0ca3drvm4fx5nskl7yn42dimy3bg35ppzc85y1p27pz215fh30sn"; libName = "opentelemetry_http"; dependencies = [ { @@ -6198,16 +6195,16 @@ rec { "internal-logs" = [ "opentelemetry/internal-logs" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking" = [ "dep:reqwest" "reqwest/blocking" ]; - "reqwest-rustls" = [ "dep:reqwest" "reqwest/rustls-tls-native-roots" ]; - "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/rustls-tls-webpki-roots" ]; + "reqwest-rustls" = [ "dep:reqwest" "reqwest/default-tls" ]; + "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/default-tls" "reqwest/webpki-roots" ]; }; - resolvedDefaultFeatures = [ "internal-logs" "reqwest" "reqwest-blocking" ]; + resolvedDefaultFeatures = [ "reqwest" "reqwest-blocking" ]; }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "07zp0b62b9dajnvvcd6j2ppw5zg7wp4ixka9z6fr3bxrrdmcss8z"; + sha256 = "0d9cys2flpidfxbr6h1103hjc633cax47ihnqgbj0xnicscr4rlr"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6268,10 +6265,9 @@ rec { usesDefaultFeatures = false; } { - name = "tracing"; - packageId = "tracing"; + name = "tonic-types"; + packageId = "tonic-types"; optional = true; - usesDefaultFeatures = false; } ]; devDependencies = [ @@ -6296,16 +6292,19 @@ rec { ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; + "experimental-grpc-retry" = [ "grpc-tonic" "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" ]; + "experimental-http-retry" = [ "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" "tokio" "httpdate" ]; "flate2" = [ "dep:flate2" ]; - "grpc-tonic" = [ "tonic" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; + "grpc-tonic" = [ "tonic" "tonic-types" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; "gzip-http" = [ "flate2" ]; "gzip-tonic" = [ "tonic/gzip" ]; "http" = [ "dep:http" ]; "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "trace" "metrics" ]; "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; + "httpdate" = [ "dep:httpdate" ]; "hyper-client" = [ "opentelemetry-http/hyper" ]; "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" "logs" ]; - "internal-logs" = [ "tracing" "opentelemetry_sdk/internal-logs" "opentelemetry-http/internal-logs" ]; + "internal-logs" = [ "opentelemetry_sdk/internal-logs" "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "opentelemetry-proto/metrics" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; @@ -6318,27 +6317,27 @@ rec { "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "serialize" = [ "serde" "serde_json" ]; - "tls" = [ "tonic/tls-ring" ]; + "tls" = [ "tls-ring" ]; "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; "tls-provider-agnostic" = [ "tonic/_tls-any" ]; "tls-ring" = [ "tonic/tls-ring" ]; - "tls-roots" = [ "tls" "tonic/tls-native-roots" ]; - "tls-webpki-roots" = [ "tls" "tonic/tls-webpki-roots" ]; + "tls-roots" = [ "tonic/tls-native-roots" ]; + "tls-webpki-roots" = [ "tonic/tls-webpki-roots" ]; "tokio" = [ "dep:tokio" ]; "tonic" = [ "dep:tonic" ]; + "tonic-types" = [ "dep:tonic-types" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" "opentelemetry-proto/trace" ]; - "tracing" = [ "dep:tracing" ]; "zstd" = [ "dep:zstd" ]; "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "03xkjsjrsm7zkkx5gascqd9bg2z20wymm06l16cyxsp5dpq5s5x7"; + sha256 = "0f5ny4rpnpq6q5q34b8k2q548rf31rpbxkwjqjwzfqxg3yx5imjn"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6382,30 +6381,29 @@ rec { "const-hex" = [ "dep:const-hex" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; - "gen-tonic-messages" = [ "tonic" "tonic-prost" "prost" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic" "tonic-prost" "tonic/channel" ]; + "gen-tonic-messages" = [ "prost" ]; "internal-logs" = [ "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" ]; "prost" = [ "dep:prost" ]; "schemars" = [ "dep:schemars" ]; "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; "testing" = [ "opentelemetry/testing" ]; "tonic" = [ "dep:tonic" ]; "tonic-prost" = [ "dep:tonic-prost" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" ]; "with-schemars" = [ "schemars" ]; - "with-serde" = [ "serde" "const-hex" "base64" "serde_json" ]; + "with-serde" = [ "serde" "const-hex" "base64" ]; "zpages" = [ "trace" ]; }; resolvedDefaultFeatures = [ "gen-tonic" "gen-tonic-messages" "logs" "metrics" "prost" "tonic" "tonic-prost" "trace" ]; }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0in8plv2l2ar7anzi7lrbll0fjfvaymkg5vc5bnvibs1w3gjjbp6"; + sha256 = "0s1x4h1cgmhkxb7i5g02la2vhkf4lg5g26cgn2s2gd1p0j5gk8kc"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -6413,9 +6411,9 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.31.0"; + version = "0.32.1"; edition = "2021"; - sha256 = "1gbjsggdxfpjbanjvaxa3nq32vfa37i3v13dvx4gsxhrk7sy8jp1"; + sha256 = "1ycl11syranrinhgn4c2hlzhyzyvpa06ryxq5mxgzmf4387ghncv"; dependencies = [ { name = "futures-channel"; @@ -6441,6 +6439,13 @@ rec { packageId = "percent-encoding"; optional = true; } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + usesDefaultFeatures = false; + target = { target, features }: (!("64" == target."has_atomic" or null)); + features = [ "fallback" ]; + } { name = "rand"; packageId = "rand 0.9.4"; @@ -6465,10 +6470,18 @@ rec { optional = true; } ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "macros" "rt-multi-thread" ]; + } + ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" "experimental_async_runtime" ]; - "experimental_logs_concurrent_log_processor" = [ "logs" ]; + "experimental_metrics_bound_instruments" = [ "metrics" "opentelemetry/experimental_metrics_bound_instruments" ]; "experimental_metrics_custom_reader" = [ "metrics" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" "experimental_async_runtime" ]; @@ -6485,15 +6498,14 @@ rec { "rt-tokio-current-thread" = [ "tokio/rt" "tokio/time" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; - "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "tokio/sync" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -7002,7 +7014,7 @@ rec { "default" = [ "fallback" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "require-cas" ]; + resolvedDefaultFeatures = [ "fallback" "require-cas" ]; }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; @@ -7270,6 +7282,34 @@ rec { ]; }; + "prost-types" = rec { + crateName = "prost-types"; + version = "0.14.3"; + edition = "2021"; + sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + libName = "prost_types"; + authors = [ + "Dan Burkert " + "Lucio Franco " + "Casper Meijn " + "Tokio Contributors " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "chrono" = [ "dep:chrono" ]; + "default" = [ "std" ]; + "std" = [ "prost/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "quote" = rec { crateName = "quote"; version = "1.0.45"; @@ -7699,9 +7739,9 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.12.28"; + version = "0.13.4"; edition = "2021"; - sha256 = "0iqidijghgqbzl3bjg5hb4zmigwa4r612bgi0yiq0c90b6jkrpgd"; + sha256 = "1hy1plns9krbh3h1dy2sdjygsfkdcnxm6pbxdi0ya9b5vq8mi711"; authors = [ "Sean McArthur " ]; @@ -7718,7 +7758,7 @@ rec { name = "futures-channel"; packageId = "futures-channel"; optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "futures-core"; @@ -7738,62 +7778,44 @@ rec { { name = "http-body"; packageId = "http-body"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "http-body-util"; packageId = "http-body-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "hyper"; packageId = "hyper"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" "client-legacy" "client-proxy" "tokio" ]; } { name = "js-sys"; packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "log"; packageId = "log"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "percent-encoding"; packageId = "percent-encoding"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "pin-project-lite"; packageId = "pin-project-lite"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "serde_urlencoded"; - packageId = "serde_urlencoded"; + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "sync_wrapper"; @@ -7804,27 +7826,27 @@ rec { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "net" "time" ]; } { name = "tower"; packageId = "tower"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "retry" "timeout" "util" ]; } { name = "tower-http"; packageId = "tower-http"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "follow-redirect" ]; } { name = "tower-service"; packageId = "tower-service"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "url"; @@ -7833,17 +7855,17 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "wasm-bindgen-futures"; packageId = "wasm-bindgen-futures"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "web-sys"; packageId = "web-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "RequestCache" ]; } ]; @@ -7852,33 +7874,27 @@ rec { name = "futures-util"; packageId = "futures-util"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "std" "alloc" ]; } { name = "hyper"; packageId = "hyper"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "server" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "client-legacy" "server-auto" "server-graceful" "tokio" ]; } - { - name = "serde"; - packageId = "serde"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "derive" ]; - } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "macros" "rt-multi-thread" ]; } { @@ -7890,40 +7906,37 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "serde-serialize" ]; } ]; features = { + "__native-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "__native-tls-alpn" = [ "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" ]; - "__rustls-ring" = [ "hyper-rustls?/ring" "tokio-rustls?/ring" "rustls?/ring" "quinn?/ring" ]; + "__rustls-aws-lc-rs" = [ "hyper-rustls?/aws-lc-rs" "tokio-rustls?/aws-lc-rs" "rustls?/aws-lc-rs" "quinn?/rustls-aws-lc-rs" ]; "__tls" = [ "dep:rustls-pki-types" "tokio/io-util" ]; "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "dep:futures-util" "futures-util?/io" "futures-util?/sink" "tokio/sync" ]; "brotli" = [ "tower-http/decompression-br" ]; "charset" = [ "dep:encoding_rs" "dep:mime" ]; "cookies" = [ "dep:cookie_crate" "dep:cookie_store" ]; "default" = [ "default-tls" "charset" "http2" "system-proxy" ]; - "default-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "default-tls" = [ "rustls" ]; "deflate" = [ "tower-http/decompression-deflate" ]; + "form" = [ "dep:serde" "dep:serde_urlencoded" ]; "gzip" = [ "tower-http/decompression-gzip" ]; - "h2" = [ "dep:h2" ]; "hickory-dns" = [ "dep:hickory-resolver" "dep:once_cell" ]; - "http2" = [ "h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; - "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; - "json" = [ "dep:serde_json" ]; - "macos-system-configuration" = [ "system-proxy" ]; + "http2" = [ "dep:h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; + "http3" = [ "rustls" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; + "json" = [ "dep:serde" "dep:serde_json" ]; "multipart" = [ "dep:mime_guess" "dep:futures-util" ]; - "native-tls" = [ "default-tls" ]; - "native-tls-alpn" = [ "native-tls" "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; - "native-tls-vendored" = [ "native-tls" "native-tls-crate?/vendored" ]; - "rustls-tls" = [ "rustls-tls-webpki-roots" ]; - "rustls-tls-manual-roots" = [ "rustls-tls-manual-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-manual-roots-no-provider" = [ "__rustls" ]; - "rustls-tls-native-roots" = [ "rustls-tls-native-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-native-roots-no-provider" = [ "dep:rustls-native-certs" "hyper-rustls?/native-tokio" "__rustls" ]; - "rustls-tls-no-provider" = [ "rustls-tls-manual-roots-no-provider" ]; - "rustls-tls-webpki-roots" = [ "rustls-tls-webpki-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-webpki-roots-no-provider" = [ "dep:webpki-roots" "hyper-rustls?/webpki-tokio" "__rustls" ]; + "native-tls" = [ "__native-tls" "__native-tls-alpn" ]; + "native-tls-no-alpn" = [ "__native-tls" ]; + "native-tls-vendored" = [ "__native-tls" "native-tls-crate?/vendored" "__native-tls-alpn" ]; + "native-tls-vendored-no-alpn" = [ "__native-tls" "native-tls-crate?/vendored" ]; + "query" = [ "dep:serde" "dep:serde_urlencoded" ]; + "rustls" = [ "__rustls-aws-lc-rs" "dep:rustls-platform-verifier" "__rustls" ]; + "rustls-no-provider" = [ "dep:rustls-platform-verifier" "__rustls" ]; "stream" = [ "tokio/fs" "dep:futures-util" "dep:tokio-util" "dep:wasm-streams" ]; "system-proxy" = [ "hyper-util/client-proxy-system" ]; "zstd" = [ "tower-http/decompression-zstd" ]; @@ -9297,29 +9310,25 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "rust_1_61" "rust_1_65" "std" ]; }; - "snafu 0.9.0" = rec { + "snafu 0.9.1" = rec { crateName = "snafu"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "1ii9r99x5qcn754m624yzgb9hzvkqkrcygf0aqh0pyb9dbnvrm6i"; + sha256 = "08k5yfydxdlshivfhrdq9km8qn02r93q28gkyvazbqz2icr1586i"; authors = [ "Jake Goulding " ]; dependencies = [ { name = "snafu-derive"; - packageId = "snafu-derive 0.9.0"; + packageId = "snafu-derive 0.9.1"; } ]; features = { - "backtrace" = [ "dep:backtrace" ]; - "backtraces-impl-backtrace-crate" = [ "backtrace" ]; + "backtraces-impl-backtrace-crate" = [ "dep:backtrace" ]; "default" = [ "std" "rust_1_81" ]; - "futures" = [ "futures-core-crate" "pin-project" ]; - "futures-core-crate" = [ "dep:futures-core-crate" ]; - "futures-crate" = [ "dep:futures-crate" ]; - "internal-dev-dependencies" = [ "futures-crate" ]; - "pin-project" = [ "dep:pin-project" ]; + "futures" = [ "dep:futures-core" "dep:pin-project" ]; + "internal-dev-dependencies" = [ "dep:futures" ]; "std" = [ "alloc" ]; "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; }; @@ -9387,11 +9396,11 @@ rec { }; resolvedDefaultFeatures = [ "rust_1_61" ]; }; - "snafu-derive 0.9.0" = rec { + "snafu-derive 0.9.1" = rec { crateName = "snafu-derive"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "0h0x61kyj4fvilcr2nj02l85shw1ika64vq9brf2gyna662ln9al"; + sha256 = "1nkfi7bis72pz3w7vb64m79w49qsv20sbf19jkd471vbhr83q42z"; procMacro = true; libName = "snafu_derive"; authors = [ @@ -9417,7 +9426,7 @@ rec { name = "syn"; packageId = "syn 2.0.117"; usesDefaultFeatures = false; - features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" ]; + features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } ]; features = { @@ -9526,7 +9535,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_certs"; @@ -9585,7 +9594,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-shared"; @@ -9658,10 +9667,6 @@ rec { name = "indoc"; packageId = "indoc"; } - { - name = "java-properties"; - packageId = "java-properties"; - } { name = "serde"; packageId = "serde"; @@ -9673,7 +9678,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator"; @@ -9721,7 +9726,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_operator"; @@ -9772,6 +9777,10 @@ rec { name = "indexmap"; packageId = "indexmap"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "jiff"; packageId = "jiff"; @@ -9829,7 +9838,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator-derive"; @@ -9887,12 +9896,17 @@ rec { name = "uuid"; packageId = "uuid"; } + { + name = "xml"; + packageId = "xml"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; + "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "crds" "certs" "time" "webhook" "kube-ws" ]; + "full" = [ "client-feature-gates" "crds" "certs" "time" "webhook" "kube-ws" ]; "kube-ws" = [ "kube/ws" ]; "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; @@ -9906,7 +9920,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; procMacro = true; @@ -9936,12 +9950,12 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.1.0"; + version = "0.1.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_shared"; @@ -9986,7 +10000,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10017,12 +10031,12 @@ rec { }; "stackable-telemetry" = rec { crateName = "stackable-telemetry"; - version = "0.6.3"; + version = "0.6.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_telemetry"; @@ -10066,7 +10080,7 @@ rec { { name = "opentelemetry_sdk"; packageId = "opentelemetry_sdk"; - features = [ "rt-tokio" "logs" "rt-tokio" "spec_unstable_logs_enabled" ]; + features = [ "rt-tokio" "logs" "rt-tokio" ]; } { name = "pin-project"; @@ -10074,7 +10088,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10132,7 +10146,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_versioned"; @@ -10166,7 +10180,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-versioned-macros"; @@ -10182,7 +10196,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; procMacro = true; @@ -10250,7 +10264,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_webhook"; @@ -10323,7 +10337,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-certs"; @@ -11414,6 +11428,33 @@ rec { } ]; + }; + "tonic-types" = rec { + crateName = "tonic-types"; + version = "0.14.5"; + edition = "2021"; + sha256 = "16bk1cxi2m0xgaabf98nnj7dn9j16ymkh27jq4s3shjm4a85m1ra"; + libName = "tonic_types"; + authors = [ + "Lucio Franco " + "Rafael Lemos " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + } + ]; + }; "tower" = rec { crateName = "tower"; @@ -11870,9 +11911,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "1z2jjmxbkm1qawlb3bm99x8xwf4g8wjkbcknm9z4fv1w14nqzhhs"; + sha256 = "09nvxy5m7nxmifz4b6szdcyczapp2jcgxcac0jw4ax8klz5n9g5d"; libName = "tracing_opentelemetry"; dependencies = [ { @@ -13992,9 +14033,9 @@ rec { }; "xml" = rec { crateName = "xml"; - version = "1.2.1"; + version = "1.3.0"; edition = "2021"; - sha256 = "0ak4k990faralbli5a0rb8kvwihccb2rp0r94d4azfy94a6lkamq"; + sha256 = "128s58qhq8whrx90zbw8r5algr7lakgbf7mn05jfk234rbjqavv3"; authors = [ "Vladimir Matveev " "Kornel (https://github.com/kornelski)" diff --git a/Cargo.toml b/Cargo.toml index 74c770d8..02c687e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,6 @@ clap = "4.5" const_format = "0.2" futures = "0.3" indoc = "2.0" -java-properties = "2.0" rstest = "0.26" semver = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/crate-hashes.json b/crate-hashes.json index 5564a89e..5b0037c5 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -3,8 +3,8 @@ "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 077ccccd..b9d9d40f 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -12,7 +12,6 @@ publish = false stackable-operator.workspace = true indoc.workspace = true -java-properties.workspace = true anyhow.workspace = true clap.workspace = true const_format.workspace = true diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index 162c9c09..ae92b3c2 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -1,4 +1,3 @@ pub mod command; pub mod jvm; pub mod node_id_hasher; -pub mod writer; diff --git a/rust/operator-binary/src/config/writer.rs b/rust/operator-binary/src/config/writer.rs deleted file mode 100644 index a74babf0..00000000 --- a/rust/operator-binary/src/config/writer.rs +++ /dev/null @@ -1,78 +0,0 @@ -//! Writer for Java `.properties` files. -//! -//! Vendored from the `product-config` crate's `writer` module so the operator no -//! longer depends on `product-config` for rendering. - -use std::io::Write; - -use java_properties::{PropertiesError, PropertiesWriter}; -use snafu::{ResultExt, Snafu}; - -#[derive(Debug, Snafu)] -pub enum PropertiesWriterError { - #[snafu(display("failed to create properties file"))] - Properties { source: PropertiesError }, - - #[snafu(display("failed to convert properties file byte array to UTF-8"))] - FromUtf8 { source: std::string::FromUtf8Error }, -} - -/// Creates a common Java properties file string in the format: -/// `property_1=value_1\nproperty_2=value_2\n`. -pub fn to_java_properties_string<'a, T>(properties: T) -> Result -where - T: Iterator)>, -{ - let mut output = Vec::new(); - write_java_properties(&mut output, properties)?; - String::from_utf8(output).context(FromUtf8Snafu) -} - -/// Writes Java properties to the given writer. A `None` value is written as an -/// empty value (`key=`). -fn write_java_properties<'a, W, T>(writer: W, properties: T) -> Result<(), PropertiesWriterError> -where - W: Write, - T: Iterator)>, -{ - let mut writer = PropertiesWriter::new(writer); - for (k, v) in properties { - let property_value = v.as_deref().unwrap_or_default(); - writer.write(k, property_value).context(PropertiesSnafu)?; - } - writer.flush().context(PropertiesSnafu)?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use super::*; - - fn props(pairs: &[(&str, Option<&str>)]) -> String { - let map: BTreeMap> = pairs - .iter() - .map(|(k, v)| (k.to_string(), v.map(str::to_string))) - .collect(); - to_java_properties_string(map.iter()).unwrap() - } - - #[test] - fn java_properties_renders_key_value() { - assert_eq!(props(&[("a", Some("1")), ("b", Some("2"))]), "a=1\nb=2\n"); - } - - #[test] - fn java_properties_renders_none_as_empty() { - assert_eq!(props(&[("none", None)]), "none=\n"); - } - - #[test] - fn java_properties_escapes_colon_in_value() { - assert_eq!( - props(&[("url", Some("file://this/location/file.abc"))]), - "url=file\\://this/location/file.abc\n" - ); - } -} diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index b018f449..2c1d1767 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -7,10 +7,10 @@ use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, k8s_openapi::api::core::v1::ConfigMap, role_utils::RoleGroupRef, + v2::config_file_writer::{PropertiesWriterError, to_java_properties_string}, }; use crate::{ - config::writer::to_java_properties_string, controller::KAFKA_CONTROLLER_NAME, crd::{ JVM_SECURITY_PROPERTIES_FILE, KafkaPodDescriptor, MetadataManager, @@ -40,7 +40,7 @@ pub enum Error { rolegroup ))] JvmSecurityProperties { - source: crate::config::writer::PropertiesWriterError, + source: PropertiesWriterError, rolegroup: String, }, @@ -56,7 +56,7 @@ pub enum Error { #[snafu(display("failed to serialize config for {rolegroup}"))] SerializeConfig { - source: crate::config::writer::PropertiesWriterError, + source: PropertiesWriterError, rolegroup: RoleGroupRef, }, From fc277503a2863b0ed78d780cc2489336ebebfb91 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 8 Jun 2026 18:01:58 +0200 Subject: [PATCH 09/33] pass validate cluster and rg rather than multiple parameters --- rust/operator-binary/src/controller.rs | 103 +++++++++++------- .../src/controller/build/config_map.rs | 28 ++--- .../src/controller/validate.rs | 41 +------ rust/operator-binary/src/discovery.rs | 11 +- rust/operator-binary/src/resource/listener.rs | 11 +- .../src/resource/statefulset.rs | 31 +++--- 6 files changed, 112 insertions(+), 113 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 8f495905..1193bb77 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,13 +1,13 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha1::KafkaCluster`]. -use std::sync::Arc; +use std::{collections::BTreeMap, sync::Arc}; use const_format::concatcp; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, - commons::rbac::build_rbac_resources, + commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, crd::listener, kube::{ Resource, @@ -32,8 +32,10 @@ mod validate; use crate::{ crd::{ self, APP_NAME, KafkaClusterStatus, OPERATOR_NAME, + authorization::KafkaAuthorizationConfig, listener::get_kafka_listener_config, role::{AnyConfig, KafkaRole}, + security::KafkaTlsSecurity, v1alpha1, }, discovery::{self, build_discovery_configmap}, @@ -208,6 +210,35 @@ impl ReconcilerError for Error { } } +/// The validated cluster. Carries everything the build steps need, resolved once +/// here so downstream code never re-derives it or touches the raw spec. +pub struct ValidatedKafkaCluster { + pub image: ResolvedProductImage, + pub kafka_security: KafkaTlsSecurity, + // DESIGN DECISION: the dereferenced authorization config is folded into the + // validated cluster (read from here downstream). The other dereferenced input, + // the authentication classes, is intentionally NOT stored: it is fully consumed + // here to build `kafka_security`. Alternative: also store the resolved auth + // classes — rejected because nothing downstream needs them beyond kafka_security. + pub authorization_config: Option, + pub role_groups: BTreeMap>, +} + +pub struct ValidatedRoleGroupConfig { + pub merged_config: AnyConfig, + // DESIGN DECISION: overrides are resolved into flat maps HERE rather than stored + // as the typed KeyValueConfigOverrides and resolved in the per-file builders (the + // hdfs-operator pattern). Reason: broker and controller use different override + // struct types (KafkaBrokerConfigOverrides vs KafkaControllerConfigOverrides), so a + // single typed field would require an enum. Resolving here keeps the build/properties + // builders taking plain `BTreeMap`. Alternative: an enum over the two + // override types threaded to builders that call resolved_overrides() — more types for + // no behavioural gain. + pub config_file_overrides: BTreeMap, + pub jvm_security_overrides: BTreeMap, + pub env_overrides: BTreeMap, +} + pub async fn reconcile_kafka( kafka: Arc>, ctx: Arc, @@ -228,15 +259,12 @@ pub async fn reconcile_kafka( .context(DereferenceSnafu)?; // validate (no client required) - let validate::ValidatedKafkaCluster { - authorization_config, - image, - kafka_security, - role_groups, - } = validate::validate(kafka, dereferenced_objects, &ctx.operator_environment) - .context(ValidateClusterSnafu)?; - - let opa_connect = authorization_config + let validated_cluster = + validate::validate(kafka, dereferenced_objects, &ctx.operator_environment) + .context(ValidateClusterSnafu)?; + + let opa_connect = validated_cluster + .authorization_config .as_ref() .map(|auth_config| auth_config.opa_connect.clone()); @@ -251,10 +279,10 @@ pub async fn reconcile_kafka( .context(CreateClusterResourcesSnafu)?; tracing::debug!( - kerberos_enabled = kafka_security.has_kerberos_enabled(), - kerberos_secret_class = ?kafka_security.kerberos_secret_class(), - tls_enabled = kafka_security.tls_enabled(), - tls_client_authentication_class = ?kafka_security.tls_client_authentication_class(), + kerberos_enabled = validated_cluster.kafka_security.has_kerberos_enabled(), + kerberos_secret_class = ?validated_cluster.kafka_security.kerberos_secret_class(), + tls_enabled = validated_cluster.kafka_security.tls_enabled(), + tls_client_authentication_class = ?validated_cluster.kafka_security.tls_client_authentication_class(), "The following security settings are used" ); @@ -280,20 +308,25 @@ pub async fn reconcile_kafka( let mut bootstrap_listeners = Vec::::new(); - for (kafka_role, rg_map) in &role_groups { + for (kafka_role, rg_map) in &validated_cluster.role_groups { for (rolegroup_name, validated_rg) in rg_map { let rolegroup_ref = kafka.rolegroup_ref(kafka_role, rolegroup_name); - let rg_headless_service = - build_rolegroup_headless_service(kafka, &image, &rolegroup_ref, &kafka_security) - .context(BuildServiceSnafu)?; + let rg_headless_service = build_rolegroup_headless_service( + kafka, + &validated_cluster.image, + &rolegroup_ref, + &validated_cluster.kafka_security, + ) + .context(BuildServiceSnafu)?; - let rg_metrics_service = build_rolegroup_metrics_service(kafka, &image, &rolegroup_ref) - .context(BuildServiceSnafu)?; + let rg_metrics_service = + build_rolegroup_metrics_service(kafka, &validated_cluster.image, &rolegroup_ref) + .context(BuildServiceSnafu)?; let kafka_listeners = get_kafka_listener_config( kafka, - &kafka_security, + &validated_cluster.kafka_security, &rolegroup_ref, &client.kubernetes_cluster_info, ) @@ -303,18 +336,15 @@ pub async fn reconcile_kafka( .pod_descriptors( None, &client.kubernetes_cluster_info, - kafka_security.client_port(), + validated_cluster.kafka_security.client_port(), ) .context(BuildPodDescriptorsSnafu)?; let rg_configmap = build::config_map::build_rolegroup_config_map( kafka, - &image, - &kafka_security, + &validated_cluster, &rolegroup_ref, - validated_rg.config_file_overrides.clone(), - validated_rg.jvm_security_overrides.clone(), - &validated_rg.merged_config, + validated_rg, &kafka_listeners, &pod_descriptors, opa_connect.as_deref(), @@ -325,11 +355,9 @@ pub async fn reconcile_kafka( KafkaRole::Broker => build_broker_rolegroup_statefulset( kafka, kafka_role, - &image, + &validated_cluster, &rolegroup_ref, - &validated_rg.env_overrides, - &kafka_security, - &validated_rg.merged_config, + validated_rg, &rbac_sa, &client.kubernetes_cluster_info, ) @@ -337,11 +365,9 @@ pub async fn reconcile_kafka( KafkaRole::Controller => build_controller_rolegroup_statefulset( kafka, kafka_role, - &image, + &validated_cluster, &rolegroup_ref, - &validated_rg.env_overrides, - &kafka_security, - &validated_rg.merged_config, + validated_rg, &rbac_sa, &client.kubernetes_cluster_info, ) @@ -351,8 +377,7 @@ pub async fn reconcile_kafka( if let AnyConfig::Broker(broker_config) = &validated_rg.merged_config { let rg_bootstrap_listener = build_broker_rolegroup_bootstrap_listener( kafka, - &image, - &kafka_security, + &validated_cluster, &rolegroup_ref, broker_config, ) @@ -409,7 +434,7 @@ pub async fn reconcile_kafka( } let discovery_cm = - build_discovery_configmap(kafka, kafka, &image, &kafka_security, &bootstrap_listeners) + build_discovery_configmap(kafka, kafka, validated_cluster, &bootstrap_listeners) .context(BuildDiscoveryConfigSnafu)?; cluster_resources diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 2c1d1767..6d99412c 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -4,20 +4,18 @@ use indoc::formatdoc; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, k8s_openapi::api::core::v1::ConfigMap, role_utils::RoleGroupRef, v2::config_file_writer::{PropertiesWriterError, to_java_properties_string}, }; use crate::{ - controller::KAFKA_CONTROLLER_NAME, + controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster, ValidatedRoleGroupConfig}, crd::{ JVM_SECURITY_PROPERTIES_FILE, KafkaPodDescriptor, MetadataManager, STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, listener::{KafkaListenerConfig, node_address_cmd}, role::AnyConfig, - security::KafkaTlsSecurity, v1alpha1, }, product_logging::extend_role_group_config_map, @@ -74,23 +72,23 @@ pub enum Error { #[allow(clippy::too_many_arguments)] pub fn build_rolegroup_config_map( kafka: &v1alpha1::KafkaCluster, - resolved_product_image: &ResolvedProductImage, - kafka_security: &KafkaTlsSecurity, + validated_cluster: &ValidatedKafkaCluster, rolegroup: &RoleGroupRef, - config_file_overrides: BTreeMap, - jvm_security_overrides: BTreeMap, - merged_config: &AnyConfig, + validated_rg: &ValidatedRoleGroupConfig, listener_config: &KafkaListenerConfig, pod_descriptors: &[KafkaPodDescriptor], opa_connect_string: Option<&str>, ) -> Result { - let kafka_config_file_name = merged_config.config_file_name(); + let kafka_security = &validated_cluster.kafka_security; + let resolved_product_image = &validated_cluster.image; + let kafka_config_file_name = validated_rg.merged_config.config_file_name(); + let config_overrides = validated_rg.config_file_overrides.clone(); let metadata_manager = kafka .effective_metadata_manager() .context(InvalidMetadataManagerSnafu)?; - let kafka_config = match merged_config { + let kafka_config = match &validated_rg.merged_config { AnyConfig::Broker(_) => crate::controller::build::properties::broker_properties::build( kafka_security, listener_config, @@ -102,7 +100,7 @@ pub fn build_rolegroup_config_map( .cluster_config .broker_id_pod_config_map_name .is_some(), - config_file_overrides, + config_overrides, ), AnyConfig::Controller(_) => { crate::controller::build::properties::controller_properties::build( @@ -110,7 +108,7 @@ pub fn build_rolegroup_config_map( listener_config, pod_descriptors, metadata_manager == MetadataManager::KRaft, - config_file_overrides, + config_overrides, ) } } @@ -123,7 +121,9 @@ pub fn build_rolegroup_config_map( .map(|(k, v)| (k, Some(v))) .collect::>(); - let jvm_sec_props: BTreeMap> = jvm_security_overrides + let jvm_sec_props: BTreeMap> = validated_rg + .jvm_security_overrides + .clone() .into_iter() .map(|(k, v)| (k, Some(v))) .collect(); @@ -189,7 +189,7 @@ pub fn build_rolegroup_config_map( extend_role_group_config_map( &resolved_product_image.product_version, rolegroup, - merged_config, + &validated_rg.merged_config, &mut cm_builder, ); diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 72d6fa6a..6c541117 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -6,18 +6,16 @@ use std::collections::BTreeMap; use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - cli::OperatorEnvironmentOptions, - commons::product_image_selection::{self, ResolvedProductImage}, -}; +use stackable_operator::{cli::OperatorEnvironmentOptions, commons::product_image_selection}; use crate::{ - controller::dereference::DereferencedObjects, + controller::{ + ValidatedKafkaCluster, ValidatedRoleGroupConfig, dereference::DereferencedObjects, + }, crd::{ self, CONTAINER_IMAGE_BASE_NAME, authentication::{self}, - authorization::KafkaAuthorizationConfig, - role::{AnyConfig, KafkaRole}, + role::KafkaRole, security::{self, KafkaTlsSecurity}, v1alpha1, }, @@ -45,35 +43,6 @@ pub enum Error { type Result = std::result::Result; -/// The validated cluster. Carries everything the build steps need, resolved once -/// here so downstream code never re-derives it or touches the raw spec. -pub struct ValidatedKafkaCluster { - pub image: ResolvedProductImage, - pub kafka_security: KafkaTlsSecurity, - // DESIGN DECISION: the dereferenced authorization config is folded into the - // validated cluster (read from here downstream). The other dereferenced input, - // the authentication classes, is intentionally NOT stored: it is fully consumed - // here to build `kafka_security`. Alternative: also store the resolved auth - // classes — rejected because nothing downstream needs them beyond kafka_security. - pub authorization_config: Option, - pub role_groups: BTreeMap>, -} - -pub struct ValidatedRoleGroupConfig { - pub merged_config: AnyConfig, - // DESIGN DECISION: overrides are resolved into flat maps HERE rather than stored - // as the typed KeyValueConfigOverrides and resolved in the per-file builders (the - // hdfs-operator pattern). Reason: broker and controller use different override - // struct types (KafkaBrokerConfigOverrides vs KafkaControllerConfigOverrides), so a - // single typed field would require an enum. Resolving here keeps the build/properties - // builders taking plain `BTreeMap`. Alternative: an enum over the two - // override types threaded to builders that call resolved_overrides() — more types for - // no behavioural gain. - pub config_file_overrides: BTreeMap, - pub jvm_security_overrides: BTreeMap, - pub env_overrides: BTreeMap, -} - /// Validates the cluster spec and the dereferenced inputs. pub fn validate( kafka: &v1alpha1::KafkaCluster, diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index e0cc6b36..a978e8dd 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -3,15 +3,14 @@ use std::num::TryFromIntError; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, crd::listener, k8s_openapi::api::core::v1::ConfigMap, kube::{Resource, ResourceExt, runtime::reflector::ObjectRef}, }; use crate::{ - controller::KAFKA_CONTROLLER_NAME, - crd::{role::KafkaRole, security::KafkaTlsSecurity, v1alpha1}, + controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster}, + crd::{role::KafkaRole, v1alpha1}, utils::build_recommended_labels, }; @@ -48,10 +47,12 @@ pub enum Error { pub fn build_discovery_configmap( kafka: &v1alpha1::KafkaCluster, owner: &impl Resource, - resolved_product_image: &ResolvedProductImage, - kafka_security: &KafkaTlsSecurity, + validated_cluster: ValidatedKafkaCluster, listeners: &[listener::v1alpha1::Listener], ) -> Result { + let kafka_security = &validated_cluster.kafka_security; + let resolved_product_image = &validated_cluster.image; + let port_name = if kafka_security.has_kerberos_enabled() { kafka_security.bootstrap_port_name() } else { diff --git a/rust/operator-binary/src/resource/listener.rs b/rust/operator-binary/src/resource/listener.rs index 4afde134..35f360f7 100644 --- a/rust/operator-binary/src/resource/listener.rs +++ b/rust/operator-binary/src/resource/listener.rs @@ -1,11 +1,10 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::meta::ObjectMetaBuilder, commons::product_image_selection::ResolvedProductImage, - crd::listener, role_utils::RoleGroupRef, + builder::meta::ObjectMetaBuilder, crd::listener, role_utils::RoleGroupRef, }; use crate::{ - controller::KAFKA_CONTROLLER_NAME, + controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster}, crd::{role::broker::BrokerConfig, security::KafkaTlsSecurity, v1alpha1}, utils::build_recommended_labels, }; @@ -28,11 +27,13 @@ pub enum Error { // TODO (@NickLarsenNZ): Move shared functionality to stackable-operator pub fn build_broker_rolegroup_bootstrap_listener( kafka: &v1alpha1::KafkaCluster, - resolved_product_image: &ResolvedProductImage, - kafka_security: &KafkaTlsSecurity, + validated_cluster: &ValidatedKafkaCluster, rolegroup: &RoleGroupRef, merged_config: &BrokerConfig, ) -> Result { + let kafka_security = &validated_cluster.kafka_security; + let resolved_product_image = &validated_cluster.image; + Ok(listener::v1alpha1::Listener { metadata: ObjectMetaBuilder::new() .name_and_namespace(kafka) diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index 7f232793..5a433d50 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, ops::Deref}; +use std::ops::Deref; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ @@ -12,7 +12,6 @@ use stackable_operator::{ volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference, VolumeBuilder}, }, }, - commons::product_image_selection::ResolvedProductImage, constants::RESTART_CONTROLLER_ENABLED_LABEL, k8s_openapi::{ DeepMerge, @@ -44,14 +43,14 @@ use crate::{ command::{broker_kafka_container_commands, controller_kafka_container_command}, node_id_hasher::node_id_hash32_offset, }, - controller::KAFKA_CONTROLLER_NAME, + controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster, ValidatedRoleGroupConfig}, crd::{ self, APP_NAME, KAFKA_HEAP_OPTS, LISTENER_BOOTSTRAP_VOLUME_NAME, LISTENER_BROKER_VOLUME_NAME, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, MetadataManager, STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, role::{ - AnyConfig, KAFKA_NODE_ID_OFFSET, KafkaRole, broker::BrokerContainer, + KAFKA_NODE_ID_OFFSET, KafkaRole, broker::BrokerContainer, controller::ControllerContainer, }, security::KafkaTlsSecurity, @@ -165,14 +164,15 @@ pub enum Error { pub fn build_broker_rolegroup_statefulset( kafka: &v1alpha1::KafkaCluster, kafka_role: &KafkaRole, - resolved_product_image: &ResolvedProductImage, + validated_cluster: &ValidatedKafkaCluster, rolegroup_ref: &RoleGroupRef, - env_overrides: &BTreeMap, - kafka_security: &KafkaTlsSecurity, - merged_config: &AnyConfig, + validated_rg: &ValidatedRoleGroupConfig, service_account: &ServiceAccount, cluster_info: &KubernetesClusterInfo, ) -> Result { + let kafka_security = &validated_cluster.kafka_security; + let resolved_product_image = &validated_cluster.image; + let merged_config = &validated_rg.merged_config; let recommended_object_labels = build_recommended_labels( kafka, KAFKA_CONTROLLER_NAME, @@ -245,7 +245,8 @@ pub fn build_broker_rolegroup_statefulset( .context(AddKerberosConfigSnafu)?; } - let mut env = env_overrides + let mut env = validated_rg + .env_overrides .iter() .map(|(k, v)| EnvVar { name: k.clone(), @@ -573,14 +574,15 @@ pub fn build_broker_rolegroup_statefulset( pub fn build_controller_rolegroup_statefulset( kafka: &v1alpha1::KafkaCluster, kafka_role: &KafkaRole, - resolved_product_image: &ResolvedProductImage, + validated_cluster: &ValidatedKafkaCluster, rolegroup_ref: &RoleGroupRef, - env_overrides: &BTreeMap, - kafka_security: &KafkaTlsSecurity, - merged_config: &AnyConfig, + validated_rg: &ValidatedRoleGroupConfig, service_account: &ServiceAccount, cluster_info: &KubernetesClusterInfo, ) -> Result { + let kafka_security = &validated_cluster.kafka_security; + let resolved_product_image = &validated_cluster.image; + let merged_config = &validated_rg.merged_config; let recommended_object_labels = build_recommended_labels( kafka, KAFKA_CONTROLLER_NAME, @@ -597,7 +599,8 @@ pub fn build_controller_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); - let mut env = env_overrides + let mut env = validated_rg + .env_overrides .iter() .map(|(k, v)| EnvVar { name: k.clone(), From ae6a58b315509fbd5c3d486c0fe567558759dbef Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 9 Jun 2026 11:11:56 +0200 Subject: [PATCH 10/33] removed redundant parameter --- rust/operator-binary/src/controller.rs | 5 ++--- .../src/controller/build/config_map.rs | 1 - rust/operator-binary/src/discovery.rs | 11 +++++------ rust/operator-binary/src/resource/statefulset.rs | 2 -- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 1193bb77..3c5ecfee 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -433,9 +433,8 @@ pub async fn reconcile_kafka( } } - let discovery_cm = - build_discovery_configmap(kafka, kafka, validated_cluster, &bootstrap_listeners) - .context(BuildDiscoveryConfigSnafu)?; + let discovery_cm = build_discovery_configmap(kafka, validated_cluster, &bootstrap_listeners) + .context(BuildDiscoveryConfigSnafu)?; cluster_resources .add(client, discovery_cm) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 6d99412c..9228675d 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -69,7 +69,6 @@ pub enum Error { } /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] pub fn build_rolegroup_config_map( kafka: &v1alpha1::KafkaCluster, validated_cluster: &ValidatedKafkaCluster, diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index a978e8dd..98c4516a 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -5,7 +5,7 @@ use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, crd::listener, k8s_openapi::api::core::v1::ConfigMap, - kube::{Resource, ResourceExt, runtime::reflector::ObjectRef}, + kube::{ResourceExt, runtime::reflector::ObjectRef}, }; use crate::{ @@ -45,8 +45,7 @@ pub enum Error { /// Build a discovery [`ConfigMap`] containing information about how to connect to a certain /// [`v1alpha1::KafkaCluster`]. pub fn build_discovery_configmap( - kafka: &v1alpha1::KafkaCluster, - owner: &impl Resource, + owner: &v1alpha1::KafkaCluster, validated_cluster: ValidatedKafkaCluster, listeners: &[listener::v1alpha1::Listener], ) -> Result { @@ -69,14 +68,14 @@ pub fn build_discovery_configmap( ConfigMapBuilder::new() .metadata( ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(owner) .name(owner.name_unchecked()) .ownerreference_from_resource(owner, None, Some(true)) .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { - kafka: ObjectRef::from_obj(kafka), + kafka: ObjectRef::from_obj(owner), })? .with_recommended_labels(&build_recommended_labels( - kafka, + owner, KAFKA_CONTROLLER_NAME, &resolved_product_image.product_version, &KafkaRole::Broker.to_string(), diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index 5a433d50..840382e9 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -160,7 +160,6 @@ pub enum Error { /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding /// [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) from [`build_rolegroup_service`](`crate::resource::service::build_rolegroup_headless_service`). -#[allow(clippy::too_many_arguments)] pub fn build_broker_rolegroup_statefulset( kafka: &v1alpha1::KafkaCluster, kafka_role: &KafkaRole, @@ -570,7 +569,6 @@ pub fn build_broker_rolegroup_statefulset( } /// The controller rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -#[allow(clippy::too_many_arguments)] pub fn build_controller_rolegroup_statefulset( kafka: &v1alpha1::KafkaCluster, kafka_role: &KafkaRole, From 7706e88d66add0e56ffae69335eaa75a70e3c541 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 9 Jun 2026 11:19:21 +0200 Subject: [PATCH 11/33] push opa connect string calc down to function --- rust/operator-binary/src/controller.rs | 6 ------ rust/operator-binary/src/controller/build/config_map.rs | 8 ++++++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 3c5ecfee..1cbf1a71 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -263,11 +263,6 @@ pub async fn reconcile_kafka( validate::validate(kafka, dereferenced_objects, &ctx.operator_environment) .context(ValidateClusterSnafu)?; - let opa_connect = validated_cluster - .authorization_config - .as_ref() - .map(|auth_config| auth_config.opa_connect.clone()); - let mut cluster_resources = ClusterResources::new( APP_NAME, OPERATOR_NAME, @@ -347,7 +342,6 @@ pub async fn reconcile_kafka( validated_rg, &kafka_listeners, &pod_descriptors, - opa_connect.as_deref(), ) .context(BuildConfigMapSnafu)?; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 9228675d..7f715ad5 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -76,13 +76,17 @@ pub fn build_rolegroup_config_map( validated_rg: &ValidatedRoleGroupConfig, listener_config: &KafkaListenerConfig, pod_descriptors: &[KafkaPodDescriptor], - opa_connect_string: Option<&str>, ) -> Result { let kafka_security = &validated_cluster.kafka_security; let resolved_product_image = &validated_cluster.image; let kafka_config_file_name = validated_rg.merged_config.config_file_name(); let config_overrides = validated_rg.config_file_overrides.clone(); + let opa_connect = validated_cluster + .authorization_config + .as_ref() + .map(|auth_config| auth_config.opa_connect.clone()); + let metadata_manager = kafka .effective_metadata_manager() .context(InvalidMetadataManagerSnafu)?; @@ -92,7 +96,7 @@ pub fn build_rolegroup_config_map( kafka_security, listener_config, pod_descriptors, - opa_connect_string, + opa_connect.as_deref(), metadata_manager == MetadataManager::KRaft, kafka .spec From 4eb786b984bf3794d23b8b7524516ade660752a8 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 9 Jun 2026 11:46:31 +0200 Subject: [PATCH 12/33] push pod_descriptors calc down to build_rolegroup_config_map --- rust/operator-binary/src/controller.rs | 15 ++------------- .../src/controller/build/config_map.rs | 16 +++++++++++++--- .../src/controller/dereference.rs | 4 +++- rust/operator-binary/src/controller/validate.rs | 1 + 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 1cbf1a71..896e12e4 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -22,6 +22,7 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, + utils::cluster_info::KubernetesClusterInfo, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -65,9 +66,6 @@ pub enum Error { #[snafu(display("failed to validate cluster"))] ValidateCluster { source: validate::Error }, - #[snafu(display("failed to build pod descriptors"))] - BuildPodDescriptors { source: crate::crd::Error }, - #[snafu(display("invalid kafka listeners"))] InvalidKafkaListeners { source: crate::crd::listener::KafkaListenerError, @@ -205,7 +203,6 @@ impl ReconcilerError for Error { Error::BuildService { .. } => None, Error::BuildListener { .. } => None, Error::InvalidKafkaListeners { .. } => None, - Error::BuildPodDescriptors { .. } => None, } } } @@ -222,6 +219,7 @@ pub struct ValidatedKafkaCluster { // classes — rejected because nothing downstream needs them beyond kafka_security. pub authorization_config: Option, pub role_groups: BTreeMap>, + pub kubernetes_cluster_info: KubernetesClusterInfo, } pub struct ValidatedRoleGroupConfig { @@ -327,21 +325,12 @@ pub async fn reconcile_kafka( ) .context(InvalidKafkaListenersSnafu)?; - let pod_descriptors = kafka - .pod_descriptors( - None, - &client.kubernetes_cluster_info, - validated_cluster.kafka_security.client_port(), - ) - .context(BuildPodDescriptorsSnafu)?; - let rg_configmap = build::config_map::build_rolegroup_config_map( kafka, &validated_cluster, &rolegroup_ref, validated_rg, &kafka_listeners, - &pod_descriptors, ) .context(BuildConfigMapSnafu)?; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 7f715ad5..d248b5ba 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -12,8 +12,8 @@ use stackable_operator::{ use crate::{ controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster, ValidatedRoleGroupConfig}, crd::{ - JVM_SECURITY_PROPERTIES_FILE, KafkaPodDescriptor, MetadataManager, - STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, + JVM_SECURITY_PROPERTIES_FILE, MetadataManager, STACKABLE_LISTENER_BOOTSTRAP_DIR, + STACKABLE_LISTENER_BROKER_DIR, listener::{KafkaListenerConfig, node_address_cmd}, role::AnyConfig, v1alpha1, @@ -66,6 +66,9 @@ pub enum Error { #[snafu(display("failed to build jaas configuration file for {rolegroup}"))] BuildJaasConfig { rolegroup: String }, + + #[snafu(display("failed to build pod descriptors"))] + BuildPodDescriptors { source: crate::crd::Error }, } /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator @@ -75,7 +78,6 @@ pub fn build_rolegroup_config_map( rolegroup: &RoleGroupRef, validated_rg: &ValidatedRoleGroupConfig, listener_config: &KafkaListenerConfig, - pod_descriptors: &[KafkaPodDescriptor], ) -> Result { let kafka_security = &validated_cluster.kafka_security; let resolved_product_image = &validated_cluster.image; @@ -87,6 +89,14 @@ pub fn build_rolegroup_config_map( .as_ref() .map(|auth_config| auth_config.opa_connect.clone()); + let pod_descriptors = &kafka + .pod_descriptors( + None, + &validated_cluster.kubernetes_cluster_info, + validated_cluster.kafka_security.client_port(), + ) + .context(BuildPodDescriptorsSnafu)?; + let metadata_manager = kafka .effective_metadata_manager() .context(InvalidMetadataManagerSnafu)?; diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs index b7a22107..c90d1258 100644 --- a/rust/operator-binary/src/controller/dereference.rs +++ b/rust/operator-binary/src/controller/dereference.rs @@ -9,7 +9,7 @@ //! and stays here as-is. use snafu::{ResultExt, Snafu}; -use stackable_operator::client::Client; +use stackable_operator::{client::Client, utils::cluster_info::KubernetesClusterInfo}; use crate::crd::{ authentication::{self, ResolvedAuthenticationClasses}, @@ -33,6 +33,7 @@ type Result = std::result::Result; pub struct DereferencedObjects { pub authentication_classes: ResolvedAuthenticationClasses, pub authorization_config: Option, + pub kubernetes_cluster_info: KubernetesClusterInfo, } /// Fetches all Kubernetes objects referenced from the [`v1alpha1::KafkaCluster`] spec. @@ -59,5 +60,6 @@ pub async fn dereference( Ok(DereferencedObjects { authentication_classes, authorization_config, + kubernetes_cluster_info: client.kubernetes_cluster_info.clone(), }) } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 6c541117..c03f381c 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -146,6 +146,7 @@ pub fn validate( kafka_security, authorization_config: dereferenced_objects.authorization_config, role_groups, + kubernetes_cluster_info: dereferenced_objects.kubernetes_cluster_info, }) } From af0ceba907103852fc0f2fdbc293f3e8170aac12 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 9 Jun 2026 12:48:44 +0200 Subject: [PATCH 13/33] move resolution of metadata_manager and pod descriptors to validate stage --- rust/operator-binary/src/controller.rs | 6 ++-- .../src/controller/build/config_map.rs | 36 +++++++------------ .../build/properties/broker_properties.rs | 10 +++--- .../build/properties/controller_properties.rs | 11 +++--- .../src/controller/build/properties/mod.rs | 19 ++-------- .../src/controller/validate.rs | 21 ++++++++++- 6 files changed, 45 insertions(+), 58 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 896e12e4..64e9a8b9 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -22,7 +22,6 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - utils::cluster_info::KubernetesClusterInfo, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -32,7 +31,7 @@ mod validate; use crate::{ crd::{ - self, APP_NAME, KafkaClusterStatus, OPERATOR_NAME, + self, APP_NAME, KafkaClusterStatus, KafkaPodDescriptor, MetadataManager, OPERATOR_NAME, authorization::KafkaAuthorizationConfig, listener::get_kafka_listener_config, role::{AnyConfig, KafkaRole}, @@ -219,7 +218,8 @@ pub struct ValidatedKafkaCluster { // classes — rejected because nothing downstream needs them beyond kafka_security. pub authorization_config: Option, pub role_groups: BTreeMap>, - pub kubernetes_cluster_info: KubernetesClusterInfo, + pub pod_descriptors: Vec, + pub metadata_manager: MetadataManager, } pub struct ValidatedRoleGroupConfig { diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index d248b5ba..c9dcd501 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -58,17 +58,14 @@ pub enum Error { rolegroup: RoleGroupRef, }, - #[snafu(display("failed to build properties for {rolegroup}"))] - BuildProperties { - source: crate::controller::build::properties::Error, - rolegroup: RoleGroupRef, - }, - #[snafu(display("failed to build jaas configuration file for {rolegroup}"))] BuildJaasConfig { rolegroup: String }, #[snafu(display("failed to build pod descriptors"))] BuildPodDescriptors { source: crate::crd::Error }, + + #[snafu(display("no Kraft controllers found to build"))] + NoKraftControllersFound, } /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator @@ -89,25 +86,19 @@ pub fn build_rolegroup_config_map( .as_ref() .map(|auth_config| auth_config.opa_connect.clone()); - let pod_descriptors = &kafka - .pod_descriptors( - None, - &validated_cluster.kubernetes_cluster_info, - validated_cluster.kafka_security.client_port(), - ) - .context(BuildPodDescriptorsSnafu)?; + let kraft_mode = validated_cluster.metadata_manager == MetadataManager::KRaft; - let metadata_manager = kafka - .effective_metadata_manager() - .context(InvalidMetadataManagerSnafu)?; + if kraft_mode && validated_cluster.pod_descriptors.is_empty() { + return NoKraftControllersFoundSnafu.fail(); + } let kafka_config = match &validated_rg.merged_config { AnyConfig::Broker(_) => crate::controller::build::properties::broker_properties::build( kafka_security, listener_config, - pod_descriptors, + &validated_cluster.pod_descriptors, opa_connect.as_deref(), - metadata_manager == MetadataManager::KRaft, + kraft_mode, kafka .spec .cluster_config @@ -119,15 +110,12 @@ pub fn build_rolegroup_config_map( crate::controller::build::properties::controller_properties::build( kafka_security, listener_config, - pod_descriptors, - metadata_manager == MetadataManager::KRaft, + &validated_cluster.pod_descriptors, + kraft_mode, config_overrides, ) } - } - .with_context(|_| BuildPropertiesSnafu { - rolegroup: rolegroup.clone(), - })?; + }; let kafka_config = kafka_config .into_iter() diff --git a/rust/operator-binary/src/controller/build/properties/broker_properties.rs b/rust/operator-binary/src/controller/build/properties/broker_properties.rs index 5d43ab79..44840363 100644 --- a/rust/operator-binary/src/controller/build/properties/broker_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/broker_properties.rs @@ -1,8 +1,6 @@ use std::collections::BTreeMap; -use snafu::OptionExt; - -use super::{Error, NoKraftControllersFoundSnafu, kraft_controllers}; +use super::kraft_controllers; use crate::{ crd::{ KafkaPodDescriptor, @@ -25,7 +23,7 @@ pub fn build( kraft_mode: bool, disable_broker_id_generation: bool, overrides: BTreeMap, -) -> Result, Error> { +) -> BTreeMap { let kraft_controllers = kraft_controllers(pod_descriptors); let mut result = BTreeMap::from([ @@ -49,7 +47,7 @@ pub fn build( ]); if kraft_mode { - let kraft_controllers = kraft_controllers.context(NoKraftControllersFoundSnafu)?; + let kraft_controllers = kraft_controllers.join(","); // Running in KRaft mode result.extend([ @@ -114,5 +112,5 @@ pub fn build( result.extend(graceful_shutdown_config_properties()); result.extend(overrides); - Ok(result) + result } diff --git a/rust/operator-binary/src/controller/build/properties/controller_properties.rs b/rust/operator-binary/src/controller/build/properties/controller_properties.rs index 6a8172a8..ba825aec 100644 --- a/rust/operator-binary/src/controller/build/properties/controller_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/controller_properties.rs @@ -1,8 +1,6 @@ use std::collections::BTreeMap; -use snafu::OptionExt; - -use super::{Error, NoKraftControllersFoundSnafu, kraft_controllers}; +use super::kraft_controllers; use crate::{ crd::{ KafkaPodDescriptor, @@ -22,9 +20,8 @@ pub fn build( pod_descriptors: &[KafkaPodDescriptor], kraft_mode: bool, overrides: BTreeMap, -) -> Result, Error> { - let kraft_controllers = - kraft_controllers(pod_descriptors).context(NoKraftControllersFoundSnafu)?; +) -> BTreeMap { + let kraft_controllers = kraft_controllers(pod_descriptors).join(","); let mut result = BTreeMap::from([ ( @@ -72,5 +69,5 @@ pub fn build( result.extend(graceful_shutdown_config_properties()); result.extend(overrides); - Ok(result) + result } diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index adcdcaae..116f5c2f 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -3,18 +3,10 @@ pub mod broker_properties; pub mod controller_properties; -use snafu::Snafu; - use crate::crd::{KafkaPodDescriptor, role::KafkaRole}; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("no Kraft controllers found to build"))] - NoKraftControllersFound, -} - -pub(crate) fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Option { - let result = pod_descriptors +pub(crate) fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Vec { + pod_descriptors .iter() .filter(|pd| pd.role == KafkaRole::Controller.to_string()) .map(|desc| { @@ -25,11 +17,4 @@ pub(crate) fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Optio ) }) .collect::>() - .join(","); - - if result.is_empty() { - None - } else { - Some(result) - } } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index c03f381c..669febd9 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -39,6 +39,12 @@ pub enum Error { #[snafu(display("failed to resolve merged config for rolegroup"))] ResolveMergedConfig { source: crate::crd::role::Error }, + + #[snafu(display("failed to build pod descriptors"))] + BuildPodDescriptors { source: crate::crd::Error }, + + #[snafu(display("invalid metadata manager"))] + InvalidMetadataManager { source: crate::crd::Error }, } type Result = std::result::Result; @@ -141,12 +147,25 @@ pub fn validate( role_groups.insert(KafkaRole::Controller, controller_groups); } + let pod_descriptors = kafka + .pod_descriptors( + None, + &dereferenced_objects.kubernetes_cluster_info, + kafka_security.client_port(), + ) + .context(BuildPodDescriptorsSnafu)?; + + let metadata_manager = kafka + .effective_metadata_manager() + .context(InvalidMetadataManagerSnafu)?; + Ok(ValidatedKafkaCluster { image, kafka_security, authorization_config: dereferenced_objects.authorization_config, role_groups, - kubernetes_cluster_info: dereferenced_objects.kubernetes_cluster_info, + pod_descriptors, + metadata_manager, }) } From b6a01a0d5198a410ebbfc0f56cd1591ee37eafb3 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 9 Jun 2026 15:08:17 +0200 Subject: [PATCH 14/33] add config map data explicitly --- .../src/controller/build/config_map.rs | 10 +- rust/operator-binary/src/product_logging.rs | 132 +++++++++--------- 2 files changed, 76 insertions(+), 66 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index c9dcd501..ffd7ef03 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -18,7 +18,7 @@ use crate::{ role::AnyConfig, v1alpha1, }, - product_logging::extend_role_group_config_map, + product_logging::role_group_config_map_data, utils::build_recommended_labels, }; @@ -187,12 +187,16 @@ pub fn build_rolegroup_config_map( tracing::debug!(?kafka_config, "Applied kafka config"); tracing::debug!(?jvm_sec_props, "Applied JVM config"); - extend_role_group_config_map( + let config_data = role_group_config_map_data( &resolved_product_image.product_version, rolegroup, &validated_rg.merged_config, - &mut cm_builder, ); + for (file_name, data) in config_data { + if let Some(data) = data { + cm_builder.add_data(file_name, data); + } + } cm_builder .build() diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 8336f5f7..40780435 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -1,7 +1,6 @@ -use std::{borrow::Cow, fmt::Display}; +use std::{borrow::Cow, collections::BTreeMap, fmt::Display}; use stackable_operator::{ - builder::configmap::ConfigMapBuilder, memory::{BinaryMultiple, MemoryQuantity}, product_logging::{ self, @@ -45,36 +44,43 @@ pub fn kafka_log_opts_env_var() -> String { "KAFKA_LOG4J_OPTS".to_string() } -/// Extend the role group ConfigMap with logging and Vector configurations -pub fn extend_role_group_config_map( +/// Get the role group ConfigMap data with logging and Vector configurations +pub fn role_group_config_map_data( product_version: &str, rolegroup: &RoleGroupRef, merged_config: &AnyConfig, - cm_builder: &mut ConfigMapBuilder, -) { +) -> BTreeMap> { let container_name = match merged_config { AnyConfig::Broker(_) => BrokerContainer::Kafka.to_string(), AnyConfig::Controller(_) => ControllerContainer::Kafka.to_string(), }; + let mut configs: BTreeMap> = BTreeMap::new(); + // Starting with Kafka 4.0, log4j2 is used instead of log4j. match product_version.starts_with("3.") { - true => add_log4j_config_if_automatic( - cm_builder, - Some(merged_config.kafka_logging()), - LOG4J_CONFIG_FILE, - container_name, - KAFKA_LOG4J_FILE, - MAX_KAFKA_LOG_FILES_SIZE, - ), - false => add_log4j2_config_if_automatic( - cm_builder, - Some(merged_config.kafka_logging()), - LOG4J2_CONFIG_FILE, - container_name, - KAFKA_LOG4J2_FILE, - MAX_KAFKA_LOG_FILES_SIZE, - ), + true => { + configs.insert( + LOG4J_CONFIG_FILE.to_string(), + log4j_config_if_automatic( + Some(merged_config.kafka_logging()), + container_name, + KAFKA_LOG4J_FILE, + MAX_KAFKA_LOG_FILES_SIZE, + ), + ); + } + false => { + configs.insert( + LOG4J2_CONFIG_FILE.to_string(), + log4j2_config_if_automatic( + Some(merged_config.kafka_logging()), + container_name, + KAFKA_LOG4J2_FILE, + MAX_KAFKA_LOG_FILES_SIZE, + ), + ); + } } let vector_log_config = merged_config.vector_logging(); @@ -88,65 +94,65 @@ pub fn extend_role_group_config_map( }; if merged_config.vector_logging_enabled() { - cm_builder.add_data( - product_logging::framework::VECTOR_CONFIG_FILE, - product_logging::framework::create_vector_config(rolegroup, vector_log_config), + configs.insert( + product_logging::framework::VECTOR_CONFIG_FILE.to_string(), + Some(product_logging::framework::create_vector_config( + rolegroup, + vector_log_config, + )), ); } + configs } -fn add_log4j_config_if_automatic( - cm_builder: &mut ConfigMapBuilder, +fn log4j_config_if_automatic( log_config: Option>, - log_config_file: &str, container_name: impl Display, log_file: &str, max_log_file_size: MemoryQuantity, -) { - if let Some(ContainerLogConfig { +) -> Option { + let config = if let Some(ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Automatic(log_config)), }) = log_config.as_deref() { - cm_builder.add_data( - log_config_file, - product_logging::framework::create_log4j_config( - &format!("{STACKABLE_LOG_DIR}/{container_name}"), - log_file, - max_log_file_size - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32, - CONSOLE_CONVERSION_PATTERN_LOG4J, - log_config, - ), - ); - } + Some(product_logging::framework::create_log4j_config( + &format!("{STACKABLE_LOG_DIR}/{container_name}"), + log_file, + max_log_file_size + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, + CONSOLE_CONVERSION_PATTERN_LOG4J, + log_config, + )) + } else { + None + }; + config } -fn add_log4j2_config_if_automatic( - cm_builder: &mut ConfigMapBuilder, +fn log4j2_config_if_automatic( log_config: Option>, - log_config_file: &str, container_name: impl Display, log_file: &str, max_log_file_size: MemoryQuantity, -) { - if let Some(ContainerLogConfig { +) -> Option { + let config = if let Some(ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Automatic(log_config)), }) = log_config.as_deref() { - cm_builder.add_data( - log_config_file, - product_logging::framework::create_log4j2_config( - &format!("{STACKABLE_LOG_DIR}/{container_name}",), - log_file, - max_log_file_size - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32, - CONSOLE_CONVERSION_PATTERN_LOG4J2, - log_config, - ), - ); - } + Some(product_logging::framework::create_log4j2_config( + &format!("{STACKABLE_LOG_DIR}/{container_name}",), + log_file, + max_log_file_size + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, + CONSOLE_CONVERSION_PATTERN_LOG4J2, + log_config, + )) + } else { + None + }; + config } From 8e48f00393ad86e7e2c212312323f9d5b22e4f7f Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 9 Jun 2026 15:24:23 +0200 Subject: [PATCH 15/33] use merge instead of extend with KafkaBrokerConfigOverrides --- extra/crds.yaml | 16 +- .../src/controller/validate.rs | 217 ++++++++++-------- rust/operator-binary/src/crd/mod.rs | 42 ++-- 3 files changed, 144 insertions(+), 131 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index 9a83de14..471aadbf 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -543,23 +543,23 @@ spec: additionalProperties: nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: @@ -1162,23 +1162,23 @@ spec: additionalProperties: nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: @@ -1792,23 +1792,23 @@ spec: additionalProperties: nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: @@ -2243,23 +2243,23 @@ spec: additionalProperties: nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 669febd9..ceaaad1a 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -6,7 +6,12 @@ use std::collections::BTreeMap; use snafu::{ResultExt, Snafu}; -use stackable_operator::{cli::OperatorEnvironmentOptions, commons::product_image_selection}; +use stackable_operator::{ + cli::OperatorEnvironmentOptions, + commons::product_image_selection, + config::merge::{Merge, merge}, + v2::config_overrides::KeyValueConfigOverrides, +}; use crate::{ controller::{ @@ -169,12 +174,30 @@ pub fn validate( }) } -// DESIGN DECISION: role-group overrides are merged role-level first, then role-group -// extended on top so role-group wins — identical to the precedent product-config used. -// We read the v2 KeyValueConfigOverrides `.overrides` map and BTreeMap::extend rather -// than using its `Merge` impl, because plain extend reproduces the old behaviour exactly -// (last-writer-wins per key) and avoids depending on Merge semantics. Alternative: -// KeyValueConfigOverrides::merge — equivalent here but an unnecessary semantic dependency. +/// Merge role-group overrides over the role-level overrides (role-group wins per key) via the +/// `Merge` impl derived on the override structs. +/// +/// NOTE on semantics: `Merge` treats a role-group `null` value as "inherit the role-level value", +/// *not* "unset it". This differs from `main`'s product-config layering, which `.extend()`ed the +/// maps so a role-group `null` *removed* a role-level key. The `tests` module has a worked +/// example of the difference. +fn merge_role_group_overrides(role: &O, role_group: Option<&O>) -> O { + match role_group { + Some(role_group) => merge(role_group.clone(), role), + None => role.clone(), + } +} + +/// Flatten resolved key/value overrides into a plain map, dropping entries whose value is +/// unset (`null`). +fn flatten_overrides(overrides: KeyValueConfigOverrides) -> BTreeMap { + overrides + .overrides + .into_iter() + .filter_map(|(key, value)| value.map(|value| (key, value))) + .collect() +} + fn collect_broker_role_group_overrides( kafka: &v1alpha1::KafkaCluster, broker_role: &crate::crd::BrokerRole, @@ -184,47 +207,15 @@ fn collect_broker_role_group_overrides( BTreeMap, BTreeMap, ) { - // --- broker.properties overrides --- - let role_broker_overrides: BTreeMap> = broker_role - .config - .config_overrides - .broker_properties - .as_ref() - .map(|o| o.overrides.clone()) - .unwrap_or_default(); - let rg_broker_overrides: BTreeMap> = broker_role - .role_groups - .get(rolegroup_name) - .and_then(|rg| rg.config.config_overrides.broker_properties.as_ref()) - .map(|o| o.overrides.clone()) - .unwrap_or_default(); - let mut merged_broker = role_broker_overrides; - merged_broker.extend(rg_broker_overrides); - let config_file_overrides: BTreeMap = merged_broker - .into_iter() - .filter_map(|(k, v)| v.map(|v| (k, v))) - .collect(); - - // --- security.properties overrides --- - let role_security_overrides: BTreeMap> = broker_role - .config - .config_overrides - .security_properties - .as_ref() - .map(|o| o.overrides.clone()) - .unwrap_or_default(); - let rg_security_overrides: BTreeMap> = broker_role - .role_groups - .get(rolegroup_name) - .and_then(|rg| rg.config.config_overrides.security_properties.as_ref()) - .map(|o| o.overrides.clone()) - .unwrap_or_default(); - let mut merged_security = role_security_overrides; - merged_security.extend(rg_security_overrides); - let jvm_security_overrides: BTreeMap = merged_security - .into_iter() - .filter_map(|(k, v)| v.map(|v| (k, v))) - .collect(); + let merged_overrides = merge_role_group_overrides( + &broker_role.config.config_overrides, + broker_role + .role_groups + .get(rolegroup_name) + .map(|rg| &rg.config.config_overrides), + ); + let config_file_overrides = flatten_overrides(merged_overrides.broker_properties); + let jvm_security_overrides = flatten_overrides(merged_overrides.security_properties); // --- env overrides --- // DESIGN DECISION: KAFKA_CLUSTER_ID is injected first, then the user env overrides @@ -252,12 +243,6 @@ fn collect_broker_role_group_overrides( (config_file_overrides, jvm_security_overrides, env_overrides) } -// DESIGN DECISION: role-group overrides are merged role-level first, then role-group -// extended on top so role-group wins — identical to the precedent product-config used. -// We read the v2 KeyValueConfigOverrides `.overrides` map and BTreeMap::extend rather -// than using its `Merge` impl, because plain extend reproduces the old behaviour exactly -// (last-writer-wins per key) and avoids depending on Merge semantics. Alternative: -// KeyValueConfigOverrides::merge — equivalent here but an unnecessary semantic dependency. fn collect_controller_role_group_overrides( kafka: &v1alpha1::KafkaCluster, controller_role: &crate::crd::ControllerRole, @@ -267,47 +252,15 @@ fn collect_controller_role_group_overrides( BTreeMap, BTreeMap, ) { - // --- controller.properties overrides --- - let role_controller_overrides: BTreeMap> = controller_role - .config - .config_overrides - .controller_properties - .as_ref() - .map(|o| o.overrides.clone()) - .unwrap_or_default(); - let rg_controller_overrides: BTreeMap> = controller_role - .role_groups - .get(rolegroup_name) - .and_then(|rg| rg.config.config_overrides.controller_properties.as_ref()) - .map(|o| o.overrides.clone()) - .unwrap_or_default(); - let mut merged_controller = role_controller_overrides; - merged_controller.extend(rg_controller_overrides); - let config_file_overrides: BTreeMap = merged_controller - .into_iter() - .filter_map(|(k, v)| v.map(|v| (k, v))) - .collect(); - - // --- security.properties overrides --- - let role_security_overrides: BTreeMap> = controller_role - .config - .config_overrides - .security_properties - .as_ref() - .map(|o| o.overrides.clone()) - .unwrap_or_default(); - let rg_security_overrides: BTreeMap> = controller_role - .role_groups - .get(rolegroup_name) - .and_then(|rg| rg.config.config_overrides.security_properties.as_ref()) - .map(|o| o.overrides.clone()) - .unwrap_or_default(); - let mut merged_security = role_security_overrides; - merged_security.extend(rg_security_overrides); - let jvm_security_overrides: BTreeMap = merged_security - .into_iter() - .filter_map(|(k, v)| v.map(|v| (k, v))) - .collect(); + let merged_overrides = merge_role_group_overrides( + &controller_role.config.config_overrides, + controller_role + .role_groups + .get(rolegroup_name) + .map(|rg| &rg.config.config_overrides), + ); + let config_file_overrides = flatten_overrides(merged_overrides.controller_properties); + let jvm_security_overrides = flatten_overrides(merged_overrides.security_properties); // --- env overrides --- // DESIGN DECISION: KAFKA_CLUSTER_ID is injected first, then the user env overrides @@ -335,3 +288,77 @@ fn collect_controller_role_group_overrides( (config_file_overrides, jvm_security_overrides, env_overrides) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; + + use super::{flatten_overrides, merge_role_group_overrides}; + + /// Build a `KeyValueConfigOverrides` from `(key, value)` pairs, where a `None` value + /// represents an explicit `null` (unset) in the CRD. + fn overrides(pairs: &[(&str, Option<&str>)]) -> KeyValueConfigOverrides { + KeyValueConfigOverrides { + overrides: pairs + .iter() + .map(|(key, value)| (key.to_string(), value.map(str::to_string))) + .collect(), + } + } + + /// Run the full role/role-group resolution (merge then flatten) for a single config file. + fn resolve( + role: KeyValueConfigOverrides, + role_group: Option, + ) -> BTreeMap { + flatten_overrides(merge_role_group_overrides(&role, role_group.as_ref())) + } + + #[test] + fn role_group_value_wins_over_role() { + let role = overrides(&[("a", Some("role")), ("b", Some("role-only"))]); + let role_group = overrides(&[("a", Some("rg"))]); + + let merged = resolve(role, Some(role_group)); + + assert_eq!( + merged, + BTreeMap::from([ + ("a".to_string(), "rg".to_string()), // role-group wins for shared keys + ("b".to_string(), "role-only".to_string()), // role-only keys are kept + ]) + ); + } + + /// Illustrates the key consequence of using `Merge` (rather than `.extend()`, as `main`'s + /// product-config did): a role-group `null` is treated as "inherit", so the role-level value + /// is *kept* — it does NOT unset the key. Under the old `.extend()` behaviour this same input + /// would have removed `a` entirely. + #[test] + fn role_group_null_inherits_role_value_rather_than_unsetting_it() { + let role = overrides(&[("a", Some("role"))]); + let role_group = overrides(&[("a", None)]); // explicit `null` at the more specific level + + let merged = resolve(role, Some(role_group)); + + assert_eq!( + merged, + BTreeMap::from([("a".to_string(), "role".to_string())]), + "a role-group `null` should inherit the role-level value under Merge semantics" + ); + } + + #[test] + fn without_a_role_group_role_values_are_kept_and_nulls_dropped() { + let role = overrides(&[("a", Some("role")), ("b", None)]); + + let merged = resolve(role, None); + + assert_eq!( + merged, + BTreeMap::from([("a".to_string(), "role".to_string())]) + ); + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index fa5b7498..132c5f8b 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -16,6 +16,7 @@ use stackable_operator::{ cluster_operation::ClusterOperation, networking::DomainName, product_image_selection::ProductImage, }, + config::merge::Merge, deep_merger::ObjectOverrides, kube::{CustomResource, runtime::reflector::ObjectRef}, role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, @@ -240,42 +241,27 @@ pub mod versioned { pub broker_id_pod_config_map_name: Option, } - // Uses the v2 KeyValueConfigOverrides (Merge-capable, `nullable` values) to match - // trino/hdfs. Resolution into flat maps happens in controller/validate.rs. - #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + // Uses the v2 KeyValueConfigOverrides (`nullable` values) to match trino/hdfs. + // Derives `Merge` so role/role-group overrides combine via the shared merge logic; + // resolution into flat maps happens in controller/validate.rs. + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Merge, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KafkaBrokerConfigOverrides { - #[serde( - default, - rename = "broker.properties", - skip_serializing_if = "Option::is_none" - )] - pub broker_properties: Option, + #[serde(default, rename = "broker.properties")] + pub broker_properties: KeyValueConfigOverrides, - #[serde( - default, - rename = "security.properties", - skip_serializing_if = "Option::is_none" - )] - pub security_properties: Option, + #[serde(default, rename = "security.properties")] + pub security_properties: KeyValueConfigOverrides, } - #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Merge, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KafkaControllerConfigOverrides { - #[serde( - default, - rename = "controller.properties", - skip_serializing_if = "Option::is_none" - )] - pub controller_properties: Option, + #[serde(default, rename = "controller.properties")] + pub controller_properties: KeyValueConfigOverrides, - #[serde( - default, - rename = "security.properties", - skip_serializing_if = "Option::is_none" - )] - pub security_properties: Option, + #[serde(default, rename = "security.properties")] + pub security_properties: KeyValueConfigOverrides, } } From d792d747de5740ddb22f34bcb291f52c9bd65484 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 9 Jun 2026 15:26:42 +0200 Subject: [PATCH 16/33] updated changelog --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d34355e..75204042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,10 +19,10 @@ All notable changes to this project will be documented in this file. - Bump `stackable-operator` to 0.111.1 and snafu to 0.9 ([#960], [#961]). - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#968]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#971]). -- Removed the product-config based configuration validation. Config and environment overrides are - now merged directly from the CRD into the validated cluster, the Java-properties writer is - vendored locally, and the `product-config` crate dependency is dropped. The `--product-config` - CLI flag is now a no-op ([#976]). +- BREAKING: Removed product-config machinery which is a breaking change in terms of configuration. + Users relying on the product-config `properties.yaml` file have to set these properties via the CRD. + Config and environment overrides are now merged directly from the CRD into the validated cluster. + The `--product-config` CLI flag is now a no-op ([#976]). [#953]: https://github.com/stackabletech/kafka-operator/pull/953 [#960]: https://github.com/stackabletech/kafka-operator/pull/960 From 94102ab34e568a9abf9020d775e67a96ef0c965a Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 9 Jun 2026 15:47:43 +0200 Subject: [PATCH 17/33] extend test comparison --- .../src/controller/validate.rs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index ceaaad1a..7afe7056 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -341,12 +341,28 @@ mod tests { let role = overrides(&[("a", Some("role"))]); let role_group = overrides(&[("a", None)]); // explicit `null` at the more specific level - let merged = resolve(role, Some(role_group)); + // For contrast: `main`'s product-config used a blind `.extend()`, so the role-group + // `null` overwrote the role value and the key was then dropped — i.e. unset entirely. + let old_extend_behaviour: BTreeMap = { + let mut combined = role.overrides.clone(); + combined.extend(role_group.overrides.clone()); + combined + .into_iter() + .filter_map(|(key, value)| value.map(|value| (key, value))) + .collect() + }; + assert!( + old_extend_behaviour.is_empty(), + "under the old `.extend()` behaviour the role-group `null` unsets `a`" + ); + // What we do now (Merge): the role-group `null` means "inherit", so the role-level + // value is kept rather than unset. + let merged = resolve(role, Some(role_group)); assert_eq!( merged, BTreeMap::from([("a".to_string(), "role".to_string())]), - "a role-group `null` should inherit the role-level value under Merge semantics" + "under Merge semantics the role-group `null` inherits the role-level value" ); } From e33b155d9154be629c862698194837917d139e68 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 9 Jun 2026 16:54:59 +0200 Subject: [PATCH 18/33] unnecessary let binding --- rust/operator-binary/src/product_logging.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 40780435..eb0dc55a 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -137,7 +137,7 @@ fn log4j2_config_if_automatic( log_file: &str, max_log_file_size: MemoryQuantity, ) -> Option { - let config = if let Some(ContainerLogConfig { + if let Some(ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Automatic(log_config)), }) = log_config.as_deref() { @@ -153,6 +153,5 @@ fn log4j2_config_if_automatic( )) } else { None - }; - config + } } From 3f09a7b9b1a521dc2154aab998cdfb26bd4d0519 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 9 Jun 2026 17:38:05 +0200 Subject: [PATCH 19/33] regenerate nix --- Cargo.nix | 18 +++++++++--------- crate-hashes.json | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index e70182ee..0de574b7 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4853,7 +4853,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "k8s_version"; authors = [ @@ -9536,7 +9536,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_certs"; authors = [ @@ -9727,7 +9727,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_operator"; authors = [ @@ -9921,7 +9921,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -9956,7 +9956,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_shared"; authors = [ @@ -10037,7 +10037,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_telemetry"; authors = [ @@ -10147,7 +10147,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_versioned"; authors = [ @@ -10197,7 +10197,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10265,7 +10265,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 5b0037c5..deac3bf4 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From 3b9390e0eb1f24c6b8be7d6aed77e3fe23dffad2 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 10 Jun 2026 16:31:02 +0200 Subject: [PATCH 20/33] bump op-rs branch and fix clippy warning --- Cargo.lock | 18 +++--- Cargo.nix | 36 +++++------ crate-hashes.json | 18 +++--- .../src/controller/build/config_map.rs | 24 ++----- .../src/controller/validate.rs | 64 +++---------------- rust/operator-binary/src/crd/mod.rs | 2 +- rust/operator-binary/src/product_logging.rs | 5 +- 7 files changed, 55 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1390fe52..92a70017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1518,7 +1518,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "darling", "regex", @@ -2897,7 +2897,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "const-oid", "ecdsa", @@ -2942,7 +2942,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "base64", "clap", @@ -2986,7 +2986,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "darling", "proc-macro2", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "jiff", "k8s-openapi", @@ -3014,7 +3014,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "axum", "clap", @@ -3038,7 +3038,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "kube", "schemars", @@ -3052,7 +3052,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "convert_case", "convert_case_extras", @@ -3070,7 +3070,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 0de574b7..43e426e5 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4852,8 +4852,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "k8s_version"; authors = [ @@ -9535,8 +9535,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_certs"; authors = [ @@ -9726,8 +9726,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_operator"; authors = [ @@ -9920,8 +9920,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -9955,8 +9955,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_shared"; authors = [ @@ -10036,8 +10036,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_telemetry"; authors = [ @@ -10146,8 +10146,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_versioned"; authors = [ @@ -10196,8 +10196,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10264,8 +10264,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index deac3bf4..c9a6e6a9 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index ffd7ef03..5941ac62 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use indoc::formatdoc; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -117,17 +115,7 @@ pub fn build_rolegroup_config_map( } }; - let kafka_config = kafka_config - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect::>(); - - let jvm_sec_props: BTreeMap> = validated_rg - .jvm_security_overrides - .clone() - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect(); + let jvm_sec_props = &validated_rg.jvm_security_overrides; let mut cm_builder = ConfigMapBuilder::new(); cm_builder @@ -149,11 +137,11 @@ pub fn build_rolegroup_config_map( ) .add_data( kafka_config_file_name, - to_java_properties_string(kafka_config.iter().map(|(k, v)| (k, v))).with_context( - |_| SerializeConfigSnafu { + to_java_properties_string(kafka_config.iter()).with_context(|_| { + SerializeConfigSnafu { rolegroup: rolegroup.clone(), - }, - )?, + } + })?, ) .add_data( JVM_SECURITY_PROPERTIES_FILE, @@ -169,7 +157,7 @@ pub fn build_rolegroup_config_map( kafka_security .client_properties() .iter() - .map(|(k, v)| (k, v)), + .filter_map(|(k, v)| v.as_ref().map(|v| (k, v))), ) .with_context(|_| JvmSecurityPropertiesSnafu { rolegroup: rolegroup.role_group.clone(), diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 7afe7056..a6e8cee3 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -176,11 +176,6 @@ pub fn validate( /// Merge role-group overrides over the role-level overrides (role-group wins per key) via the /// `Merge` impl derived on the override structs. -/// -/// NOTE on semantics: `Merge` treats a role-group `null` value as "inherit the role-level value", -/// *not* "unset it". This differs from `main`'s product-config layering, which `.extend()`ed the -/// maps so a role-group `null` *removed* a role-level key. The `tests` module has a worked -/// example of the difference. fn merge_role_group_overrides(role: &O, role_group: Option<&O>) -> O { match role_group { Some(role_group) => merge(role_group.clone(), role), @@ -188,14 +183,10 @@ fn merge_role_group_overrides(role: &O, role_group: Option<&O> } } -/// Flatten resolved key/value overrides into a plain map, dropping entries whose value is -/// unset (`null`). +/// Flatten resolved key/value overrides into a plain map. operator-rs #1219 made the override +/// values plain `String`, so there is no longer any `null`/unset entry to drop. fn flatten_overrides(overrides: KeyValueConfigOverrides) -> BTreeMap { - overrides - .overrides - .into_iter() - .filter_map(|(key, value)| value.map(|value| (key, value))) - .collect() + overrides.overrides } fn collect_broker_role_group_overrides( @@ -297,13 +288,12 @@ mod tests { use super::{flatten_overrides, merge_role_group_overrides}; - /// Build a `KeyValueConfigOverrides` from `(key, value)` pairs, where a `None` value - /// represents an explicit `null` (unset) in the CRD. - fn overrides(pairs: &[(&str, Option<&str>)]) -> KeyValueConfigOverrides { + /// Build a `KeyValueConfigOverrides` from `(key, value)` pairs. + fn overrides(pairs: &[(&str, &str)]) -> KeyValueConfigOverrides { KeyValueConfigOverrides { overrides: pairs .iter() - .map(|(key, value)| (key.to_string(), value.map(str::to_string))) + .map(|(key, value)| (key.to_string(), value.to_string())) .collect(), } } @@ -318,8 +308,8 @@ mod tests { #[test] fn role_group_value_wins_over_role() { - let role = overrides(&[("a", Some("role")), ("b", Some("role-only"))]); - let role_group = overrides(&[("a", Some("rg"))]); + let role = overrides(&[("a", "role"), ("b", "role-only")]); + let role_group = overrides(&[("a", "rg")]); let merged = resolve(role, Some(role_group)); @@ -332,43 +322,9 @@ mod tests { ); } - /// Illustrates the key consequence of using `Merge` (rather than `.extend()`, as `main`'s - /// product-config did): a role-group `null` is treated as "inherit", so the role-level value - /// is *kept* — it does NOT unset the key. Under the old `.extend()` behaviour this same input - /// would have removed `a` entirely. #[test] - fn role_group_null_inherits_role_value_rather_than_unsetting_it() { - let role = overrides(&[("a", Some("role"))]); - let role_group = overrides(&[("a", None)]); // explicit `null` at the more specific level - - // For contrast: `main`'s product-config used a blind `.extend()`, so the role-group - // `null` overwrote the role value and the key was then dropped — i.e. unset entirely. - let old_extend_behaviour: BTreeMap = { - let mut combined = role.overrides.clone(); - combined.extend(role_group.overrides.clone()); - combined - .into_iter() - .filter_map(|(key, value)| value.map(|value| (key, value))) - .collect() - }; - assert!( - old_extend_behaviour.is_empty(), - "under the old `.extend()` behaviour the role-group `null` unsets `a`" - ); - - // What we do now (Merge): the role-group `null` means "inherit", so the role-level - // value is kept rather than unset. - let merged = resolve(role, Some(role_group)); - assert_eq!( - merged, - BTreeMap::from([("a".to_string(), "role".to_string())]), - "under Merge semantics the role-group `null` inherits the role-level value" - ); - } - - #[test] - fn without_a_role_group_role_values_are_kept_and_nulls_dropped() { - let role = overrides(&[("a", Some("role")), ("b", None)]); + fn without_a_role_group_role_values_are_kept() { + let role = overrides(&[("a", "role")]); let merged = resolve(role, None); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 132c5f8b..e69c7a4c 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -241,7 +241,7 @@ pub mod versioned { pub broker_id_pod_config_map_name: Option, } - // Uses the v2 KeyValueConfigOverrides (`nullable` values) to match trino/hdfs. + // Uses the v2 KeyValueConfigOverrides (plain string values) to match trino/hdfs. // Derives `Merge` so role/role-group overrides combine via the shared merge logic; // resolution into flat maps happens in controller/validate.rs. #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Merge, PartialEq, Serialize)] diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index eb0dc55a..a16d0148 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -111,7 +111,7 @@ fn log4j_config_if_automatic( log_file: &str, max_log_file_size: MemoryQuantity, ) -> Option { - let config = if let Some(ContainerLogConfig { + if let Some(ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Automatic(log_config)), }) = log_config.as_deref() { @@ -127,8 +127,7 @@ fn log4j_config_if_automatic( )) } else { None - }; - config + } } fn log4j2_config_if_automatic( From 38f278ca1e3b787b8f603fffbe254ba8537c2f42 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 10 Jun 2026 16:38:31 +0200 Subject: [PATCH 21/33] regenerate charts --- extra/crds.yaml | 56 +++++++------------------------------------------ 1 file changed, 8 insertions(+), 48 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index 471aadbf..ea71c1c4 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -541,25 +541,15 @@ spec: properties: broker.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -1160,25 +1150,15 @@ spec: properties: broker.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -1790,25 +1770,15 @@ spec: properties: controller.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -2241,25 +2211,15 @@ spec: properties: controller.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: From 662e9a14e0d1ba993bb6624cf151fbdc9ce73770 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 10 Jun 2026 17:23:05 +0200 Subject: [PATCH 22/33] thread through name, namespace, uid via ObjectMeta --- rust/operator-binary/src/controller.rs | 102 ++++++++++++++++-- .../src/controller/build/config_map.rs | 6 +- .../src/{ => controller/build}/discovery.rs | 24 +++-- .../src/controller/build/mod.rs | 1 + .../src/controller/validate.rs | 48 +++++++-- rust/operator-binary/src/main.rs | 1 - rust/operator-binary/src/resource/listener.rs | 6 +- rust/operator-binary/src/resource/service.rs | 22 ++-- .../src/resource/statefulset.rs | 18 ++-- rust/operator-binary/src/utils.rs | 13 ++- 10 files changed, 181 insertions(+), 60 deletions(-) rename rust/operator-binary/src/{ => controller/build}/discovery.rs (83%) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 64e9a8b9..ec00b969 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,6 +1,6 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha1::KafkaCluster`]. -use std::{collections::BTreeMap, sync::Arc}; +use std::{borrow::Cow, collections::BTreeMap, sync::Arc}; use const_format::concatcp; use snafu::{ResultExt, Snafu}; @@ -11,7 +11,7 @@ use stackable_operator::{ crd::listener, kube::{ Resource, - api::DynamicObject, + api::{DynamicObject, ObjectMeta}, core::{DeserializeGuard, error_boundary}, runtime::{controller::Action, reflector::ObjectRef}, }, @@ -22,6 +22,10 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, + v2::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -38,7 +42,6 @@ use crate::{ security::KafkaTlsSecurity, v1alpha1, }, - discovery::{self, build_discovery_configmap}, operations::pdb::add_pdbs, resource::{ listener::build_broker_rolegroup_bootstrap_listener, @@ -94,7 +97,7 @@ pub enum Error { }, #[snafu(display("failed to build discovery ConfigMap"))] - BuildDiscoveryConfig { source: discovery::Error }, + BuildDiscoveryConfig { source: build::discovery::Error }, #[snafu(display("failed to apply discovery ConfigMap"))] ApplyDiscoveryConfig { @@ -208,7 +211,17 @@ impl ReconcilerError for Error { /// The validated cluster. Carries everything the build steps need, resolved once /// here so downstream code never re-derives it or touches the raw spec. +/// +/// The cluster identity (`name`, `namespace`, `uid`) is captured here so that owner +/// references for child objects can be built straight from this struct (via its +/// [`Resource`] impl) without threading the raw [`v1alpha1::KafkaCluster`] around. +/// This mirrors the hive-/opensearch-operator's `ValidatedCluster`. pub struct ValidatedKafkaCluster { + /// `ObjectMeta` carrying `name`, `namespace` and `uid`, so this struct can act as the + /// owner [`Resource`] for child objects. + metadata: ObjectMeta, + pub name: ClusterName, + pub namespace: NamespaceName, pub image: ResolvedProductImage, pub kafka_security: KafkaTlsSecurity, // DESIGN DECISION: the dereferenced authorization config is folded into the @@ -222,6 +235,69 @@ pub struct ValidatedKafkaCluster { pub metadata_manager: MetadataManager, } +impl ValidatedKafkaCluster { + #[allow(clippy::too_many_arguments)] + pub fn new( + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + image: ResolvedProductImage, + kafka_security: KafkaTlsSecurity, + authorization_config: Option, + role_groups: BTreeMap>, + pod_descriptors: Vec, + metadata_manager: MetadataManager, + ) -> Self { + Self { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }, + name, + namespace, + image, + kafka_security, + authorization_config, + role_groups, + pod_descriptors, + metadata_manager, + } + } +} + +/// Lets [`ValidatedKafkaCluster`] act as the owner [`Resource`] for child objects, so owner +/// references are built from it (via the captured `metadata`) rather than the raw CR. +impl Resource for ValidatedKafkaCluster { + type DynamicType = ::DynamicType; + type Scope = ::Scope; + + fn kind(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::kind(dt) + } + + fn group(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::group(dt) + } + + fn version(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::version(dt) + } + + fn plural(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::plural(dt) + } + + fn meta(&self) -> &ObjectMeta { + &self.metadata + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} + pub struct ValidatedRoleGroupConfig { pub merged_config: AnyConfig, // DESIGN DECISION: overrides are resolved into flat maps HERE rather than stored @@ -265,7 +341,7 @@ pub async fn reconcile_kafka( APP_NAME, OPERATOR_NAME, KAFKA_CONTROLLER_NAME, - &kafka.object_ref(&()), + &validated_cluster.object_ref(&()), ClusterResourceApplyStrategy::from(&kafka.spec.cluster_operation), &kafka.spec.object_overrides, ) @@ -306,16 +382,19 @@ pub async fn reconcile_kafka( let rolegroup_ref = kafka.rolegroup_ref(kafka_role, rolegroup_name); let rg_headless_service = build_rolegroup_headless_service( - kafka, + &validated_cluster, &validated_cluster.image, &rolegroup_ref, &validated_cluster.kafka_security, ) .context(BuildServiceSnafu)?; - let rg_metrics_service = - build_rolegroup_metrics_service(kafka, &validated_cluster.image, &rolegroup_ref) - .context(BuildServiceSnafu)?; + let rg_metrics_service = build_rolegroup_metrics_service( + &validated_cluster, + &validated_cluster.image, + &rolegroup_ref, + ) + .context(BuildServiceSnafu)?; let kafka_listeners = get_kafka_listener_config( kafka, @@ -416,8 +495,9 @@ pub async fn reconcile_kafka( } } - let discovery_cm = build_discovery_configmap(kafka, validated_cluster, &bootstrap_listeners) - .context(BuildDiscoveryConfigSnafu)?; + let discovery_cm = + build::discovery::build_discovery_configmap(&validated_cluster, &bootstrap_listeners) + .context(BuildDiscoveryConfigSnafu)?; cluster_resources .add(client, discovery_cm) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 5941ac62..0af0fcf8 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -121,12 +121,12 @@ pub fn build_rolegroup_config_map( cm_builder .metadata( ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(rolegroup.object_name()) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup.role, diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs similarity index 83% rename from rust/operator-binary/src/discovery.rs rename to rust/operator-binary/src/controller/build/discovery.rs index cf19f32e..60e30940 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -5,7 +5,7 @@ use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, crd::listener, k8s_openapi::api::core::v1::ConfigMap, - kube::{ResourceExt, runtime::reflector::ObjectRef}, + kube::runtime::reflector::ObjectRef, }; use crate::{ @@ -22,9 +22,6 @@ pub enum Error { kafka: ObjectRef, }, - #[snafu(display("object has no name associated"))] - NoName, - #[snafu(display("could not find service port with name {}", port_name))] NoServicePort { port_name: String }, @@ -45,8 +42,7 @@ pub enum Error { /// Build a discovery [`ConfigMap`] containing information about how to connect to a certain /// [`v1alpha1::KafkaCluster`]. pub fn build_discovery_configmap( - owner: &v1alpha1::KafkaCluster, - validated_cluster: ValidatedKafkaCluster, + validated_cluster: &ValidatedKafkaCluster, listeners: &[listener::v1alpha1::Listener], ) -> Result { let kafka_security = &validated_cluster.kafka_security; @@ -68,14 +64,14 @@ pub fn build_discovery_configmap( ConfigMapBuilder::new() .metadata( ObjectMetaBuilder::new() - .name_and_namespace(owner) - .name(owner.name_unchecked()) - .ownerreference_from_resource(owner, None, Some(true)) + .name_and_namespace(validated_cluster) + .name(validated_cluster.name.to_string()) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { - kafka: ObjectRef::from_obj(owner), + kafka: cluster_object_ref(validated_cluster), })? .with_recommended_labels(&build_recommended_labels( - owner, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.product_version, &KafkaRole::Broker.to_string(), @@ -89,6 +85,12 @@ pub fn build_discovery_configmap( .context(BuildConfigMapSnafu) } +/// An [`ObjectRef`] to the owning cluster, built from the validated identity — used only for +/// error context. +fn cluster_object_ref(cluster: &ValidatedKafkaCluster) -> ObjectRef { + ObjectRef::new(cluster.name.as_ref()).within(cluster.namespace.as_ref()) +} + fn listener_hosts( listeners: &[listener::v1alpha1::Listener], port_name: &str, diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index b8c4c422..0cbab809 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -1,4 +1,5 @@ //! Builders that assemble Kubernetes resources for kafka rolegroups. pub mod config_map; +pub mod discovery; pub mod properties; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index a6e8cee3..85c2e844 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -3,14 +3,21 @@ //! Synchronously validates inputs that don't require a Kubernetes client. Produces //! [`ValidatedKafkaCluster`], consumed by the rest of `reconcile_kafka`. -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr}; -use snafu::{ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection, config::merge::{Merge, merge}, - v2::config_overrides::KeyValueConfigOverrides, + kube::ResourceExt, + v2::{ + config_overrides::KeyValueConfigOverrides, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, + }, }; use crate::{ @@ -50,6 +57,27 @@ pub enum Error { #[snafu(display("invalid metadata manager"))] InvalidMetadataManager { source: crate::crd::Error }, + + #[snafu(display("invalid cluster name"))] + InvalidClusterName { + source: stackable_operator::v2::macros::attributed_string_type::Error, + }, + + #[snafu(display("object defines no namespace"))] + ObjectHasNoNamespace, + + #[snafu(display("invalid cluster namespace"))] + InvalidNamespace { + source: stackable_operator::v2::macros::attributed_string_type::Error, + }, + + #[snafu(display("object has no uid"))] + ObjectHasNoUid, + + #[snafu(display("invalid cluster uid"))] + InvalidUid { + source: stackable_operator::v2::macros::attributed_string_type::Error, + }, } type Result = std::result::Result; @@ -164,14 +192,22 @@ pub fn validate( .effective_metadata_manager() .context(InvalidMetadataManagerSnafu)?; - Ok(ValidatedKafkaCluster { + let name = ClusterName::from_str(&kafka.name_any()).context(InvalidClusterNameSnafu)?; + let namespace = NamespaceName::from_str(&kafka.namespace().context(ObjectHasNoNamespaceSnafu)?) + .context(InvalidNamespaceSnafu)?; + let uid = Uid::from_str(&kafka.uid().context(ObjectHasNoUidSnafu)?).context(InvalidUidSnafu)?; + + Ok(ValidatedKafkaCluster::new( + name, + namespace, + uid, image, kafka_security, - authorization_config: dereferenced_objects.authorization_config, + dereferenced_objects.authorization_config, role_groups, pod_descriptors, metadata_manager, - }) + )) } /// Merge role-group overrides over the role-level overrides (role-group wins per key) via the diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 25362c54..a967294a 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -43,7 +43,6 @@ use crate::{ mod config; mod controller; mod crd; -mod discovery; mod kerberos; mod operations; mod product_logging; diff --git a/rust/operator-binary/src/resource/listener.rs b/rust/operator-binary/src/resource/listener.rs index 35f360f7..3161ed0e 100644 --- a/rust/operator-binary/src/resource/listener.rs +++ b/rust/operator-binary/src/resource/listener.rs @@ -36,12 +36,12 @@ pub fn build_broker_rolegroup_bootstrap_listener( Ok(listener::v1alpha1::Listener { metadata: ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(kafka.bootstrap_service_name(rolegroup)) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup.role, diff --git a/rust/operator-binary/src/resource/service.rs b/rust/operator-binary/src/resource/service.rs index f430fc8e..bc2fdbfe 100644 --- a/rust/operator-binary/src/resource/service.rs +++ b/rust/operator-binary/src/resource/service.rs @@ -8,7 +8,7 @@ use stackable_operator::{ }; use crate::{ - controller::KAFKA_CONTROLLER_NAME, + controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster}, crd::{APP_NAME, METRICS_PORT, METRICS_PORT_NAME, security::KafkaTlsSecurity, v1alpha1}, utils::build_recommended_labels, }; @@ -35,19 +35,19 @@ pub enum Error { /// /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. pub fn build_rolegroup_headless_service( - kafka: &v1alpha1::KafkaCluster, + validated_cluster: &ValidatedKafkaCluster, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, kafka_security: &KafkaTlsSecurity, ) -> Result { Ok(Service { metadata: ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(rolegroup.rolegroup_headless_service_name()) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup.role, @@ -60,7 +60,7 @@ pub fn build_rolegroup_headless_service( ports: Some(headless_ports(kafka_security)), selector: Some( Labels::role_group_selector( - kafka, + validated_cluster, APP_NAME, &rolegroup.role, &rolegroup.role_group, @@ -77,18 +77,18 @@ pub fn build_rolegroup_headless_service( /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label pub fn build_rolegroup_metrics_service( - kafka: &v1alpha1::KafkaCluster, + validated_cluster: &ValidatedKafkaCluster, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, ) -> Result { let metrics_service = Service { metadata: ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(rolegroup.rolegroup_metrics_service_name()) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup.role, @@ -105,7 +105,7 @@ pub fn build_rolegroup_metrics_service( ports: Some(metrics_ports()), selector: Some( Labels::role_group_selector( - kafka, + validated_cluster, APP_NAME, &rolegroup.role, &rolegroup.role_group, diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index 840382e9..3212c650 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -173,7 +173,7 @@ pub fn build_broker_rolegroup_statefulset( let resolved_product_image = &validated_cluster.image; let merged_config = &validated_rg.merged_config; let recommended_object_labels = build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup_ref.role, @@ -183,7 +183,7 @@ pub fn build_broker_rolegroup_statefulset( Labels::recommended(&recommended_object_labels).context(LabelBuildSnafu)?; // Used for PVC templates that cannot be modified once they are deployed let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, // A version value is required, and we do want to use the "recommended" format for the other desired labels "none", @@ -526,12 +526,12 @@ pub fn build_broker_rolegroup_statefulset( Ok(StatefulSet { metadata: ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup_ref.role, @@ -582,7 +582,7 @@ pub fn build_controller_rolegroup_statefulset( let resolved_product_image = &validated_cluster.image; let merged_config = &validated_rg.merged_config; let recommended_object_labels = build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup_ref.role, @@ -852,12 +852,12 @@ pub fn build_controller_rolegroup_statefulset( Ok(StatefulSet { metadata: ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup_ref.role, diff --git a/rust/operator-binary/src/utils.rs b/rust/operator-binary/src/utils.rs index 7abbafff..1db2241a 100644 --- a/rust/operator-binary/src/utils.rs +++ b/rust/operator-binary/src/utils.rs @@ -1,15 +1,18 @@ use stackable_operator::kvp::ObjectLabels; -use crate::crd::{APP_NAME, OPERATOR_NAME, v1alpha1}; +use crate::crd::{APP_NAME, OPERATOR_NAME}; -/// Build recommended values for labels -pub fn build_recommended_labels<'a>( - owner: &'a v1alpha1::KafkaCluster, +/// Build recommended values for labels. +/// +/// Generic over the owner `T` so the owner can be either the raw `KafkaCluster` or the +/// `ValidatedKafkaCluster` (which also implements `Resource`). +pub fn build_recommended_labels<'a, T>( + owner: &'a T, controller_name: &'a str, app_version: &'a str, role: &'a str, role_group: &'a str, -) -> ObjectLabels<'a, v1alpha1::KafkaCluster> { +) -> ObjectLabels<'a, T> { ObjectLabels { owner, app_name: APP_NAME, From 6c2336112540a19c83a0a2dbfec456f48c64fde8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 19:02:41 +0200 Subject: [PATCH 23/33] refactor: introduce ValidatedClusterConfig --- rust/operator-binary/src/controller.rs | 62 +++++++++---------- .../src/controller/build/config_map.rs | 15 ++--- .../src/controller/build/discovery.rs | 8 +-- .../src/controller/validate.rs | 41 ++++++------ rust/operator-binary/src/resource/listener.rs | 6 +- rust/operator-binary/src/resource/service.rs | 6 +- .../src/resource/statefulset.rs | 10 +-- rust/operator-binary/src/utils.rs | 2 +- 8 files changed, 75 insertions(+), 75 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index ec00b969..142c77e1 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -209,6 +209,8 @@ impl ReconcilerError for Error { } } +pub type RoleGroupName = String; + /// The validated cluster. Carries everything the build steps need, resolved once /// here so downstream code never re-derives it or touches the raw spec. /// @@ -216,37 +218,25 @@ impl ReconcilerError for Error { /// references for child objects can be built straight from this struct (via its /// [`Resource`] impl) without threading the raw [`v1alpha1::KafkaCluster`] around. /// This mirrors the hive-/opensearch-operator's `ValidatedCluster`. -pub struct ValidatedKafkaCluster { +pub struct ValidatedCluster { /// `ObjectMeta` carrying `name`, `namespace` and `uid`, so this struct can act as the /// owner [`Resource`] for child objects. metadata: ObjectMeta, pub name: ClusterName, pub namespace: NamespaceName, pub image: ResolvedProductImage, - pub kafka_security: KafkaTlsSecurity, - // DESIGN DECISION: the dereferenced authorization config is folded into the - // validated cluster (read from here downstream). The other dereferenced input, - // the authentication classes, is intentionally NOT stored: it is fully consumed - // here to build `kafka_security`. Alternative: also store the resolved auth - // classes — rejected because nothing downstream needs them beyond kafka_security. - pub authorization_config: Option, - pub role_groups: BTreeMap>, - pub pod_descriptors: Vec, - pub metadata_manager: MetadataManager, + pub cluster_config: ValidatedClusterConfig, + pub role_group_configs: BTreeMap>, } -impl ValidatedKafkaCluster { - #[allow(clippy::too_many_arguments)] +impl ValidatedCluster { pub fn new( name: ClusterName, namespace: NamespaceName, uid: Uid, image: ResolvedProductImage, - kafka_security: KafkaTlsSecurity, - authorization_config: Option, - role_groups: BTreeMap>, - pod_descriptors: Vec, - metadata_manager: MetadataManager, + cluster_config: ValidatedClusterConfig, + role_group_configs: BTreeMap>, ) -> Self { Self { metadata: ObjectMeta { @@ -258,18 +248,26 @@ impl ValidatedKafkaCluster { name, namespace, image, - kafka_security, - authorization_config, - role_groups, - pod_descriptors, - metadata_manager, + cluster_config, + role_group_configs, } } } -/// Lets [`ValidatedKafkaCluster`] act as the owner [`Resource`] for child objects, so owner +/// Cluster-wide settings resolved during validation and dereferencing. +/// +/// Everything the build steps need is resolved here so they never have to read the +/// raw [`v1alpha1::KafkaCluster`] spec. +pub struct ValidatedClusterConfig { + pub kafka_security: KafkaTlsSecurity, + pub authorization_config: Option, + pub pod_descriptors: Vec, + pub metadata_manager: MetadataManager, +} + +/// Lets [`ValidatedCluster`] act as the owner [`Resource`] for child objects, so owner /// references are built from it (via the captured `metadata`) rather than the raw CR. -impl Resource for ValidatedKafkaCluster { +impl Resource for ValidatedCluster { type DynamicType = ::DynamicType; type Scope = ::Scope; @@ -348,10 +346,10 @@ pub async fn reconcile_kafka( .context(CreateClusterResourcesSnafu)?; tracing::debug!( - kerberos_enabled = validated_cluster.kafka_security.has_kerberos_enabled(), - kerberos_secret_class = ?validated_cluster.kafka_security.kerberos_secret_class(), - tls_enabled = validated_cluster.kafka_security.tls_enabled(), - tls_client_authentication_class = ?validated_cluster.kafka_security.tls_client_authentication_class(), + kerberos_enabled = validated_cluster.cluster_config.kafka_security.has_kerberos_enabled(), + kerberos_secret_class = ?validated_cluster.cluster_config.kafka_security.kerberos_secret_class(), + tls_enabled = validated_cluster.cluster_config.kafka_security.tls_enabled(), + tls_client_authentication_class = ?validated_cluster.cluster_config.kafka_security.tls_client_authentication_class(), "The following security settings are used" ); @@ -377,7 +375,7 @@ pub async fn reconcile_kafka( let mut bootstrap_listeners = Vec::::new(); - for (kafka_role, rg_map) in &validated_cluster.role_groups { + for (kafka_role, rg_map) in &validated_cluster.role_group_configs { for (rolegroup_name, validated_rg) in rg_map { let rolegroup_ref = kafka.rolegroup_ref(kafka_role, rolegroup_name); @@ -385,7 +383,7 @@ pub async fn reconcile_kafka( &validated_cluster, &validated_cluster.image, &rolegroup_ref, - &validated_cluster.kafka_security, + &validated_cluster.cluster_config.kafka_security, ) .context(BuildServiceSnafu)?; @@ -398,7 +396,7 @@ pub async fn reconcile_kafka( let kafka_listeners = get_kafka_listener_config( kafka, - &validated_cluster.kafka_security, + &validated_cluster.cluster_config.kafka_security, &rolegroup_ref, &client.kubernetes_cluster_info, ) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 0af0fcf8..207caa3a 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -8,7 +8,7 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster, ValidatedRoleGroupConfig}, + controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster, ValidatedRoleGroupConfig}, crd::{ JVM_SECURITY_PROPERTIES_FILE, MetadataManager, STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, @@ -69,24 +69,25 @@ pub enum Error { /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator pub fn build_rolegroup_config_map( kafka: &v1alpha1::KafkaCluster, - validated_cluster: &ValidatedKafkaCluster, + validated_cluster: &ValidatedCluster, rolegroup: &RoleGroupRef, validated_rg: &ValidatedRoleGroupConfig, listener_config: &KafkaListenerConfig, ) -> Result { - let kafka_security = &validated_cluster.kafka_security; + let kafka_security = &validated_cluster.cluster_config.kafka_security; let resolved_product_image = &validated_cluster.image; let kafka_config_file_name = validated_rg.merged_config.config_file_name(); let config_overrides = validated_rg.config_file_overrides.clone(); let opa_connect = validated_cluster + .cluster_config .authorization_config .as_ref() .map(|auth_config| auth_config.opa_connect.clone()); - let kraft_mode = validated_cluster.metadata_manager == MetadataManager::KRaft; + let kraft_mode = validated_cluster.cluster_config.metadata_manager == MetadataManager::KRaft; - if kraft_mode && validated_cluster.pod_descriptors.is_empty() { + if kraft_mode && validated_cluster.cluster_config.pod_descriptors.is_empty() { return NoKraftControllersFoundSnafu.fail(); } @@ -94,7 +95,7 @@ pub fn build_rolegroup_config_map( AnyConfig::Broker(_) => crate::controller::build::properties::broker_properties::build( kafka_security, listener_config, - &validated_cluster.pod_descriptors, + &validated_cluster.cluster_config.pod_descriptors, opa_connect.as_deref(), kraft_mode, kafka @@ -108,7 +109,7 @@ pub fn build_rolegroup_config_map( crate::controller::build::properties::controller_properties::build( kafka_security, listener_config, - &validated_cluster.pod_descriptors, + &validated_cluster.cluster_config.pod_descriptors, kraft_mode, config_overrides, ) diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs index 60e30940..10ac621c 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -9,7 +9,7 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster}, + controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster}, crd::{role::KafkaRole, v1alpha1}, utils::build_recommended_labels, }; @@ -42,10 +42,10 @@ pub enum Error { /// Build a discovery [`ConfigMap`] containing information about how to connect to a certain /// [`v1alpha1::KafkaCluster`]. pub fn build_discovery_configmap( - validated_cluster: &ValidatedKafkaCluster, + validated_cluster: &ValidatedCluster, listeners: &[listener::v1alpha1::Listener], ) -> Result { - let kafka_security = &validated_cluster.kafka_security; + let kafka_security = &validated_cluster.cluster_config.kafka_security; let resolved_product_image = &validated_cluster.image; let port_name = if kafka_security.has_kerberos_enabled() { @@ -87,7 +87,7 @@ pub fn build_discovery_configmap( /// An [`ObjectRef`] to the owning cluster, built from the validated identity — used only for /// error context. -fn cluster_object_ref(cluster: &ValidatedKafkaCluster) -> ObjectRef { +fn cluster_object_ref(cluster: &ValidatedCluster) -> ObjectRef { ObjectRef::new(cluster.name.as_ref()).within(cluster.namespace.as_ref()) } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 85c2e844..228156b1 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,7 +1,7 @@ //! The validate step in the KafkaCluster controller. //! //! Synchronously validates inputs that don't require a Kubernetes client. Produces -//! [`ValidatedKafkaCluster`], consumed by the rest of `reconcile_kafka`. +//! [`ValidatedCluster`], consumed by the rest of `reconcile_kafka`. use std::{collections::BTreeMap, str::FromStr}; @@ -22,7 +22,8 @@ use stackable_operator::{ use crate::{ controller::{ - ValidatedKafkaCluster, ValidatedRoleGroupConfig, dereference::DereferencedObjects, + RoleGroupName, ValidatedCluster, ValidatedClusterConfig, ValidatedRoleGroupConfig, + dereference::DereferencedObjects, }, crd::{ self, CONTAINER_IMAGE_BASE_NAME, @@ -87,7 +88,7 @@ pub fn validate( kafka: &v1alpha1::KafkaCluster, dereferenced_objects: DereferencedObjects, operator_environment: &OperatorEnvironmentOptions, -) -> Result { +) -> Result { let image = kafka .spec .image @@ -115,13 +116,10 @@ pub fn validate( .validate_authentication_methods() .context(FailedToValidateAuthenticationMethodSnafu)?; - // DESIGN DECISION: build the per-rolegroup config (merged config + resolved overrides) - // here, so reconcile reads a fully-typed ValidatedKafkaCluster instead of re-deriving - // merged_config in the loop and threading a product-config HashMap. Alternative: keep - // deriving merged_config in the reconcile loop — rejected; validation is the right place - // to prove every rolegroup resolves before any resource is built. - let mut role_groups: BTreeMap> = - BTreeMap::new(); + let mut role_group_configs: BTreeMap< + KafkaRole, + BTreeMap, + > = BTreeMap::new(); // Brokers always exist. let broker_role = kafka @@ -131,7 +129,7 @@ pub fn validate( role: KafkaRole::Broker, })?; - let mut broker_groups: BTreeMap = BTreeMap::new(); + let mut broker_groups: BTreeMap = BTreeMap::new(); for rolegroup_name in broker_role.role_groups.keys() { let merged_config = KafkaRole::Broker .merged_config(kafka, rolegroup_name) @@ -148,7 +146,7 @@ pub fn validate( }, ); } - role_groups.insert(KafkaRole::Broker, broker_groups); + role_group_configs.insert(KafkaRole::Broker, broker_groups); // We need this guard because controller_role() returns an error if controllers is None, // which would stop reconciliation for ZooKeeper-mode clusters. @@ -160,7 +158,8 @@ pub fn validate( role: KafkaRole::Controller, })?; - let mut controller_groups: BTreeMap = BTreeMap::new(); + let mut controller_groups: BTreeMap = + BTreeMap::new(); for rolegroup_name in controller_role.role_groups.keys() { let merged_config = KafkaRole::Controller .merged_config(kafka, rolegroup_name) @@ -177,7 +176,7 @@ pub fn validate( }, ); } - role_groups.insert(KafkaRole::Controller, controller_groups); + role_group_configs.insert(KafkaRole::Controller, controller_groups); } let pod_descriptors = kafka @@ -197,16 +196,18 @@ pub fn validate( .context(InvalidNamespaceSnafu)?; let uid = Uid::from_str(&kafka.uid().context(ObjectHasNoUidSnafu)?).context(InvalidUidSnafu)?; - Ok(ValidatedKafkaCluster::new( + Ok(ValidatedCluster::new( name, namespace, uid, image, - kafka_security, - dereferenced_objects.authorization_config, - role_groups, - pod_descriptors, - metadata_manager, + ValidatedClusterConfig { + kafka_security, + authorization_config: dereferenced_objects.authorization_config, + pod_descriptors, + metadata_manager, + }, + role_group_configs, )) } diff --git a/rust/operator-binary/src/resource/listener.rs b/rust/operator-binary/src/resource/listener.rs index 3161ed0e..bd62f668 100644 --- a/rust/operator-binary/src/resource/listener.rs +++ b/rust/operator-binary/src/resource/listener.rs @@ -4,7 +4,7 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster}, + controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster}, crd::{role::broker::BrokerConfig, security::KafkaTlsSecurity, v1alpha1}, utils::build_recommended_labels, }; @@ -27,11 +27,11 @@ pub enum Error { // TODO (@NickLarsenNZ): Move shared functionality to stackable-operator pub fn build_broker_rolegroup_bootstrap_listener( kafka: &v1alpha1::KafkaCluster, - validated_cluster: &ValidatedKafkaCluster, + validated_cluster: &ValidatedCluster, rolegroup: &RoleGroupRef, merged_config: &BrokerConfig, ) -> Result { - let kafka_security = &validated_cluster.kafka_security; + let kafka_security = &validated_cluster.cluster_config.kafka_security; let resolved_product_image = &validated_cluster.image; Ok(listener::v1alpha1::Listener { diff --git a/rust/operator-binary/src/resource/service.rs b/rust/operator-binary/src/resource/service.rs index bc2fdbfe..631be187 100644 --- a/rust/operator-binary/src/resource/service.rs +++ b/rust/operator-binary/src/resource/service.rs @@ -8,7 +8,7 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster}, + controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster}, crd::{APP_NAME, METRICS_PORT, METRICS_PORT_NAME, security::KafkaTlsSecurity, v1alpha1}, utils::build_recommended_labels, }; @@ -35,7 +35,7 @@ pub enum Error { /// /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. pub fn build_rolegroup_headless_service( - validated_cluster: &ValidatedKafkaCluster, + validated_cluster: &ValidatedCluster, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, kafka_security: &KafkaTlsSecurity, @@ -77,7 +77,7 @@ pub fn build_rolegroup_headless_service( /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label pub fn build_rolegroup_metrics_service( - validated_cluster: &ValidatedKafkaCluster, + validated_cluster: &ValidatedCluster, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, ) -> Result { diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index 3212c650..d3a225de 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -43,7 +43,7 @@ use crate::{ command::{broker_kafka_container_commands, controller_kafka_container_command}, node_id_hasher::node_id_hash32_offset, }, - controller::{KAFKA_CONTROLLER_NAME, ValidatedKafkaCluster, ValidatedRoleGroupConfig}, + controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster, ValidatedRoleGroupConfig}, crd::{ self, APP_NAME, KAFKA_HEAP_OPTS, LISTENER_BOOTSTRAP_VOLUME_NAME, LISTENER_BROKER_VOLUME_NAME, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, @@ -163,13 +163,13 @@ pub enum Error { pub fn build_broker_rolegroup_statefulset( kafka: &v1alpha1::KafkaCluster, kafka_role: &KafkaRole, - validated_cluster: &ValidatedKafkaCluster, + validated_cluster: &ValidatedCluster, rolegroup_ref: &RoleGroupRef, validated_rg: &ValidatedRoleGroupConfig, service_account: &ServiceAccount, cluster_info: &KubernetesClusterInfo, ) -> Result { - let kafka_security = &validated_cluster.kafka_security; + let kafka_security = &validated_cluster.cluster_config.kafka_security; let resolved_product_image = &validated_cluster.image; let merged_config = &validated_rg.merged_config; let recommended_object_labels = build_recommended_labels( @@ -572,13 +572,13 @@ pub fn build_broker_rolegroup_statefulset( pub fn build_controller_rolegroup_statefulset( kafka: &v1alpha1::KafkaCluster, kafka_role: &KafkaRole, - validated_cluster: &ValidatedKafkaCluster, + validated_cluster: &ValidatedCluster, rolegroup_ref: &RoleGroupRef, validated_rg: &ValidatedRoleGroupConfig, service_account: &ServiceAccount, cluster_info: &KubernetesClusterInfo, ) -> Result { - let kafka_security = &validated_cluster.kafka_security; + let kafka_security = &validated_cluster.cluster_config.kafka_security; let resolved_product_image = &validated_cluster.image; let merged_config = &validated_rg.merged_config; let recommended_object_labels = build_recommended_labels( diff --git a/rust/operator-binary/src/utils.rs b/rust/operator-binary/src/utils.rs index 1db2241a..a6edb3c2 100644 --- a/rust/operator-binary/src/utils.rs +++ b/rust/operator-binary/src/utils.rs @@ -5,7 +5,7 @@ use crate::crd::{APP_NAME, OPERATOR_NAME}; /// Build recommended values for labels. /// /// Generic over the owner `T` so the owner can be either the raw `KafkaCluster` or the -/// `ValidatedKafkaCluster` (which also implements `Resource`). +/// `ValidatedCluster` (which also implements `Resource`). pub fn build_recommended_labels<'a, T>( owner: &'a T, controller_name: &'a str, From bc3e8e232652033f7eebe5715b7f2f9e07b2b442 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 19:29:07 +0200 Subject: [PATCH 24/33] refactor: add framework module and merging --- rust/operator-binary/src/config/jvm.rs | 23 +- rust/operator-binary/src/controller.rs | 30 +- .../src/controller/build/config_map.rs | 17 +- .../src/controller/validate.rs | 324 +++++++----------- rust/operator-binary/src/crd/affinity.rs | 16 +- rust/operator-binary/src/crd/role/mod.rs | 121 ++----- rust/operator-binary/src/framework.rs | 11 + .../src/framework/role_utils.rs | 152 ++++++++ rust/operator-binary/src/main.rs | 1 + .../src/resource/statefulset.rs | 24 +- 10 files changed, 383 insertions(+), 336 deletions(-) create mode 100644 rust/operator-binary/src/framework.rs create mode 100644 rust/operator-binary/src/framework/role_utils.rs diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index 79023618..b8e0acd6 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -110,8 +110,17 @@ fn is_heap_jvm_argument(jvm_argument: &str) -> bool { #[cfg(test)] mod tests { + use stackable_operator::kube::ResourceExt; + use super::*; - use crate::crd::{BrokerRole, role::KafkaRole, v1alpha1}; + use crate::{ + crd::{ + BrokerRole, + role::{KafkaRole, broker::BrokerConfig}, + v1alpha1, + }, + framework::role_utils::with_validated_config, + }; #[test] fn test_construct_jvm_arguments_defaults() { @@ -197,12 +206,12 @@ mod tests { let kafka: v1alpha1::KafkaCluster = serde_yaml::from_str(kafka_cluster).expect("illegal test input"); - let kafka_role = KafkaRole::Broker; - let rolegroup_ref = kafka.rolegroup_ref(&kafka_role, "default"); - let merged_config = kafka_role - .merged_config(&kafka, &rolegroup_ref.role_group) - .unwrap(); - let role = kafka.spec.brokers.unwrap(); + let role = kafka.spec.brokers.clone().unwrap(); + let role_group = role.role_groups.get("default").unwrap(); + let default_config = + BrokerConfig::default_config(&kafka.name_any(), &KafkaRole::Broker.to_string()); + let validated = with_validated_config(role_group, &role, &default_config).unwrap(); + let merged_config = AnyConfig::Broker(validated.config); (merged_config, role, "default".to_owned()) } diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 142c77e1..7eca5ff7 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -38,7 +38,7 @@ use crate::{ self, APP_NAME, KafkaClusterStatus, KafkaPodDescriptor, MetadataManager, OPERATOR_NAME, authorization::KafkaAuthorizationConfig, listener::get_kafka_listener_config, - role::{AnyConfig, KafkaRole}, + role::{AnyConfig, AnyConfigOverrides, KafkaRole}, security::KafkaTlsSecurity, v1alpha1, }, @@ -296,20 +296,18 @@ impl Resource for ValidatedCluster { } } -pub struct ValidatedRoleGroupConfig { - pub merged_config: AnyConfig, - // DESIGN DECISION: overrides are resolved into flat maps HERE rather than stored - // as the typed KeyValueConfigOverrides and resolved in the per-file builders (the - // hdfs-operator pattern). Reason: broker and controller use different override - // struct types (KafkaBrokerConfigOverrides vs KafkaControllerConfigOverrides), so a - // single typed field would require an enum. Resolving here keeps the build/properties - // builders taking plain `BTreeMap`. Alternative: an enum over the two - // override types threaded to builders that call resolved_overrides() — more types for - // no behavioural gain. - pub config_file_overrides: BTreeMap, - pub jvm_security_overrides: BTreeMap, - pub env_overrides: BTreeMap, -} +/// A validated, merged Kafka role-group config. +/// +/// The merged config fragment is wrapped in [`AnyConfig`] and the merged +/// `configOverrides` in [`AnyConfigOverrides`], so a single role-agnostic type +/// carries both broker and controller role groups (their concrete config and +/// override types differ). Produced via the local-`framework` +/// [`with_validated_config`](crate::framework::role_utils::with_validated_config). +pub type ValidatedRoleGroupConfig = crate::framework::role_utils::RoleGroupConfig< + AnyConfig, + stackable_operator::role_utils::JavaCommonConfig, + AnyConfigOverrides, +>; pub async fn reconcile_kafka( kafka: Arc>, @@ -434,7 +432,7 @@ pub async fn reconcile_kafka( .context(BuildStatefulsetSnafu)?, }; - if let AnyConfig::Broker(broker_config) = &validated_rg.merged_config { + if let AnyConfig::Broker(broker_config) = &validated_rg.config { let rg_bootstrap_listener = build_broker_rolegroup_bootstrap_listener( kafka, &validated_cluster, diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 207caa3a..e966d30a 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -76,8 +76,12 @@ pub fn build_rolegroup_config_map( ) -> Result { let kafka_security = &validated_cluster.cluster_config.kafka_security; let resolved_product_image = &validated_cluster.image; - let kafka_config_file_name = validated_rg.merged_config.config_file_name(); - let config_overrides = validated_rg.config_file_overrides.clone(); + let kafka_config_file_name = validated_rg.config.config_file_name(); + let config_overrides = validated_rg + .config_overrides + .config_file_overrides() + .overrides + .clone(); let opa_connect = validated_cluster .cluster_config @@ -91,7 +95,7 @@ pub fn build_rolegroup_config_map( return NoKraftControllersFoundSnafu.fail(); } - let kafka_config = match &validated_rg.merged_config { + let kafka_config = match &validated_rg.config { AnyConfig::Broker(_) => crate::controller::build::properties::broker_properties::build( kafka_security, listener_config, @@ -116,7 +120,10 @@ pub fn build_rolegroup_config_map( } }; - let jvm_sec_props = &validated_rg.jvm_security_overrides; + let jvm_sec_props = &validated_rg + .config_overrides + .security_properties() + .overrides; let mut cm_builder = ConfigMapBuilder::new(); cm_builder @@ -179,7 +186,7 @@ pub fn build_rolegroup_config_map( let config_data = role_group_config_map_data( &resolved_product_image.product_version, rolegroup, - &validated_rg.merged_config, + &validated_rg.config, ); for (file_name, data) in config_data { if let Some(data) = data { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 228156b1..1f60f095 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -5,14 +5,17 @@ use std::{collections::BTreeMap, str::FromStr}; +use serde::Serialize; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection, - config::merge::{Merge, merge}, + config::{fragment::FromFragment, merge::Merge}, kube::ResourceExt, + role_utils::{GenericRoleConfig, JavaCommonConfig, Role}, + schemars::JsonSchema, v2::{ - config_overrides::KeyValueConfigOverrides, + builder::pod::container::{self, EnvVarName, EnvVarSet}, types::{ kubernetes::{NamespaceName, Uid}, operator::ClusterName, @@ -28,12 +31,19 @@ use crate::{ crd::{ self, CONTAINER_IMAGE_BASE_NAME, authentication::{self}, - role::KafkaRole, + role::{ + AnyConfig, AnyConfigOverrides, KafkaRole, broker::BrokerConfig, + controller::ControllerConfig, + }, security::{self, KafkaTlsSecurity}, v1alpha1, }, + framework::role_utils::with_validated_config, }; +/// The operator-managed env var carrying the Kafka cluster id. +const KAFKA_CLUSTER_ID_ENV: &str = "KAFKA_CLUSTER_ID"; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to resolve product image"))] @@ -50,8 +60,13 @@ pub enum Error { #[snafu(display("cluster object defines no '{role}' role"))] MissingKafkaRole { source: crd::Error, role: KafkaRole }, - #[snafu(display("failed to resolve merged config for rolegroup"))] - ResolveMergedConfig { source: crate::crd::role::Error }, + #[snafu(display("failed to merge and validate the role group config"))] + ValidateRoleGroupConfig { + source: crate::framework::role_utils::Error, + }, + + #[snafu(display("invalid environment variable name"))] + InvalidEnvVarName { source: container::Error }, #[snafu(display("failed to build pod descriptors"))] BuildPodDescriptors { source: crate::crd::Error }, @@ -116,66 +131,36 @@ pub fn validate( .validate_authentication_methods() .context(FailedToValidateAuthenticationMethodSnafu)?; + let cluster_id = kafka.cluster_id(); + let mut role_group_configs: BTreeMap< KafkaRole, BTreeMap, > = BTreeMap::new(); // Brokers always exist. - let broker_role = kafka - .broker_role() - .cloned() - .context(MissingKafkaRoleSnafu { - role: KafkaRole::Broker, - })?; - - let mut broker_groups: BTreeMap = BTreeMap::new(); - for rolegroup_name in broker_role.role_groups.keys() { - let merged_config = KafkaRole::Broker - .merged_config(kafka, rolegroup_name) - .context(ResolveMergedConfigSnafu)?; - let (config_file_overrides, jvm_security_overrides, env_overrides) = - collect_broker_role_group_overrides(kafka, &broker_role, rolegroup_name); - broker_groups.insert( - rolegroup_name.clone(), - ValidatedRoleGroupConfig { - merged_config, - config_file_overrides, - jvm_security_overrides, - env_overrides, - }, - ); - } + let broker_role = kafka.broker_role().context(MissingKafkaRoleSnafu { + role: KafkaRole::Broker, + })?; + let broker_groups = validate_role_group_configs( + broker_role, + BrokerConfig::default_config(&kafka.name_any(), &KafkaRole::Broker.to_string()), + cluster_id, + AnyConfig::Broker, + AnyConfigOverrides::Broker, + )?; role_group_configs.insert(KafkaRole::Broker, broker_groups); - // We need this guard because controller_role() returns an error if controllers is None, - // which would stop reconciliation for ZooKeeper-mode clusters. - if kafka.spec.controllers.is_some() { - let controller_role = kafka - .controller_role() - .cloned() - .context(MissingKafkaRoleSnafu { - role: KafkaRole::Controller, - })?; - - let mut controller_groups: BTreeMap = - BTreeMap::new(); - for rolegroup_name in controller_role.role_groups.keys() { - let merged_config = KafkaRole::Controller - .merged_config(kafka, rolegroup_name) - .context(ResolveMergedConfigSnafu)?; - let (config_file_overrides, jvm_security_overrides, env_overrides) = - collect_controller_role_group_overrides(kafka, &controller_role, rolegroup_name); - controller_groups.insert( - rolegroup_name.clone(), - ValidatedRoleGroupConfig { - merged_config, - config_file_overrides, - jvm_security_overrides, - env_overrides, - }, - ); - } + // Controllers are optional: ZooKeeper-mode clusters have none, and `controller_role()` + // errors when `controllers` is unset, which would stop their reconciliation. + if let Some(controller_role) = kafka.spec.controllers.as_ref() { + let controller_groups = validate_role_group_configs( + controller_role, + ControllerConfig::default_config(&kafka.name_any(), &KafkaRole::Controller.to_string()), + cluster_id, + AnyConfig::Controller, + AnyConfigOverrides::Controller, + )?; role_group_configs.insert(KafkaRole::Controller, controller_groups); } @@ -211,163 +196,108 @@ pub fn validate( )) } -/// Merge role-group overrides over the role-level overrides (role-group wins per key) via the -/// `Merge` impl derived on the override structs. -fn merge_role_group_overrides(role: &O, role_group: Option<&O>) -> O { - match role_group { - Some(role_group) => merge(role_group.clone(), role), - None => role.clone(), - } -} - -/// Flatten resolved key/value overrides into a plain map. operator-rs #1219 made the override -/// values plain `String`, so there is no longer any `null`/unset entry to drop. -fn flatten_overrides(overrides: KeyValueConfigOverrides) -> BTreeMap { - overrides.overrides +/// Validates every role group of a role into a map keyed by role group name. +/// +/// Each role group is merged and validated via the local-`framework` +/// [`with_validated_config`], which folds the config fragment (default <- role <- +/// role group) plus the `configOverrides`, `envOverrides`, `cliOverrides` and +/// `podOverrides` (role group wins) into a single +/// [`RoleGroupConfig`](crate::framework::role_utils::RoleGroupConfig). The concrete +/// per-role validated config and overrides are wrapped into the role-agnostic +/// [`AnyConfig`]/[`AnyConfigOverrides`] via `wrap_config`/`wrap_overrides`, and the +/// operator-managed `KAFKA_CLUSTER_ID` is injected into the env overrides. +fn validate_role_group_configs( + role: &Role, + default_config: Config, + cluster_id: Option<&str>, + wrap_config: fn(ValidatedConfig) -> AnyConfig, + wrap_overrides: fn(ConfigOverrides) -> AnyConfigOverrides, +) -> Result> +where + Config: Clone + Merge, + ValidatedConfig: FromFragment, + ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, +{ + role.role_groups + .iter() + .map(|(role_group_name, role_group)| { + let validated = with_validated_config::< + ValidatedConfig, + JavaCommonConfig, + Config, + GenericRoleConfig, + ConfigOverrides, + >(role_group, role, &default_config) + .context(ValidateRoleGroupConfigSnafu)?; + + // Re-wrap the per-role validated config and overrides into the role-agnostic + // enums; the merged env/cli/pod overrides carry over unchanged, except that + // `KAFKA_CLUSTER_ID` is injected into the env overrides. + let validated = ValidatedRoleGroupConfig { + replicas: validated.replicas, + config: wrap_config(validated.config), + config_overrides: wrap_overrides(validated.config_overrides), + env_overrides: inject_cluster_id(validated.env_overrides, cluster_id)?, + cli_overrides: validated.cli_overrides, + pod_overrides: validated.pod_overrides, + product_specific_common_config: validated.product_specific_common_config, + }; + Ok((role_group_name.clone(), validated)) + }) + .collect() } -fn collect_broker_role_group_overrides( - kafka: &v1alpha1::KafkaCluster, - broker_role: &crate::crd::BrokerRole, - rolegroup_name: &str, -) -> ( - BTreeMap, - BTreeMap, - BTreeMap, -) { - let merged_overrides = merge_role_group_overrides( - &broker_role.config.config_overrides, - broker_role - .role_groups - .get(rolegroup_name) - .map(|rg| &rg.config.config_overrides), - ); - let config_file_overrides = flatten_overrides(merged_overrides.broker_properties); - let jvm_security_overrides = flatten_overrides(merged_overrides.security_properties); - - // --- env overrides --- - // DESIGN DECISION: KAFKA_CLUSTER_ID is injected first, then the user env overrides - // (role then role-group) are extended on top, so a user override of the same key wins. - // This mirrors product-config's old merge of compute_env() output with user envOverrides. - // Alternative: inject after user overrides (operator wins) — rejected to preserve the - // previous precedence. - // - // KAFKA_CLUSTER_ID injection moved here from crd/role/broker.rs::Configuration::compute_env. - let mut env_overrides: BTreeMap = BTreeMap::new(); - if let Some(cluster_id) = kafka.cluster_id() { - env_overrides.insert("KAFKA_CLUSTER_ID".to_string(), cluster_id.to_string()); - } - let role_env: &std::collections::HashMap = &broker_role.config.env_overrides; - env_overrides.extend(role_env.iter().map(|(k, v)| (k.clone(), v.clone()))); - if let Some(rg) = broker_role.role_groups.get(rolegroup_name) { - env_overrides.extend( - rg.config - .env_overrides - .iter() - .map(|(k, v)| (k.clone(), v.clone())), - ); +/// Injects the operator-managed `KAFKA_CLUSTER_ID` into the merged env overrides, +/// but only when the user has not already set it via `envOverrides` (user value +/// wins, preserving product-config's old precedence). +/// +/// `KAFKA_CLUSTER_ID` injection moved here from the now-removed +/// `crd::role::*::Configuration::compute_env`. +fn inject_cluster_id(env_overrides: EnvVarSet, cluster_id: Option<&str>) -> Result { + let Some(cluster_id) = cluster_id else { + return Ok(env_overrides); + }; + let name = EnvVarName::from_str(KAFKA_CLUSTER_ID_ENV).context(InvalidEnvVarNameSnafu)?; + if env_overrides.get(&name).is_some() { + // The user set `KAFKA_CLUSTER_ID` via envOverrides; their value wins. + Ok(env_overrides) + } else { + Ok(env_overrides.with_value(&name, cluster_id)) } - - (config_file_overrides, jvm_security_overrides, env_overrides) -} - -fn collect_controller_role_group_overrides( - kafka: &v1alpha1::KafkaCluster, - controller_role: &crate::crd::ControllerRole, - rolegroup_name: &str, -) -> ( - BTreeMap, - BTreeMap, - BTreeMap, -) { - let merged_overrides = merge_role_group_overrides( - &controller_role.config.config_overrides, - controller_role - .role_groups - .get(rolegroup_name) - .map(|rg| &rg.config.config_overrides), - ); - let config_file_overrides = flatten_overrides(merged_overrides.controller_properties); - let jvm_security_overrides = flatten_overrides(merged_overrides.security_properties); - - // --- env overrides --- - // DESIGN DECISION: KAFKA_CLUSTER_ID is injected first, then the user env overrides - // (role then role-group) are extended on top, so a user override of the same key wins. - // This mirrors product-config's old merge of compute_env() output with user envOverrides. - // Alternative: inject after user overrides (operator wins) — rejected to preserve the - // previous precedence. - // - // KAFKA_CLUSTER_ID injection moved here from crd/role/controller.rs::Configuration::compute_env. - let mut env_overrides: BTreeMap = BTreeMap::new(); - if let Some(cluster_id) = kafka.cluster_id() { - env_overrides.insert("KAFKA_CLUSTER_ID".to_string(), cluster_id.to_string()); - } - let role_env: &std::collections::HashMap = - &controller_role.config.env_overrides; - env_overrides.extend(role_env.iter().map(|(k, v)| (k.clone(), v.clone()))); - if let Some(rg) = controller_role.role_groups.get(rolegroup_name) { - env_overrides.extend( - rg.config - .env_overrides - .iter() - .map(|(k, v)| (k.clone(), v.clone())), - ); - } - - (config_file_overrides, jvm_security_overrides, env_overrides) } #[cfg(test)] mod tests { - use std::collections::BTreeMap; + use std::str::FromStr; - use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; + use stackable_operator::v2::builder::pod::container::{EnvVarName, EnvVarSet}; - use super::{flatten_overrides, merge_role_group_overrides}; + use super::{KAFKA_CLUSTER_ID_ENV, inject_cluster_id}; - /// Build a `KeyValueConfigOverrides` from `(key, value)` pairs. - fn overrides(pairs: &[(&str, &str)]) -> KeyValueConfigOverrides { - KeyValueConfigOverrides { - overrides: pairs - .iter() - .map(|(key, value)| (key.to_string(), value.to_string())) - .collect(), - } - } - - /// Run the full role/role-group resolution (merge then flatten) for a single config file. - fn resolve( - role: KeyValueConfigOverrides, - role_group: Option, - ) -> BTreeMap { - flatten_overrides(merge_role_group_overrides(&role, role_group.as_ref())) + fn cluster_id_value(env: &EnvVarSet) -> Option { + let name = EnvVarName::from_str(KAFKA_CLUSTER_ID_ENV).unwrap(); + env.get(&name).and_then(|var| var.value.clone()) } #[test] - fn role_group_value_wins_over_role() { - let role = overrides(&[("a", "role"), ("b", "role-only")]); - let role_group = overrides(&[("a", "rg")]); - - let merged = resolve(role, Some(role_group)); - - assert_eq!( - merged, - BTreeMap::from([ - ("a".to_string(), "rg".to_string()), // role-group wins for shared keys - ("b".to_string(), "role-only".to_string()), // role-only keys are kept - ]) - ); + fn injects_cluster_id_when_absent() { + let env = inject_cluster_id(EnvVarSet::new(), Some("my-id")).unwrap(); + assert_eq!(cluster_id_value(&env), Some("my-id".to_string())); } #[test] - fn without_a_role_group_role_values_are_kept() { - let role = overrides(&[("a", "role")]); + fn user_cluster_id_override_wins() { + let name = EnvVarName::from_str(KAFKA_CLUSTER_ID_ENV).unwrap(); + let env = EnvVarSet::new().with_value(&name, "user-value"); - let merged = resolve(role, None); + let env = inject_cluster_id(env, Some("operator-value")).unwrap(); - assert_eq!( - merged, - BTreeMap::from([("a".to_string(), "role".to_string())]) - ); + assert_eq!(cluster_id_value(&env), Some("user-value".to_string())); + } + + #[test] + fn without_cluster_id_nothing_is_injected() { + let env = inject_cluster_id(EnvVarSet::new(), None).unwrap(); + assert_eq!(cluster_id_value(&env), None); } } diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index da01acca..51e0ae6b 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -30,9 +30,17 @@ mod tests { api::core::v1::{PodAffinityTerm, PodAntiAffinity, WeightedPodAffinityTerm}, apimachinery::pkg::apis::meta::v1::LabelSelector, }, + kube::ResourceExt, }; - use crate::crd::{KafkaRole, v1alpha1}; + use crate::{ + crd::{ + KafkaRole, + role::{AnyConfig, broker::BrokerConfig}, + v1alpha1, + }, + framework::role_utils::with_validated_config, + }; #[rstest] #[case(KafkaRole::Broker)] @@ -55,7 +63,11 @@ mod tests { let kafka: v1alpha1::KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); - let merged_config = role.merged_config(&kafka, "default").unwrap(); + let broker_role = kafka.spec.brokers.clone().unwrap(); + let role_group = broker_role.role_groups.get("default").unwrap(); + let default_config = BrokerConfig::default_config(&kafka.name_any(), &role.to_string()); + let validated = with_validated_config(role_group, &broker_role, &default_config).unwrap(); + let merged_config = AnyConfig::Broker(validated.config); assert_eq!( merged_config.affinity, diff --git a/rust/operator-binary/src/crd/role/mod.rs b/rust/operator-binary/src/crd/role/mod.rs index 6c7bac4f..a7e05881 100644 --- a/rust/operator-binary/src/crd/role/mod.rs +++ b/rust/operator-binary/src/crd/role/mod.rs @@ -8,15 +8,12 @@ use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ commons::resources::{NoRuntimeLimits, Resources}, - config::{ - fragment::{self, ValidationError}, - merge::Merge, - }, k8s_openapi::api::core::v1::PodTemplateSpec, - kube::{ResourceExt, runtime::reflector::ObjectRef}, + kube::runtime::reflector::ObjectRef, product_logging::spec::ContainerLogConfig, role_utils::RoleGroupRef, schemars::{self, JsonSchema}, + v2::config_overrides::KeyValueConfigOverrides, }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; @@ -73,9 +70,6 @@ pub const KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS: &str = "controller.quorum.b #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("fragment validation failure"))] - FragmentValidationFailure { source: ValidationError }, - #[snafu(display("the Kafka role [{role}] is missing from spec"))] MissingRole { source: crate::crd::Error, @@ -140,87 +134,6 @@ impl KafkaRole { "kafka" } - /// Merge the [Broker|Controller]ConfigFragment defaults, role and role group settings. - /// The priority is: default < role config < role_group config - pub fn merged_config( - &self, - kafka: &v1alpha1::KafkaCluster, - rolegroup: &str, - ) -> Result { - match self { - Self::Broker => { - // Initialize the result with all default values as baseline - let default_config = - BrokerConfig::default_config(&kafka.name_any(), &self.to_string()); - - // Retrieve role resource config - let role = kafka.broker_role().with_context(|_| MissingRoleSnafu { - role: self.to_string(), - })?; - - let mut role_config = role.config.config.clone(); - // Retrieve rolegroup specific resource config - let mut role_group_config = role - .role_groups - .get(rolegroup) - .with_context(|| MissingRoleGroupSnafu { - role: self.to_string(), - rolegroup: rolegroup.to_string(), - })? - .config - .config - .clone(); - - // Merge more specific configs into default config - // Hierarchy is: - // 1. RoleGroup - // 2. Role - // 3. Default - role_config.merge(&default_config); - role_group_config.merge(&role_config); - Ok(AnyConfig::Broker( - fragment::validate::(role_group_config) - .context(FragmentValidationFailureSnafu)?, - )) - } - Self::Controller => { - // Initialize the result with all default values as baseline - let default_config = - ControllerConfig::default_config(&kafka.name_any(), &self.to_string()); - - // Retrieve role resource config - let role = kafka.controller_role().with_context(|_| MissingRoleSnafu { - role: self.to_string(), - })?; - - let mut role_config = role.config.config.clone(); - // Retrieve rolegroup specific resource config - let mut role_group_config = role - .role_groups - .get(rolegroup) - .with_context(|| MissingRoleGroupSnafu { - role: self.to_string(), - rolegroup: rolegroup.to_string(), - })? - .config - .config - .clone(); - - // Merge more specific configs into default config - // Hierarchy is: - // 1. RoleGroup - // 2. Role - // 3. Default - role_config.merge(&default_config); - role_group_config.merge(&role_config); - Ok(AnyConfig::Controller( - fragment::validate::(role_group_config) - .context(FragmentValidationFailureSnafu)?, - )) - } - } - } - pub fn construct_non_heap_jvm_args( &self, merged_config: &AnyConfig, @@ -449,3 +362,33 @@ impl AnyConfig { } } } + +/// Merged role/role-group `configOverrides` for a role group of an unknown type. +/// +/// Mirrors [`AnyConfig`] for the override side: broker and controller use distinct +/// override structs, so this enum lets the build layer carry the typed, merged +/// overrides through a single role-agnostic `RoleGroupConfig`. +#[derive(Clone, Debug, PartialEq)] +pub enum AnyConfigOverrides { + Broker(v1alpha1::KafkaBrokerConfigOverrides), + Controller(v1alpha1::KafkaControllerConfigOverrides), +} + +impl AnyConfigOverrides { + /// The merged product config-file overrides (`broker.properties` for brokers, + /// `controller.properties` for controllers). + pub fn config_file_overrides(&self) -> &KeyValueConfigOverrides { + match self { + AnyConfigOverrides::Broker(o) => &o.broker_properties, + AnyConfigOverrides::Controller(o) => &o.controller_properties, + } + } + + /// The merged `security.properties` overrides (shared by both roles). + pub fn security_properties(&self) -> &KeyValueConfigOverrides { + match self { + AnyConfigOverrides::Broker(o) => &o.security_properties, + AnyConfigOverrides::Controller(o) => &o.security_properties, + } + } +} diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs new file mode 100644 index 00000000..6e28e88a --- /dev/null +++ b/rust/operator-binary/src/framework.rs @@ -0,0 +1,11 @@ +//! Local framework helpers that mirror the work-in-progress upstream +//! `stackable_operator::v2::*` modules. +//! +//! We vendor `role_utils` because the upstream `v2::role_utils` requires +//! `CommonConfig: Merge`. Kafka (like hdfs and trino) uses `JavaCommonConfig`, +//! whose JVM-argument merge is fallible and so does not implement `Merge`. +//! +//! Follow-up: replace with `stackable_operator::v2::role_utils::*` once upstream +//! relaxes the `Merge` bound. + +pub mod role_utils; diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs new file mode 100644 index 00000000..b4bc9a8b --- /dev/null +++ b/rust/operator-binary/src/framework/role_utils.rs @@ -0,0 +1,152 @@ +//! Vendored variant of `stackable_operator::v2::role_utils` from the +//! `smooth-operator` branch, with simplifications appropriate for kafka-operator. +//! +//! Differences from upstream: +//! - No `cli_overrides_to_vec` helper, `ResourceNames`, or service-account helpers. +//! - The `CommonConfig` (a.k.a. `product_specific_common_config`) does NOT need to +//! implement `Merge`. Kafka uses `JavaCommonConfig`, which intentionally does not +//! implement `Merge` because its inner `JvmArgumentOverrides::try_merge` is +//! fallible (regex validation). The `RoleGroupConfig::product_specific_common_config` +//! field here simply carries the role-group level value through. +//! +//! Replace with `stackable_operator::v2::role_utils::*` once upstream relaxes the +//! `Merge` bound. + +use std::{ + collections::{BTreeMap, HashMap}, + str::FromStr, +}; + +use serde::Serialize; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + config::{ + fragment::{self, FromFragment}, + merge::{Merge, merge}, + }, + k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, + role_utils::{Role, RoleGroup}, + schemars::JsonSchema, + v2::builder::pod::container::{self, EnvVarName, EnvVarSet}, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to validate the role group config"))] + ValidateConfig { source: fragment::ValidationError }, + + #[snafu(display("invalid environment variable override name"))] + ParseEnvVarName { source: container::Error }, +} + +/// Kafka-friendly view of a validated, merged `RoleGroup`. +#[derive(Clone, Debug, PartialEq)] +pub struct RoleGroupConfig { + pub replicas: u16, + pub config: Config, + pub config_overrides: ConfigOverrides, + pub env_overrides: EnvVarSet, + pub cli_overrides: BTreeMap, + pub pod_overrides: PodTemplateSpec, + pub product_specific_common_config: CommonConfig, +} + +/// Merges and validates the `RoleGroup` with the given `role` and `default_config`. +pub fn with_validated_config( + role_group: &RoleGroup, + role: &Role, + default_config: &Config, +) -> Result, Error> +where + ValidatedConfig: FromFragment, + CommonConfig: Clone + Default + JsonSchema + Serialize, + Config: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, +{ + let validated_config = + validate_config(role_group, role, default_config).context(ValidateConfigSnafu)?; + Ok(RoleGroupConfig { + replicas: role_group.replicas.unwrap_or(1), + config: validated_config, + config_overrides: merged_config_overrides( + &role.config.config_overrides, + role_group.config.config_overrides.clone(), + ), + env_overrides: merged_env_overrides( + &role.config.env_overrides, + &role_group.config.env_overrides, + )?, + cli_overrides: merged_cli_overrides( + role.config.cli_overrides.clone(), + role_group.config.cli_overrides.clone(), + ), + pod_overrides: merged_pod_overrides( + role.config.pod_overrides.clone(), + role_group.config.pod_overrides.clone(), + ), + product_specific_common_config: role_group.config.product_specific_common_config.clone(), + }) +} + +fn validate_config( + role_group: &RoleGroup, + role: &Role, + default_config: &Config, +) -> Result +where + ValidatedConfig: FromFragment, + CommonConfig: Default + JsonSchema + Serialize, + Config: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, +{ + role_group.validate_config(role, default_config) +} + +fn merged_config_overrides( + role_config_overrides: &ConfigOverrides, + role_group_config_overrides: ConfigOverrides, +) -> ConfigOverrides +where + ConfigOverrides: Merge, +{ + merge(role_group_config_overrides, role_config_overrides) +} + +fn merged_env_overrides( + role_env_overrides: &HashMap, + role_group_env_overrides: &HashMap, +) -> Result { + // Process the role first, then the role group, so that role-group overrides win on key + // collisions (`EnvVarSet::with_value` overrides earlier entries with the same name). + let mut env_overrides = EnvVarSet::new(); + for (name, value) in role_env_overrides + .iter() + .chain(role_group_env_overrides.iter()) + { + env_overrides = env_overrides.with_value( + &EnvVarName::from_str(name).context(ParseEnvVarNameSnafu)?, + value.clone(), + ); + } + Ok(env_overrides) +} + +fn merged_cli_overrides( + role_cli_overrides: BTreeMap, + role_group_cli_overrides: BTreeMap, +) -> BTreeMap { + let mut merged = role_cli_overrides; + merged.extend(role_group_cli_overrides); + merged +} + +fn merged_pod_overrides( + role_pod_overrides: PodTemplateSpec, + role_group_pod_overrides: PodTemplateSpec, +) -> PodTemplateSpec { + let mut merged = role_pod_overrides; + merged.merge_from(role_group_pod_overrides); + merged +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index a967294a..cd1c9434 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -43,6 +43,7 @@ use crate::{ mod config; mod controller; mod crd; +mod framework; mod kerberos; mod operations; mod product_logging; diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index d3a225de..cea62ae0 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -171,7 +171,7 @@ pub fn build_broker_rolegroup_statefulset( ) -> Result { let kafka_security = &validated_cluster.cluster_config.kafka_security; let resolved_product_image = &validated_cluster.image; - let merged_config = &validated_rg.merged_config; + let merged_config = &validated_rg.config; let recommended_object_labels = build_recommended_labels( validated_cluster, KAFKA_CONTROLLER_NAME, @@ -244,15 +244,7 @@ pub fn build_broker_rolegroup_statefulset( .context(AddKerberosConfigSnafu)?; } - let mut env = validated_rg - .env_overrides - .iter() - .map(|(k, v)| EnvVar { - name: k.clone(), - value: Some(v.clone()), - ..EnvVar::default() - }) - .collect::>(); + let mut env = Vec::::from(validated_rg.env_overrides.clone()); if let Some(zookeeper_config_map_name) = &kafka.spec.cluster_config.zookeeper_config_map_name { env.push(EnvVar { @@ -580,7 +572,7 @@ pub fn build_controller_rolegroup_statefulset( ) -> Result { let kafka_security = &validated_cluster.cluster_config.kafka_security; let resolved_product_image = &validated_cluster.image; - let merged_config = &validated_rg.merged_config; + let merged_config = &validated_rg.config; let recommended_object_labels = build_recommended_labels( validated_cluster, KAFKA_CONTROLLER_NAME, @@ -597,15 +589,7 @@ pub fn build_controller_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); - let mut env = validated_rg - .env_overrides - .iter() - .map(|(k, v)| EnvVar { - name: k.clone(), - value: Some(v.clone()), - ..EnvVar::default() - }) - .collect::>(); + let mut env = Vec::::from(validated_rg.env_overrides.clone()); env.push(EnvVar { name: "NAMESPACE".to_string(), From 241284eb77637dafb0f87457a7bf8485697b724a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 19:34:39 +0200 Subject: [PATCH 25/33] chore: regenerate charts --- extra/crds.yaml | 160 ------------------------------------------------ 1 file changed, 160 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index ea71c1c4..6177a6e5 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -98,86 +98,6 @@ spec: containers: description: Log configuration per container. properties: - get-service: - anyOf: - - required: - - custom - - {} - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object kafka: anyOf: - required: @@ -707,86 +627,6 @@ spec: containers: description: Log configuration per container. properties: - get-service: - anyOf: - - required: - - custom - - {} - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object kafka: anyOf: - required: From b4fd4dcf7f8d0e88f0a46b68cf092fe9a0266f2c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 19:35:21 +0200 Subject: [PATCH 26/33] fix: remove obsolete errors, dead enum variant, tighten visibility of constants --- rust/operator-binary/src/controller.rs | 14 +++++--------- .../src/controller/build/config_map.rs | 9 --------- rust/operator-binary/src/crd/role/broker.rs | 1 - rust/operator-binary/src/product_logging.rs | 8 ++++---- rust/operator-binary/src/resource/statefulset.rs | 12 +----------- 5 files changed, 10 insertions(+), 34 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 7eca5ff7..5b14b9e7 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -35,7 +35,7 @@ mod validate; use crate::{ crd::{ - self, APP_NAME, KafkaClusterStatus, KafkaPodDescriptor, MetadataManager, OPERATOR_NAME, + APP_NAME, KafkaClusterStatus, KafkaPodDescriptor, MetadataManager, OPERATOR_NAME, authorization::KafkaAuthorizationConfig, listener::get_kafka_listener_config, role::{AnyConfig, AnyConfigOverrides, KafkaRole}, @@ -73,8 +73,8 @@ pub enum Error { source: crate::crd::listener::KafkaListenerError, }, - #[snafu(display("failed to apply role Service"))] - ApplyRoleService { + #[snafu(display("failed to apply bootstrap Listener"))] + ApplyBootstrapListener { source: stackable_operator::cluster_resources::Error, }, @@ -150,9 +150,6 @@ pub enum Error { source: error_boundary::InvalidObject, }, - #[snafu(display("KafkaCluster object is misconfigured"))] - MisconfiguredKafkaCluster { source: crd::Error }, - #[snafu(display("failed to build statefulset"))] BuildStatefulset { source: crate::resource::statefulset::Error, @@ -184,7 +181,7 @@ impl ReconcilerError for Error { match self { Error::Dereference { .. } => None, Error::ValidateCluster { .. } => None, - Error::ApplyRoleService { .. } => None, + Error::ApplyBootstrapListener { .. } => None, Error::ApplyRoleGroupService { .. } => None, Error::ApplyRoleGroupConfig { .. } => None, Error::ApplyRoleGroupStatefulSet { .. } => None, @@ -199,7 +196,6 @@ impl ReconcilerError for Error { Error::FailedToCreatePdb { .. } => None, Error::GetRequiredLabels { .. } => None, Error::InvalidKafkaCluster { .. } => None, - Error::MisconfiguredKafkaCluster { .. } => None, Error::BuildStatefulset { .. } => None, Error::BuildConfigMap { .. } => None, Error::BuildService { .. } => None, @@ -444,7 +440,7 @@ pub async fn reconcile_kafka( cluster_resources .add(client, rg_bootstrap_listener) .await - .context(ApplyRoleServiceSnafu)?, + .context(ApplyBootstrapListenerSnafu)?, ); } diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index e966d30a..5445e6fd 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -22,9 +22,6 @@ use crate::{ #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("invalid metadata manager"))] - InvalidMetadataManager { source: crate::crd::Error }, - #[snafu(display("failed to build ConfigMap for {}", rolegroup))] BuildRoleGroupConfig { source: stackable_operator::builder::configmap::Error, @@ -56,12 +53,6 @@ pub enum Error { rolegroup: RoleGroupRef, }, - #[snafu(display("failed to build jaas configuration file for {rolegroup}"))] - BuildJaasConfig { rolegroup: String }, - - #[snafu(display("failed to build pod descriptors"))] - BuildPodDescriptors { source: crate::crd::Error }, - #[snafu(display("no Kraft controllers found to build"))] NoKraftControllersFound, } diff --git a/rust/operator-binary/src/crd/role/broker.rs b/rust/operator-binary/src/crd/role/broker.rs index 674b9feb..64ebf848 100644 --- a/rust/operator-binary/src/crd/role/broker.rs +++ b/rust/operator-binary/src/crd/role/broker.rs @@ -33,7 +33,6 @@ pub const BROKER_PROPERTIES_FILE: &str = "broker.properties"; pub enum BrokerContainer { Vector, KcatProber, - GetService, Kafka, } diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index a16d0148..74afff30 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -18,11 +18,11 @@ pub const BROKER_ID_POD_MAP_DIR: &str = "/stackable/broker-id-pod-map"; pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; // log4j -pub const LOG4J_CONFIG_FILE: &str = "log4j.properties"; -pub const KAFKA_LOG4J_FILE: &str = "kafka.log4j.xml"; +const LOG4J_CONFIG_FILE: &str = "log4j.properties"; +const KAFKA_LOG4J_FILE: &str = "kafka.log4j.xml"; // log4j2 -pub const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; -pub const KAFKA_LOG4J2_FILE: &str = "kafka.log4j2.xml"; +const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; +const KAFKA_LOG4J2_FILE: &str = "kafka.log4j2.xml"; // max size pub const MAX_KAFKA_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { value: 10.0, diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index cea62ae0..1d274bd6 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -118,11 +118,6 @@ pub enum Error { source: stackable_operator::builder::pod::container::Error, }, - #[snafu(display("invalid kafka listeners"))] - InvalidKafkaListeners { - source: crate::crd::listener::KafkaListenerError, - }, - #[snafu(display("failed to build Labels"))] LabelBuild { source: stackable_operator::kvp::LabelError, @@ -147,11 +142,6 @@ pub enum Error { #[snafu(display("failed to retrieve rolegroup replicas"))] RoleGroupReplicas { source: crd::role::Error }, - #[snafu(display( - "cluster does not define 'metadata.name' which is required for the Kafka cluster id" - ))] - ClusterIdMissing, - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] VectorAggregatorConfigMapMissing, } @@ -159,7 +149,7 @@ pub enum Error { /// The broker rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding -/// [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) from [`build_rolegroup_service`](`crate::resource::service::build_rolegroup_headless_service`). +/// [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) from [`build_rolegroup_headless_service`](`crate::resource::service::build_rolegroup_headless_service`). pub fn build_broker_rolegroup_statefulset( kafka: &v1alpha1::KafkaCluster, kafka_role: &KafkaRole, From 63d820c1399a7d75745ea9f87a8caec2e2a9a941 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 19:56:05 +0200 Subject: [PATCH 27/33] refactor: move logging mod to controller, remove util mod --- rust/operator-binary/src/config/command.rs | 28 ++++++++++---- rust/operator-binary/src/controller.rs | 30 +++++++++++++++ .../src/controller/build/config_map.rs | 7 ++-- .../src/controller/build/discovery.rs | 3 +- .../build/properties/logging.rs} | 37 +++++-------------- .../src/controller/build/properties/mod.rs | 1 + rust/operator-binary/src/crd/mod.rs | 6 +++ rust/operator-binary/src/main.rs | 2 - rust/operator-binary/src/resource/listener.rs | 3 +- rust/operator-binary/src/resource/service.rs | 3 +- .../src/resource/statefulset.rs | 20 +++++----- rust/operator-binary/src/utils.rs | 25 ------------- 12 files changed, 86 insertions(+), 79 deletions(-) rename rust/operator-binary/src/{product_logging.rs => controller/build/properties/logging.rs} (80%) delete mode 100644 rust/operator-binary/src/utils.rs diff --git a/rust/operator-binary/src/config/command.rs b/rust/operator-binary/src/config/command.rs index a4540001..233ef4b1 100644 --- a/rust/operator-binary/src/config/command.rs +++ b/rust/operator-binary/src/config/command.rs @@ -6,15 +6,29 @@ use stackable_operator::{ utils::COMMON_BASH_TRAP_FUNCTIONS, }; -use crate::{ - crd::{ - KafkaPodDescriptor, STACKABLE_CONFIG_DIR, STACKABLE_KERBEROS_KRB5_PATH, - role::{broker::BROKER_PROPERTIES_FILE, controller::CONTROLLER_PROPERTIES_FILE}, - security::KafkaTlsSecurity, - }, - product_logging::{BROKER_ID_POD_MAP_DIR, STACKABLE_LOG_DIR}, +use crate::crd::{ + BROKER_ID_POD_MAP_DIR, KafkaPodDescriptor, LOG4J_CONFIG_FILE, LOG4J2_CONFIG_FILE, + STACKABLE_CONFIG_DIR, STACKABLE_KERBEROS_KRB5_PATH, STACKABLE_LOG_CONFIG_DIR, + STACKABLE_LOG_DIR, + role::{broker::BROKER_PROPERTIES_FILE, controller::CONTROLLER_PROPERTIES_FILE}, + security::KafkaTlsSecurity, }; +/// The JVM options selecting the Kafka log4j/log4j2 config file. Kafka 3.x uses log4j, +/// Kafka 4.0 and higher use log4j2. +pub fn kafka_log_opts(product_version: &str) -> String { + if product_version.starts_with("3.") { + format!("-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J_CONFIG_FILE}") + } else { + format!("-Dlog4j2.configurationFile=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J2_CONFIG_FILE}") + } +} + +/// The env var carrying the Kafka log4j options (see [`kafka_log_opts`]). +pub fn kafka_log_opts_env_var() -> String { + "KAFKA_LOG4J_OPTS".to_string() +} + /// Returns the commands to start the main Kafka container pub fn broker_kafka_container_commands( kraft_mode: bool, diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 5b14b9e7..09d16d4f 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -15,7 +15,9 @@ use stackable_operator::{ core::{DeserializeGuard, error_boundary}, runtime::{controller::Action, reflector::ObjectRef}, }, + kvp::ObjectLabels, logging::controller::ReconcilerError, + memory::{BinaryMultiple, MemoryQuantity}, role_utils::{GenericRoleConfig, RoleGroupRef}, shared::time::Duration, status::condition::{ @@ -53,6 +55,34 @@ use crate::{ pub const KAFKA_CONTROLLER_NAME: &str = "kafkacluster"; pub const KAFKA_FULL_CONTROLLER_NAME: &str = concatcp!(KAFKA_CONTROLLER_NAME, '.', OPERATOR_NAME); +/// The maximum size of a single Kafka log file before it is rotated. +pub const MAX_KAFKA_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 10.0, + unit: BinaryMultiple::Mebi, +}; + +/// Build recommended values for labels. +/// +/// Generic over the owner `T` so the owner can be either the raw `KafkaCluster` or the +/// [`ValidatedCluster`] (which also implements `Resource`). +pub fn build_recommended_labels<'a, T>( + owner: &'a T, + controller_name: &'a str, + app_version: &'a str, + role: &'a str, + role_group: &'a str, +) -> ObjectLabels<'a, T> { + ObjectLabels { + owner, + app_name: APP_NAME, + app_version, + operator_name: OPERATOR_NAME, + controller_name, + role, + role_group, + } +} + pub struct Ctx { pub client: stackable_operator::client::Client, pub operator_environment: OperatorEnvironmentOptions, diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 5445e6fd..9f87b8be 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -8,7 +8,10 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster, ValidatedRoleGroupConfig}, + controller::{ + KAFKA_CONTROLLER_NAME, ValidatedCluster, ValidatedRoleGroupConfig, + build::properties::logging::role_group_config_map_data, build_recommended_labels, + }, crd::{ JVM_SECURITY_PROPERTIES_FILE, MetadataManager, STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, @@ -16,8 +19,6 @@ use crate::{ role::AnyConfig, v1alpha1, }, - product_logging::role_group_config_map_data, - utils::build_recommended_labels, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs index 10ac621c..58a90499 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -9,9 +9,8 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster}, + controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster, build_recommended_labels}, crd::{role::KafkaRole, v1alpha1}, - utils::build_recommended_labels, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs similarity index 80% rename from rust/operator-binary/src/product_logging.rs rename to rust/operator-binary/src/controller/build/properties/logging.rs index 74afff30..76570a91 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/controller/build/properties/logging.rs @@ -1,3 +1,6 @@ +//! Renders the logging config files (`log4j.properties` / `log4j2.properties` and the +//! Vector agent config) assembled into the rolegroup `ConfigMap`. + use std::{borrow::Cow, collections::BTreeMap, fmt::Display}; use stackable_operator::{ @@ -9,41 +12,21 @@ use stackable_operator::{ role_utils::RoleGroupRef, }; -use crate::crd::{ - role::{AnyConfig, broker::BrokerContainer, controller::ControllerContainer}, - v1alpha1, +use crate::{ + controller::MAX_KAFKA_LOG_FILES_SIZE, + crd::{ + LOG4J_CONFIG_FILE, LOG4J2_CONFIG_FILE, STACKABLE_LOG_DIR, + role::{AnyConfig, broker::BrokerContainer, controller::ControllerContainer}, + v1alpha1, + }, }; -pub const BROKER_ID_POD_MAP_DIR: &str = "/stackable/broker-id-pod-map"; -pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config"; -pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; -// log4j -const LOG4J_CONFIG_FILE: &str = "log4j.properties"; const KAFKA_LOG4J_FILE: &str = "kafka.log4j.xml"; -// log4j2 -const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; const KAFKA_LOG4J2_FILE: &str = "kafka.log4j2.xml"; -// max size -pub const MAX_KAFKA_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { - value: 10.0, - unit: BinaryMultiple::Mebi, -}; const CONSOLE_CONVERSION_PATTERN_LOG4J: &str = "[%d] %p %m (%c)%n"; const CONSOLE_CONVERSION_PATTERN_LOG4J2: &str = "%d{ISO8601} %p [%t] %c - %m%n"; -pub fn kafka_log_opts(product_version: &str) -> String { - if product_version.starts_with("3.") { - format!("-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J_CONFIG_FILE}") - } else { - format!("-Dlog4j2.configurationFile=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J2_CONFIG_FILE}") - } -} - -pub fn kafka_log_opts_env_var() -> String { - "KAFKA_LOG4J_OPTS".to_string() -} - /// Get the role group ConfigMap data with logging and Vector configurations pub fn role_group_config_map_data( product_version: &str, diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 116f5c2f..4f83b22c 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -2,6 +2,7 @@ pub mod broker_properties; pub mod controller_properties; +pub mod logging; use crate::crd::{KafkaPodDescriptor, role::KafkaRole}; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index e69c7a4c..740bc9ad 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -60,6 +60,12 @@ pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; // kerberos pub const STACKABLE_KERBEROS_DIR: &str = "/stackable/kerberos"; pub const STACKABLE_KERBEROS_KRB5_PATH: &str = "/stackable/kerberos/krb5.conf"; +// logging +pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; +pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config"; +pub const BROKER_ID_POD_MAP_DIR: &str = "/stackable/broker-id-pod-map"; +pub const LOG4J_CONFIG_FILE: &str = "log4j.properties"; +pub const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; #[derive(Snafu, Debug)] pub enum Error { diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index cd1c9434..21b2f466 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -46,9 +46,7 @@ mod crd; mod framework; mod kerberos; mod operations; -mod product_logging; mod resource; -mod utils; mod webhooks; mod built_info { diff --git a/rust/operator-binary/src/resource/listener.rs b/rust/operator-binary/src/resource/listener.rs index bd62f668..f85245ec 100644 --- a/rust/operator-binary/src/resource/listener.rs +++ b/rust/operator-binary/src/resource/listener.rs @@ -4,9 +4,8 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster}, + controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster, build_recommended_labels}, crd::{role::broker::BrokerConfig, security::KafkaTlsSecurity, v1alpha1}, - utils::build_recommended_labels, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/resource/service.rs b/rust/operator-binary/src/resource/service.rs index 631be187..5b3802d3 100644 --- a/rust/operator-binary/src/resource/service.rs +++ b/rust/operator-binary/src/resource/service.rs @@ -8,9 +8,8 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster}, + controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster, build_recommended_labels}, crd::{APP_NAME, METRICS_PORT, METRICS_PORT_NAME, security::KafkaTlsSecurity, v1alpha1}, - utils::build_recommended_labels, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index 1d274bd6..330331af 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -40,15 +40,22 @@ use stackable_operator::{ use crate::{ config::{ - command::{broker_kafka_container_commands, controller_kafka_container_command}, + command::{ + broker_kafka_container_commands, controller_kafka_container_command, kafka_log_opts, + kafka_log_opts_env_var, + }, node_id_hasher::node_id_hash32_offset, }, - controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster, ValidatedRoleGroupConfig}, + controller::{ + KAFKA_CONTROLLER_NAME, MAX_KAFKA_LOG_FILES_SIZE, ValidatedCluster, + ValidatedRoleGroupConfig, build_recommended_labels, + }, crd::{ - self, APP_NAME, KAFKA_HEAP_OPTS, LISTENER_BOOTSTRAP_VOLUME_NAME, + self, APP_NAME, BROKER_ID_POD_MAP_DIR, KAFKA_HEAP_OPTS, LISTENER_BOOTSTRAP_VOLUME_NAME, LISTENER_BROKER_VOLUME_NAME, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, MetadataManager, STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, - STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, + STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, STACKABLE_LOG_CONFIG_DIR, + STACKABLE_LOG_DIR, role::{ KAFKA_NODE_ID_OFFSET, KafkaRole, broker::BrokerContainer, controller::ControllerContainer, @@ -58,11 +65,6 @@ use crate::{ }, kerberos::add_kerberos_pod_config, operations::graceful_shutdown::add_graceful_shutdown_config, - product_logging::{ - BROKER_ID_POD_MAP_DIR, MAX_KAFKA_LOG_FILES_SIZE, STACKABLE_LOG_CONFIG_DIR, - STACKABLE_LOG_DIR, kafka_log_opts, kafka_log_opts_env_var, - }, - utils::build_recommended_labels, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/utils.rs b/rust/operator-binary/src/utils.rs deleted file mode 100644 index a6edb3c2..00000000 --- a/rust/operator-binary/src/utils.rs +++ /dev/null @@ -1,25 +0,0 @@ -use stackable_operator::kvp::ObjectLabels; - -use crate::crd::{APP_NAME, OPERATOR_NAME}; - -/// Build recommended values for labels. -/// -/// Generic over the owner `T` so the owner can be either the raw `KafkaCluster` or the -/// `ValidatedCluster` (which also implements `Resource`). -pub fn build_recommended_labels<'a, T>( - owner: &'a T, - controller_name: &'a str, - app_version: &'a str, - role: &'a str, - role_group: &'a str, -) -> ObjectLabels<'a, T> { - ObjectLabels { - owner, - app_name: APP_NAME, - app_version, - operator_name: OPERATOR_NAME, - controller_name, - role, - role_group, - } -} From ef12f34839d683fe7703cf367f8bc064988f0db4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 20:21:27 +0200 Subject: [PATCH 28/33] refactor: move ValidatedCluster to controller/mod.rs; add ConfigFileName enum; remove raw KafkaCluster from build_configmap --- rust/operator-binary/src/config/command.rs | 29 ++-- rust/operator-binary/src/config/jvm.rs | 9 +- .../src/controller/build/config_map.rs | 27 ++-- .../src/controller/build/discovery.rs | 3 +- .../controller/build/properties/logging.rs | 8 +- rust/operator-binary/src/controller/mod.rs | 137 ++++++++++++++++++ .../src/controller/validate.rs | 5 + rust/operator-binary/src/crd/config_file.rs | 51 +++++++ rust/operator-binary/src/crd/mod.rs | 6 +- rust/operator-binary/src/crd/role/broker.rs | 2 - .../src/crd/role/controller.rs | 2 - rust/operator-binary/src/crd/role/mod.rs | 17 ++- .../{controller.rs => kafka_controller.rs} | 122 +--------------- rust/operator-binary/src/main.rs | 9 +- rust/operator-binary/src/operations/pdb.rs | 2 +- rust/operator-binary/src/resource/listener.rs | 3 +- rust/operator-binary/src/resource/service.rs | 3 +- .../src/resource/statefulset.rs | 6 +- 18 files changed, 262 insertions(+), 179 deletions(-) create mode 100644 rust/operator-binary/src/controller/mod.rs create mode 100644 rust/operator-binary/src/crd/config_file.rs rename rust/operator-binary/src/{controller.rs => kafka_controller.rs} (78%) diff --git a/rust/operator-binary/src/config/command.rs b/rust/operator-binary/src/config/command.rs index 233ef4b1..7cdc5738 100644 --- a/rust/operator-binary/src/config/command.rs +++ b/rust/operator-binary/src/config/command.rs @@ -7,10 +7,8 @@ use stackable_operator::{ }; use crate::crd::{ - BROKER_ID_POD_MAP_DIR, KafkaPodDescriptor, LOG4J_CONFIG_FILE, LOG4J2_CONFIG_FILE, - STACKABLE_CONFIG_DIR, STACKABLE_KERBEROS_KRB5_PATH, STACKABLE_LOG_CONFIG_DIR, - STACKABLE_LOG_DIR, - role::{broker::BROKER_PROPERTIES_FILE, controller::CONTROLLER_PROPERTIES_FILE}, + BROKER_ID_POD_MAP_DIR, ConfigFileName, KafkaPodDescriptor, STACKABLE_CONFIG_DIR, + STACKABLE_KERBEROS_KRB5_PATH, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, security::KafkaTlsSecurity, }; @@ -18,9 +16,15 @@ use crate::crd::{ /// Kafka 4.0 and higher use log4j2. pub fn kafka_log_opts(product_version: &str) -> String { if product_version.starts_with("3.") { - format!("-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J_CONFIG_FILE}") + format!( + "-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{log4j}", + log4j = ConfigFileName::Log4j + ) } else { - format!("-Dlog4j2.configurationFile=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J2_CONFIG_FILE}") + format!( + "-Dlog4j2.configurationFile=file:{STACKABLE_LOG_CONFIG_DIR}/{log4j2}", + log4j2 = ConfigFileName::Log4j2 + ) } } @@ -77,12 +81,13 @@ fn broker_start_command( cp {config_dir}/{properties_file} /tmp/{properties_file} config-utils template /tmp/{properties_file} - cp {config_dir}/jaas.properties /tmp/jaas.properties - config-utils template /tmp/jaas.properties + cp {config_dir}/{jaas_file} /tmp/{jaas_file} + config-utils template /tmp/{jaas_file} ", broker_id_pod_map_dir = BROKER_ID_POD_MAP_DIR, config_dir = STACKABLE_CONFIG_DIR, - properties_file = BROKER_PROPERTIES_FILE, + properties_file = ConfigFileName::BrokerProperties, + jaas_file = ConfigFileName::Jaas, }; if kraft_mode { @@ -92,7 +97,7 @@ fn broker_start_command( bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted {initial_controller_command} bin/kafka-server-start.sh /tmp/{properties_file} & ", - properties_file = BROKER_PROPERTIES_FILE, + properties_file = ConfigFileName::BrokerProperties, initial_controller_command = initial_controllers_command(&controller_descriptors, product_version), } } else { @@ -100,7 +105,7 @@ fn broker_start_command( {common_command} bin/kafka-server-start.sh /tmp/{properties_file} &", - properties_file = BROKER_PROPERTIES_FILE, + properties_file = ConfigFileName::BrokerProperties, } } } @@ -172,7 +177,7 @@ pub fn controller_kafka_container_command( ", remove_vector_shutdown_file_command = remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), config_dir = STACKABLE_CONFIG_DIR, - properties_file = CONTROLLER_PROPERTIES_FILE, + properties_file = ConfigFileName::ControllerProperties, initial_controller_command = initial_controllers_command(&controller_descriptors, product_version), create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR) } diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index b8e0acd6..f0233b66 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -6,9 +6,7 @@ use stackable_operator::{ schemars::JsonSchema, }; -use crate::crd::{ - JVM_SECURITY_PROPERTIES_FILE, METRICS_PORT, STACKABLE_CONFIG_DIR, role::AnyConfig, -}; +use crate::crd::{ConfigFileName, METRICS_PORT, STACKABLE_CONFIG_DIR, role::AnyConfig}; const JAVA_HEAP_FACTOR: f32 = 0.8; @@ -54,7 +52,10 @@ where // Heap settings format!("-Xmx{java_heap}"), format!("-Xms{java_heap}"), - format!("-Djava.security.properties={STACKABLE_CONFIG_DIR}/{JVM_SECURITY_PROPERTIES_FILE}"), + format!( + "-Djava.security.properties={STACKABLE_CONFIG_DIR}/{security}", + security = ConfigFileName::Security + ), format!( "-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/server.yaml" ), diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 9f87b8be..919f3a66 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -9,16 +9,17 @@ use stackable_operator::{ use crate::{ controller::{ - KAFKA_CONTROLLER_NAME, ValidatedCluster, ValidatedRoleGroupConfig, - build::properties::logging::role_group_config_map_data, build_recommended_labels, + ValidatedCluster, ValidatedRoleGroupConfig, + build::properties::logging::role_group_config_map_data, }, crd::{ - JVM_SECURITY_PROPERTIES_FILE, MetadataManager, STACKABLE_LISTENER_BOOTSTRAP_DIR, + ConfigFileName, MetadataManager, STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, listener::{KafkaListenerConfig, node_address_cmd}, role::AnyConfig, v1alpha1, }, + kafka_controller::{KAFKA_CONTROLLER_NAME, build_recommended_labels}, }; #[derive(Snafu, Debug)] @@ -29,10 +30,7 @@ pub enum Error { rolegroup: RoleGroupRef, }, - #[snafu(display( - "failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", - rolegroup - ))] + #[snafu(display("failed to serialize [{}] for {rolegroup}", ConfigFileName::Security))] JvmSecurityProperties { source: PropertiesWriterError, rolegroup: String, @@ -60,7 +58,6 @@ pub enum Error { /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator pub fn build_rolegroup_config_map( - kafka: &v1alpha1::KafkaCluster, validated_cluster: &ValidatedCluster, rolegroup: &RoleGroupRef, validated_rg: &ValidatedRoleGroupConfig, @@ -68,7 +65,7 @@ pub fn build_rolegroup_config_map( ) -> Result { let kafka_security = &validated_cluster.cluster_config.kafka_security; let resolved_product_image = &validated_cluster.image; - let kafka_config_file_name = validated_rg.config.config_file_name(); + let kafka_config_file_name = validated_rg.config.config_file_name().to_string(); let config_overrides = validated_rg .config_overrides .config_file_overrides() @@ -94,11 +91,9 @@ pub fn build_rolegroup_config_map( &validated_cluster.cluster_config.pod_descriptors, opa_connect.as_deref(), kraft_mode, - kafka - .spec + validated_cluster .cluster_config - .broker_id_pod_config_map_name - .is_some(), + .disable_broker_id_generation, config_overrides, ), AnyConfig::Controller(_) => { @@ -144,7 +139,7 @@ pub fn build_rolegroup_config_map( })?, ) .add_data( - JVM_SECURITY_PROPERTIES_FILE, + ConfigFileName::Security.to_string(), to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { JvmSecurityPropertiesSnafu { rolegroup: rolegroup.role_group.clone(), @@ -152,7 +147,7 @@ pub fn build_rolegroup_config_map( })?, ) .add_data( - "client.properties", + ConfigFileName::Client.to_string(), to_java_properties_string( kafka_security .client_properties() @@ -168,7 +163,7 @@ pub fn build_rolegroup_config_map( // It is processed by `config-utils` to substitute "env:" and "file:" variables // and this tool currently doesn't support the JAAS login configuration format. .add_data( - "jaas.properties", + ConfigFileName::Jaas.to_string(), jaas_config_file(kafka_security.has_kerberos_enabled()), ); diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs index 58a90499..598cb20e 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -9,8 +9,9 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster, build_recommended_labels}, + controller::ValidatedCluster, crd::{role::KafkaRole, v1alpha1}, + kafka_controller::{KAFKA_CONTROLLER_NAME, build_recommended_labels}, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/controller/build/properties/logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs index 76570a91..da198541 100644 --- a/rust/operator-binary/src/controller/build/properties/logging.rs +++ b/rust/operator-binary/src/controller/build/properties/logging.rs @@ -13,12 +13,12 @@ use stackable_operator::{ }; use crate::{ - controller::MAX_KAFKA_LOG_FILES_SIZE, crd::{ - LOG4J_CONFIG_FILE, LOG4J2_CONFIG_FILE, STACKABLE_LOG_DIR, + ConfigFileName, STACKABLE_LOG_DIR, role::{AnyConfig, broker::BrokerContainer, controller::ControllerContainer}, v1alpha1, }, + kafka_controller::MAX_KAFKA_LOG_FILES_SIZE, }; const KAFKA_LOG4J_FILE: &str = "kafka.log4j.xml"; @@ -44,7 +44,7 @@ pub fn role_group_config_map_data( match product_version.starts_with("3.") { true => { configs.insert( - LOG4J_CONFIG_FILE.to_string(), + ConfigFileName::Log4j.to_string(), log4j_config_if_automatic( Some(merged_config.kafka_logging()), container_name, @@ -55,7 +55,7 @@ pub fn role_group_config_map_data( } false => { configs.insert( - LOG4J2_CONFIG_FILE.to_string(), + ConfigFileName::Log4j2.to_string(), log4j2_config_if_automatic( Some(merged_config.kafka_logging()), container_name, diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs new file mode 100644 index 00000000..98ffb4f8 --- /dev/null +++ b/rust/operator-binary/src/controller/mod.rs @@ -0,0 +1,137 @@ +//! The validated cluster model and the steps that produce it. +//! +//! [`ValidatedCluster`] carries everything the build steps need, resolved once during +//! [`validate`] (after [`dereference`]) so downstream code never re-derives it or +//! touches the raw [`v1alpha1::KafkaCluster`] spec. The reconcile loop that consumes +//! it lives in [`crate::kafka_controller`]. + +use std::{borrow::Cow, collections::BTreeMap}; + +use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, + kube::{Resource, api::ObjectMeta}, + v2::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, +}; + +pub(crate) mod build; +pub(crate) mod dereference; +pub(crate) mod validate; + +use crate::{ + crd::{ + KafkaPodDescriptor, MetadataManager, + authorization::KafkaAuthorizationConfig, + role::{AnyConfig, AnyConfigOverrides, KafkaRole}, + security::KafkaTlsSecurity, + v1alpha1, + }, + framework::role_utils::RoleGroupConfig, +}; + +pub type RoleGroupName = String; + +/// The validated cluster. Carries everything the build steps need, resolved once +/// here so downstream code never re-derives it or touches the raw spec. +/// +/// The cluster identity (`name`, `namespace`, `uid`) is captured here so that owner +/// references for child objects can be built straight from this struct (via its +/// [`Resource`] impl) without threading the raw [`v1alpha1::KafkaCluster`] around. +/// This mirrors the hive-/opensearch-operator's `ValidatedCluster`. +pub struct ValidatedCluster { + /// `ObjectMeta` carrying `name`, `namespace` and `uid`, so this struct can act as the + /// owner [`Resource`] for child objects. + metadata: ObjectMeta, + pub name: ClusterName, + pub namespace: NamespaceName, + pub image: ResolvedProductImage, + pub cluster_config: ValidatedClusterConfig, + pub role_group_configs: BTreeMap>, +} + +impl ValidatedCluster { + pub fn new( + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + image: ResolvedProductImage, + cluster_config: ValidatedClusterConfig, + role_group_configs: BTreeMap>, + ) -> Self { + Self { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }, + name, + namespace, + image, + cluster_config, + role_group_configs, + } + } +} + +/// Cluster-wide settings resolved during validation and dereferencing. +/// +/// Everything the build steps need is resolved here so they never have to read the +/// raw [`v1alpha1::KafkaCluster`] spec. +pub struct ValidatedClusterConfig { + pub kafka_security: KafkaTlsSecurity, + pub authorization_config: Option, + pub pod_descriptors: Vec, + pub metadata_manager: MetadataManager, + + /// Whether the operator must not generate broker ids itself, because the user + /// supplied a `broker_id_pod_config_map_name`. Resolved from the raw spec during + /// validation so the config-map builder never has to read it. + pub disable_broker_id_generation: bool, +} + +/// Lets [`ValidatedCluster`] act as the owner [`Resource`] for child objects, so owner +/// references are built from it (via the captured `metadata`) rather than the raw CR. +impl Resource for ValidatedCluster { + type DynamicType = ::DynamicType; + type Scope = ::Scope; + + fn kind(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::kind(dt) + } + + fn group(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::group(dt) + } + + fn version(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::version(dt) + } + + fn plural(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::plural(dt) + } + + fn meta(&self) -> &ObjectMeta { + &self.metadata + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} + +/// A validated, merged Kafka role-group config. +/// +/// The merged config fragment is wrapped in [`AnyConfig`] and the merged +/// `configOverrides` in [`AnyConfigOverrides`], so a single role-agnostic type +/// carries both broker and controller role groups (their concrete config and +/// override types differ). Produced via the local-`framework` +/// [`with_validated_config`](crate::framework::role_utils::with_validated_config). +pub type ValidatedRoleGroupConfig = RoleGroupConfig< + AnyConfig, + stackable_operator::role_utils::JavaCommonConfig, + AnyConfigOverrides, +>; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 1f60f095..5c5edc06 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -191,6 +191,11 @@ pub fn validate( authorization_config: dereferenced_objects.authorization_config, pod_descriptors, metadata_manager, + disable_broker_id_generation: kafka + .spec + .cluster_config + .broker_id_pod_config_map_name + .is_some(), }, role_group_configs, )) diff --git a/rust/operator-binary/src/crd/config_file.rs b/rust/operator-binary/src/crd/config_file.rs new file mode 100644 index 00000000..1801be9d --- /dev/null +++ b/rust/operator-binary/src/crd/config_file.rs @@ -0,0 +1,51 @@ +//! The names of the config files assembled into the rolegroup `ConfigMap`. +//! +//! A single source of truth for the on-disk file names, used by the config-map +//! builder, the per-file property builders, the JVM/command builders and +//! [`AnyConfig::config_file_name`](crate::crd::role::AnyConfig::config_file_name). +//! Mirrors the hive-operator's `ConfigFileName`. + +/// The names of the Kafka config files assembled into the rolegroup `ConfigMap`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] +pub enum ConfigFileName { + #[strum(serialize = "broker.properties")] + BrokerProperties, + #[strum(serialize = "controller.properties")] + ControllerProperties, + #[strum(serialize = "security.properties")] + Security, + #[strum(serialize = "client.properties")] + Client, + /// JAAS configuration for Kerberos authentication. It has the `.properties` + /// extension but is not a Java properties file. + #[strum(serialize = "jaas.properties")] + Jaas, + /// Used by Kafka 3.x. + #[strum(serialize = "log4j.properties")] + Log4j, + /// Used by Kafka 4.0 and later. + #[strum(serialize = "log4j2.properties")] + Log4j2, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_names_match_the_kafka_on_disk_names() { + assert_eq!( + ConfigFileName::BrokerProperties.to_string(), + "broker.properties" + ); + assert_eq!( + ConfigFileName::ControllerProperties.to_string(), + "controller.properties" + ); + assert_eq!(ConfigFileName::Security.to_string(), "security.properties"); + assert_eq!(ConfigFileName::Client.to_string(), "client.properties"); + assert_eq!(ConfigFileName::Jaas.to_string(), "jaas.properties"); + assert_eq!(ConfigFileName::Log4j.to_string(), "log4j.properties"); + assert_eq!(ConfigFileName::Log4j2.to_string(), "log4j2.properties"); + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 740bc9ad..01a38ffd 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,6 +1,7 @@ pub mod affinity; pub mod authentication; pub mod authorization; +pub mod config_file; pub mod listener; pub mod role; pub mod security; @@ -9,6 +10,7 @@ pub mod tls; use std::collections::{BTreeMap, HashMap}; use authentication::KafkaAuthentication; +pub use config_file::ConfigFileName; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, Snafu}; use stackable_operator::{ @@ -44,8 +46,6 @@ pub const FIELD_MANAGER: &str = "kafka-operator"; // metrics pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 9606; -// config files -pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; // env vars pub const KAFKA_HEAP_OPTS: &str = "KAFKA_HEAP_OPTS"; // server_properties @@ -64,8 +64,6 @@ pub const STACKABLE_KERBEROS_KRB5_PATH: &str = "/stackable/kerberos/krb5.conf"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config"; pub const BROKER_ID_POD_MAP_DIR: &str = "/stackable/broker-id-pod-map"; -pub const LOG4J_CONFIG_FILE: &str = "log4j.properties"; -pub const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; #[derive(Snafu, Debug)] pub enum Error { diff --git a/rust/operator-binary/src/crd/role/broker.rs b/rust/operator-binary/src/crd/role/broker.rs index 64ebf848..67396876 100644 --- a/rust/operator-binary/src/crd/role/broker.rs +++ b/rust/operator-binary/src/crd/role/broker.rs @@ -13,8 +13,6 @@ use strum::{Display, EnumIter}; use crate::crd::role::commons::{CommonConfig, Storage, StorageFragment}; -pub const BROKER_PROPERTIES_FILE: &str = "broker.properties"; - #[derive( Clone, Debug, diff --git a/rust/operator-binary/src/crd/role/controller.rs b/rust/operator-binary/src/crd/role/controller.rs index 27756ee1..75f93cdb 100644 --- a/rust/operator-binary/src/crd/role/controller.rs +++ b/rust/operator-binary/src/crd/role/controller.rs @@ -13,8 +13,6 @@ use strum::{Display, EnumIter}; use crate::crd::role::commons::{CommonConfig, Storage, StorageFragment}; -pub const CONTROLLER_PROPERTIES_FILE: &str = "controller.properties"; - #[derive( Clone, Debug, diff --git a/rust/operator-binary/src/crd/role/mod.rs b/rust/operator-binary/src/crd/role/mod.rs index a7e05881..41cdcf1f 100644 --- a/rust/operator-binary/src/crd/role/mod.rs +++ b/rust/operator-binary/src/crd/role/mod.rs @@ -19,10 +19,13 @@ use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ config::jvm::{construct_heap_jvm_args, construct_non_heap_jvm_args}, - crd::role::{ - broker::{BROKER_PROPERTIES_FILE, BrokerConfig}, - commons::{CommonConfig, Storage}, - controller::{CONTROLLER_PROPERTIES_FILE, ControllerConfig}, + crd::{ + ConfigFileName, + role::{ + broker::BrokerConfig, + commons::{CommonConfig, Storage}, + controller::ControllerConfig, + }, }, v1alpha1, }; @@ -355,10 +358,10 @@ impl AnyConfig { } } - pub fn config_file_name(&self) -> &str { + pub fn config_file_name(&self) -> ConfigFileName { match self { - AnyConfig::Broker(_) => BROKER_PROPERTIES_FILE, - AnyConfig::Controller(_) => CONTROLLER_PROPERTIES_FILE, + AnyConfig::Broker(_) => ConfigFileName::BrokerProperties, + AnyConfig::Controller(_) => ConfigFileName::ControllerProperties, } } } diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/kafka_controller.rs similarity index 78% rename from rust/operator-binary/src/controller.rs rename to rust/operator-binary/src/kafka_controller.rs index 09d16d4f..a1c0bf54 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/kafka_controller.rs @@ -1,17 +1,17 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha1::KafkaCluster`]. -use std::{borrow::Cow, collections::BTreeMap, sync::Arc}; +use std::sync::Arc; use const_format::concatcp; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, - commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, + commons::rbac::build_rbac_resources, crd::listener, kube::{ Resource, - api::{DynamicObject, ObjectMeta}, + api::DynamicObject, core::{DeserializeGuard, error_boundary}, runtime::{controller::Action, reflector::ObjectRef}, }, @@ -24,24 +24,15 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - v2::types::{ - kubernetes::{NamespaceName, Uid}, - operator::ClusterName, - }, }; use strum::{EnumDiscriminants, IntoStaticStr}; -pub(crate) mod build; -mod dereference; -mod validate; - use crate::{ + controller::{build, dereference, validate}, crd::{ - APP_NAME, KafkaClusterStatus, KafkaPodDescriptor, MetadataManager, OPERATOR_NAME, - authorization::KafkaAuthorizationConfig, + APP_NAME, KafkaClusterStatus, OPERATOR_NAME, listener::get_kafka_listener_config, - role::{AnyConfig, AnyConfigOverrides, KafkaRole}, - security::KafkaTlsSecurity, + role::{AnyConfig, KafkaRole}, v1alpha1, }, operations::pdb::add_pdbs, @@ -235,106 +226,6 @@ impl ReconcilerError for Error { } } -pub type RoleGroupName = String; - -/// The validated cluster. Carries everything the build steps need, resolved once -/// here so downstream code never re-derives it or touches the raw spec. -/// -/// The cluster identity (`name`, `namespace`, `uid`) is captured here so that owner -/// references for child objects can be built straight from this struct (via its -/// [`Resource`] impl) without threading the raw [`v1alpha1::KafkaCluster`] around. -/// This mirrors the hive-/opensearch-operator's `ValidatedCluster`. -pub struct ValidatedCluster { - /// `ObjectMeta` carrying `name`, `namespace` and `uid`, so this struct can act as the - /// owner [`Resource`] for child objects. - metadata: ObjectMeta, - pub name: ClusterName, - pub namespace: NamespaceName, - pub image: ResolvedProductImage, - pub cluster_config: ValidatedClusterConfig, - pub role_group_configs: BTreeMap>, -} - -impl ValidatedCluster { - pub fn new( - name: ClusterName, - namespace: NamespaceName, - uid: Uid, - image: ResolvedProductImage, - cluster_config: ValidatedClusterConfig, - role_group_configs: BTreeMap>, - ) -> Self { - Self { - metadata: ObjectMeta { - name: Some(name.to_string()), - namespace: Some(namespace.to_string()), - uid: Some(uid.to_string()), - ..ObjectMeta::default() - }, - name, - namespace, - image, - cluster_config, - role_group_configs, - } - } -} - -/// Cluster-wide settings resolved during validation and dereferencing. -/// -/// Everything the build steps need is resolved here so they never have to read the -/// raw [`v1alpha1::KafkaCluster`] spec. -pub struct ValidatedClusterConfig { - pub kafka_security: KafkaTlsSecurity, - pub authorization_config: Option, - pub pod_descriptors: Vec, - pub metadata_manager: MetadataManager, -} - -/// Lets [`ValidatedCluster`] act as the owner [`Resource`] for child objects, so owner -/// references are built from it (via the captured `metadata`) rather than the raw CR. -impl Resource for ValidatedCluster { - type DynamicType = ::DynamicType; - type Scope = ::Scope; - - fn kind(dt: &Self::DynamicType) -> Cow<'_, str> { - v1alpha1::KafkaCluster::kind(dt) - } - - fn group(dt: &Self::DynamicType) -> Cow<'_, str> { - v1alpha1::KafkaCluster::group(dt) - } - - fn version(dt: &Self::DynamicType) -> Cow<'_, str> { - v1alpha1::KafkaCluster::version(dt) - } - - fn plural(dt: &Self::DynamicType) -> Cow<'_, str> { - v1alpha1::KafkaCluster::plural(dt) - } - - fn meta(&self) -> &ObjectMeta { - &self.metadata - } - - fn meta_mut(&mut self) -> &mut ObjectMeta { - &mut self.metadata - } -} - -/// A validated, merged Kafka role-group config. -/// -/// The merged config fragment is wrapped in [`AnyConfig`] and the merged -/// `configOverrides` in [`AnyConfigOverrides`], so a single role-agnostic type -/// carries both broker and controller role groups (their concrete config and -/// override types differ). Produced via the local-`framework` -/// [`with_validated_config`](crate::framework::role_utils::with_validated_config). -pub type ValidatedRoleGroupConfig = crate::framework::role_utils::RoleGroupConfig< - AnyConfig, - stackable_operator::role_utils::JavaCommonConfig, - AnyConfigOverrides, ->; - pub async fn reconcile_kafka( kafka: Arc>, ctx: Arc, @@ -427,7 +318,6 @@ pub async fn reconcile_kafka( .context(InvalidKafkaListenersSnafu)?; let rg_configmap = build::config_map::build_rolegroup_config_map( - kafka, &validated_cluster, &rolegroup_ref, validated_rg, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 21b2f466..fd1e1b9a 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -35,8 +35,8 @@ use stackable_operator::{ }; use crate::{ - controller::KAFKA_FULL_CONTROLLER_NAME, crd::{KafkaCluster, KafkaClusterVersion, OPERATOR_NAME, v1alpha1}, + kafka_controller::KAFKA_FULL_CONTROLLER_NAME, webhooks::conversion::create_webhook_server, }; @@ -44,6 +44,7 @@ mod config; mod controller; mod crd; mod framework; +mod kafka_controller; mod kerberos; mod operations; mod resource; @@ -176,9 +177,9 @@ async fn main() -> anyhow::Result<()> { ) .graceful_shutdown_on(sigterm_watcher.handle()) .run( - controller::reconcile_kafka, - controller::error_policy, - Arc::new(controller::Ctx { + kafka_controller::reconcile_kafka, + kafka_controller::error_policy, + Arc::new(kafka_controller::Ctx { client: client.clone(), operator_environment, }), diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index e42888ca..18f46dc7 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -5,8 +5,8 @@ use stackable_operator::{ }; use crate::{ - controller::KAFKA_CONTROLLER_NAME, crd::{APP_NAME, OPERATOR_NAME, role::KafkaRole, v1alpha1}, + kafka_controller::KAFKA_CONTROLLER_NAME, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/resource/listener.rs b/rust/operator-binary/src/resource/listener.rs index f85245ec..d22a2a96 100644 --- a/rust/operator-binary/src/resource/listener.rs +++ b/rust/operator-binary/src/resource/listener.rs @@ -4,8 +4,9 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster, build_recommended_labels}, + controller::ValidatedCluster, crd::{role::broker::BrokerConfig, security::KafkaTlsSecurity, v1alpha1}, + kafka_controller::{KAFKA_CONTROLLER_NAME, build_recommended_labels}, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/resource/service.rs b/rust/operator-binary/src/resource/service.rs index 5b3802d3..8f4fa0e3 100644 --- a/rust/operator-binary/src/resource/service.rs +++ b/rust/operator-binary/src/resource/service.rs @@ -8,8 +8,9 @@ use stackable_operator::{ }; use crate::{ - controller::{KAFKA_CONTROLLER_NAME, ValidatedCluster, build_recommended_labels}, + controller::ValidatedCluster, crd::{APP_NAME, METRICS_PORT, METRICS_PORT_NAME, security::KafkaTlsSecurity, v1alpha1}, + kafka_controller::{KAFKA_CONTROLLER_NAME, build_recommended_labels}, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index 330331af..3df261a2 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -46,10 +46,7 @@ use crate::{ }, node_id_hasher::node_id_hash32_offset, }, - controller::{ - KAFKA_CONTROLLER_NAME, MAX_KAFKA_LOG_FILES_SIZE, ValidatedCluster, - ValidatedRoleGroupConfig, build_recommended_labels, - }, + controller::{ValidatedCluster, ValidatedRoleGroupConfig}, crd::{ self, APP_NAME, BROKER_ID_POD_MAP_DIR, KAFKA_HEAP_OPTS, LISTENER_BOOTSTRAP_VOLUME_NAME, LISTENER_BROKER_VOLUME_NAME, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, @@ -63,6 +60,7 @@ use crate::{ security::KafkaTlsSecurity, v1alpha1, }, + kafka_controller::{KAFKA_CONTROLLER_NAME, MAX_KAFKA_LOG_FILES_SIZE, build_recommended_labels}, kerberos::add_kerberos_pod_config, operations::graceful_shutdown::add_graceful_shutdown_config, }; From c72aa851188375feb40782ace47f53d2c7e36263 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 08:49:00 +0200 Subject: [PATCH 29/33] refactor: make better use of ValidatedCluster in property file builders --- .../src/controller/build/config_map.rs | 28 ++++--------------- .../build/properties/broker_properties.rs | 21 ++++++-------- .../build/properties/controller_properties.rs | 13 ++++----- rust/operator-binary/src/controller/mod.rs | 14 ++++++++++ 4 files changed, 33 insertions(+), 43 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 919f3a66..3db8337e 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -13,8 +13,7 @@ use crate::{ build::properties::logging::role_group_config_map_data, }, crd::{ - ConfigFileName, MetadataManager, STACKABLE_LISTENER_BOOTSTRAP_DIR, - STACKABLE_LISTENER_BROKER_DIR, + ConfigFileName, STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, listener::{KafkaListenerConfig, node_address_cmd}, role::AnyConfig, v1alpha1, @@ -63,7 +62,8 @@ pub fn build_rolegroup_config_map( validated_rg: &ValidatedRoleGroupConfig, listener_config: &KafkaListenerConfig, ) -> Result { - let kafka_security = &validated_cluster.cluster_config.kafka_security; + let cluster_config = &validated_cluster.cluster_config; + let kafka_security = &cluster_config.kafka_security; let resolved_product_image = &validated_cluster.image; let kafka_config_file_name = validated_rg.config.config_file_name().to_string(); let config_overrides = validated_rg @@ -72,36 +72,20 @@ pub fn build_rolegroup_config_map( .overrides .clone(); - let opa_connect = validated_cluster - .cluster_config - .authorization_config - .as_ref() - .map(|auth_config| auth_config.opa_connect.clone()); - - let kraft_mode = validated_cluster.cluster_config.metadata_manager == MetadataManager::KRaft; - - if kraft_mode && validated_cluster.cluster_config.pod_descriptors.is_empty() { + if cluster_config.is_kraft_mode() && cluster_config.pod_descriptors.is_empty() { return NoKraftControllersFoundSnafu.fail(); } let kafka_config = match &validated_rg.config { AnyConfig::Broker(_) => crate::controller::build::properties::broker_properties::build( - kafka_security, + cluster_config, listener_config, - &validated_cluster.cluster_config.pod_descriptors, - opa_connect.as_deref(), - kraft_mode, - validated_cluster - .cluster_config - .disable_broker_id_generation, config_overrides, ), AnyConfig::Controller(_) => { crate::controller::build::properties::controller_properties::build( - kafka_security, + cluster_config, listener_config, - &validated_cluster.cluster_config.pod_descriptors, - kraft_mode, config_overrides, ) } diff --git a/rust/operator-binary/src/controller/build/properties/broker_properties.rs b/rust/operator-binary/src/controller/build/properties/broker_properties.rs index 44840363..700e0822 100644 --- a/rust/operator-binary/src/controller/build/properties/broker_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/broker_properties.rs @@ -2,29 +2,24 @@ use std::collections::BTreeMap; use super::kraft_controllers; use crate::{ + controller::ValidatedClusterConfig, crd::{ - KafkaPodDescriptor, listener::{KafkaListenerConfig, KafkaListenerName}, role::{ KAFKA_ADVERTISED_LISTENERS, KAFKA_BROKER_ID, KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, KAFKA_PROCESS_ROLES, KafkaRole, }, - security::KafkaTlsSecurity, }, operations::graceful_shutdown::graceful_shutdown_config_properties, }; pub fn build( - kafka_security: &KafkaTlsSecurity, + cluster_config: &ValidatedClusterConfig, listener_config: &KafkaListenerConfig, - pod_descriptors: &[KafkaPodDescriptor], - opa_connect_string: Option<&str>, - kraft_mode: bool, - disable_broker_id_generation: bool, overrides: BTreeMap, ) -> BTreeMap { - let kraft_controllers = kraft_controllers(pod_descriptors); + let kraft_controllers = kraft_controllers(&cluster_config.pod_descriptors); let mut result = BTreeMap::from([ ( @@ -46,7 +41,7 @@ pub fn build( ), ]); - if kraft_mode { + if cluster_config.is_kraft_mode() { let kraft_controllers = kraft_controllers.join(","); // Running in KRaft mode @@ -79,7 +74,7 @@ pub fn build( // so we disable automatic id generation. // This check ensures that existing clusters running in ZooKeeper mode do not // suddenly break after the introduction of this change. - if disable_broker_id_generation { + if cluster_config.disable_broker_id_generation { result.extend([ ( "broker.id.generation.enable".to_string(), @@ -91,7 +86,7 @@ pub fn build( } // Enable OPA authorization - if opa_connect_string.is_some() { + if let Some(opa_connect_string) = cluster_config.opa_connect() { result.extend([ ( "authorizer.class.name".to_string(), @@ -103,12 +98,12 @@ pub fn build( ), ( "opa.authorizer.url".to_string(), - opa_connect_string.unwrap_or_default().to_string(), + opa_connect_string.to_string(), ), ]); } - result.extend(kafka_security.broker_config_settings()); + result.extend(cluster_config.kafka_security.broker_config_settings()); result.extend(graceful_shutdown_config_properties()); result.extend(overrides); diff --git a/rust/operator-binary/src/controller/build/properties/controller_properties.rs b/rust/operator-binary/src/controller/build/properties/controller_properties.rs index ba825aec..289d0cb2 100644 --- a/rust/operator-binary/src/controller/build/properties/controller_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/controller_properties.rs @@ -2,26 +2,23 @@ use std::collections::BTreeMap; use super::kraft_controllers; use crate::{ + controller::ValidatedClusterConfig, crd::{ - KafkaPodDescriptor, listener::{KafkaListenerConfig, KafkaListenerName}, role::{ KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, KAFKA_PROCESS_ROLES, KafkaRole, }, - security::KafkaTlsSecurity, }, operations::graceful_shutdown::graceful_shutdown_config_properties, }; pub fn build( - kafka_security: &KafkaTlsSecurity, + cluster_config: &ValidatedClusterConfig, listener_config: &KafkaListenerConfig, - pod_descriptors: &[KafkaPodDescriptor], - kraft_mode: bool, overrides: BTreeMap, ) -> BTreeMap { - let kraft_controllers = kraft_controllers(pod_descriptors).join(","); + let kraft_controllers = kraft_controllers(&cluster_config.pod_descriptors).join(","); let mut result = BTreeMap::from([ ( @@ -58,14 +55,14 @@ pub fn build( // The ZooKeeper connection is needed for migration from ZooKeeper to KRaft mode. // It is not needed once the controller is fully running in KRaft mode. - if !kraft_mode { + if !cluster_config.is_kraft_mode() { result.insert( "zookeeper.connect".to_string(), "${env:ZOOKEEPER}".to_string(), ); } - result.extend(kafka_security.controller_config_settings()); + result.extend(cluster_config.kafka_security.controller_config_settings()); result.extend(graceful_shutdown_config_properties()); result.extend(overrides); diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 98ffb4f8..2c907057 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -92,6 +92,20 @@ pub struct ValidatedClusterConfig { pub disable_broker_id_generation: bool, } +impl ValidatedClusterConfig { + /// Whether the cluster runs in KRaft mode (as opposed to ZooKeeper mode). + pub fn is_kraft_mode(&self) -> bool { + self.metadata_manager == MetadataManager::KRaft + } + + /// The OPA connect string, if OPA authorization is configured. + pub fn opa_connect(&self) -> Option<&str> { + self.authorization_config + .as_ref() + .map(|auth_config| auth_config.opa_connect.as_str()) + } +} + /// Lets [`ValidatedCluster`] act as the owner [`Resource`] for child objects, so owner /// references are built from it (via the captured `metadata`) rather than the raw CR. impl Resource for ValidatedCluster { From f580049730adf5718cf89647e7cb5da12b4a89e6 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:14:43 +0200 Subject: [PATCH 30/33] Update rust/operator-binary/src/crd/mod.rs Co-authored-by: maltesander --- rust/operator-binary/src/crd/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 01a38ffd..630595f6 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -245,9 +245,6 @@ pub mod versioned { pub broker_id_pod_config_map_name: Option, } - // Uses the v2 KeyValueConfigOverrides (plain string values) to match trino/hdfs. - // Derives `Merge` so role/role-group overrides combine via the shared merge logic; - // resolution into flat maps happens in controller/validate.rs. #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Merge, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KafkaBrokerConfigOverrides { From 18df1437145552c4009851dd5e9774441360195b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:16:53 +0200 Subject: [PATCH 31/33] Apply suggestions from code review Co-authored-by: maltesander --- rust/operator-binary/src/controller/build/discovery.rs | 1 - rust/operator-binary/src/crd/config_file.rs | 1 - rust/operator-binary/src/crd/role/broker.rs | 1 - rust/operator-binary/src/crd/role/controller.rs | 1 - 4 files changed, 4 deletions(-) diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs index 598cb20e..ef78a7e8 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -65,7 +65,6 @@ pub fn build_discovery_configmap( .metadata( ObjectMetaBuilder::new() .name_and_namespace(validated_cluster) - .name(validated_cluster.name.to_string()) .ownerreference_from_resource(validated_cluster, None, Some(true)) .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { kafka: cluster_object_ref(validated_cluster), diff --git a/rust/operator-binary/src/crd/config_file.rs b/rust/operator-binary/src/crd/config_file.rs index 1801be9d..aec47d39 100644 --- a/rust/operator-binary/src/crd/config_file.rs +++ b/rust/operator-binary/src/crd/config_file.rs @@ -3,7 +3,6 @@ //! A single source of truth for the on-disk file names, used by the config-map //! builder, the per-file property builders, the JVM/command builders and //! [`AnyConfig::config_file_name`](crate::crd::role::AnyConfig::config_file_name). -//! Mirrors the hive-operator's `ConfigFileName`. /// The names of the Kafka config files assembled into the rolegroup `ConfigMap`. #[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] diff --git a/rust/operator-binary/src/crd/role/broker.rs b/rust/operator-binary/src/crd/role/broker.rs index 67396876..97d09613 100644 --- a/rust/operator-binary/src/crd/role/broker.rs +++ b/rust/operator-binary/src/crd/role/broker.rs @@ -93,4 +93,3 @@ impl BrokerConfig { } } -// KAFKA_CLUSTER_ID injection moved to controller/validate.rs::collect_broker_role_group_overrides. diff --git a/rust/operator-binary/src/crd/role/controller.rs b/rust/operator-binary/src/crd/role/controller.rs index 75f93cdb..7e83c770 100644 --- a/rust/operator-binary/src/crd/role/controller.rs +++ b/rust/operator-binary/src/crd/role/controller.rs @@ -84,4 +84,3 @@ impl ControllerConfig { } } -// KAFKA_CLUSTER_ID injection moved to controller/validate.rs::collect_controller_role_group_overrides. From f3a1ae12e9ab9e78a5b3f180b985ce16007fe451 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 11 Jun 2026 09:42:58 +0200 Subject: [PATCH 32/33] linting --- rust/operator-binary/src/crd/role/broker.rs | 1 - rust/operator-binary/src/crd/role/controller.rs | 1 - rust/operator-binary/src/kafka_controller.rs | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/rust/operator-binary/src/crd/role/broker.rs b/rust/operator-binary/src/crd/role/broker.rs index 97d09613..2024d519 100644 --- a/rust/operator-binary/src/crd/role/broker.rs +++ b/rust/operator-binary/src/crd/role/broker.rs @@ -92,4 +92,3 @@ impl BrokerConfig { } } } - diff --git a/rust/operator-binary/src/crd/role/controller.rs b/rust/operator-binary/src/crd/role/controller.rs index 7e83c770..fbaf898c 100644 --- a/rust/operator-binary/src/crd/role/controller.rs +++ b/rust/operator-binary/src/crd/role/controller.rs @@ -83,4 +83,3 @@ impl ControllerConfig { } } } - diff --git a/rust/operator-binary/src/kafka_controller.rs b/rust/operator-binary/src/kafka_controller.rs index a1c0bf54..e1186e10 100644 --- a/rust/operator-binary/src/kafka_controller.rs +++ b/rust/operator-binary/src/kafka_controller.rs @@ -55,7 +55,7 @@ pub const MAX_KAFKA_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { /// Build recommended values for labels. /// /// Generic over the owner `T` so the owner can be either the raw `KafkaCluster` or the -/// [`ValidatedCluster`] (which also implements `Resource`). +/// `ValidatedCluster` (which also implements `Resource`). pub fn build_recommended_labels<'a, T>( owner: &'a T, controller_name: &'a str, From c3a263e4db39379f9e4e24a682746ba888bf2dab Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 11 Jun 2026 09:44:04 +0200 Subject: [PATCH 33/33] cleaned up comment --- rust/operator-binary/src/controller/validate.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 5c5edc06..973a8ce1 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -254,10 +254,7 @@ where /// Injects the operator-managed `KAFKA_CLUSTER_ID` into the merged env overrides, /// but only when the user has not already set it via `envOverrides` (user value -/// wins, preserving product-config's old precedence). -/// -/// `KAFKA_CLUSTER_ID` injection moved here from the now-removed -/// `crd::role::*::Configuration::compute_env`. +/// wins). fn inject_cluster_id(env_overrides: EnvVarSet, cluster_id: Option<&str>) -> Result { let Some(cluster_id) = cluster_id else { return Ok(env_overrides);