diff --git a/CHANGELOG.md b/CHANGELOG.md index 323d05c6..75204042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ All notable changes to this project will be documented in this file. - Bump `stackable-operator` to 0.111.1 and snafu to 0.9 ([#960], [#961]). - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#968]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#971]). +- BREAKING: Removed product-config machinery which is a breaking change in terms of configuration. + Users relying on the product-config `properties.yaml` file have to set these properties via the CRD. + Config and environment overrides are now merged directly from the CRD into the validated cluster. + The `--product-config` CLI flag is now a no-op ([#976]). [#953]: https://github.com/stackabletech/kafka-operator/pull/953 [#960]: https://github.com/stackabletech/kafka-operator/pull/960 @@ -26,6 +30,7 @@ All notable changes to this project will be documented in this file. [#968]: https://github.com/stackabletech/kafka-operator/pull/968 [#971]: https://github.com/stackabletech/kafka-operator/pull/971 [#973]: https://github.com/stackabletech/kafka-operator/pull/973 +[#976]: https://github.com/stackabletech/kafka-operator/pull/976 ## [26.3.0] - 2026-03-16 diff --git a/Cargo.lock b/Cargo.lock index 5357bf1f..92a70017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1473,6 +1473,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" dependencies = [ "jsonptr", + "schemars", "serde", "serde_json", "thiserror 1.0.69", @@ -1517,11 +1518,11 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "darling", "regex", - "snafu 0.9.0", + "snafu 0.9.1", ] [[package]] @@ -1842,9 +1843,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -1856,9 +1857,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" dependencies = [ "opentelemetry", "tracing", @@ -1868,9 +1869,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", @@ -1881,9 +1882,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -1895,14 +1896,14 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1913,21 +1914,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", "tokio", @@ -2210,6 +2212,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.45" @@ -2349,9 +2360,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -2367,9 +2378,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -2812,11 +2820,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ - "snafu-derive 0.9.0", + "snafu-derive 0.9.1", ] [[package]] @@ -2844,9 +2852,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -2889,7 +2897,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "const-oid", "ecdsa", @@ -2901,7 +2909,7 @@ dependencies = [ "rsa", "sha2", "signature", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-shared", "tokio", "tokio-rustls", @@ -2920,12 +2928,11 @@ dependencies = [ "const_format", "futures", "indoc", - "product-config", "rstest", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator", "strum", "tokio", @@ -2935,7 +2942,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "base64", "clap", @@ -2947,6 +2954,7 @@ dependencies = [ "futures", "http", "indexmap", + "java-properties", "jiff", "json-patch", "k8s-openapi", @@ -2959,7 +2967,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -2971,12 +2979,14 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "url", + "uuid", + "xml", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "darling", "proc-macro2", @@ -2986,8 +2996,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +version = "0.1.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "jiff", "k8s-openapi", @@ -2996,15 +3006,15 @@ dependencies = [ "semver", "serde", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "time", ] [[package]] name = "stackable-telemetry" -version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +version = "0.6.4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "axum", "clap", @@ -3015,7 +3025,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "pin-project", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "tokio", "tower", @@ -3028,21 +3038,21 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "kube", "schemars", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "convert_case", "convert_case_extras", @@ -3060,7 +3070,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "arc-swap", "async-trait", @@ -3076,7 +3086,7 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-certs", "stackable-shared", "stackable-telemetry", @@ -3412,6 +3422,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a875a902255423d34c1f20838ab374126db8eb41625b7947a1d54113b0b7399" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3523,9 +3544,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", @@ -3641,6 +3662,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3936,9 +3967,9 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" diff --git a/Cargo.nix b/Cargo.nix index 73891047..43e426e5 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4690,6 +4690,11 @@ rec { name = "jsonptr"; packageId = "jsonptr"; } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } { name = "serde"; packageId = "serde"; @@ -4705,6 +4710,10 @@ rec { } ]; devDependencies = [ + { + name = "schemars"; + packageId = "schemars"; + } { name = "serde_json"; packageId = "serde_json"; @@ -4716,7 +4725,7 @@ rec { "schemars" = [ "dep:schemars" ]; "utoipa" = [ "dep:utoipa" ]; }; - resolvedDefaultFeatures = [ "default" "diff" ]; + resolvedDefaultFeatures = [ "default" "diff" "schemars" ]; }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; @@ -4842,9 +4851,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "k8s_version"; authors = [ @@ -4862,7 +4871,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } ]; features = { @@ -6043,9 +6052,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "18629xsj4rsyiby9aj511q6wcw6s9m09gx3ymw1yjcvix1mcsjxq"; + sha256 = "10ln14d1jgc8rvw97mblc9blzcgpg1bimim4d170b7ia4mijq55h"; dependencies = [ { name = "futures-core"; @@ -6082,24 +6091,24 @@ rec { ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "experimental_metrics_bound_instruments" = [ "metrics" ]; "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; "pin-project-lite" = [ "dep:pin-project-lite" ]; - "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "1hnwizzgfhpjfnvml638yy846py8hf2gl1n3p1igbk1srb2ilspg"; + sha256 = "0dyq4myan64sl8wly02jx0gb3jjz7575mn3w8rpphz0xvkq8001c"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6142,18 +6151,15 @@ rec { ]; features = { "experimental_metadata_attributes" = [ "dep:tracing-log" ]; - "experimental_use_tracing_span_context" = [ "tracing-opentelemetry" ]; "log" = [ "dep:log" ]; - "spec_unstable_logs_enabled" = [ "opentelemetry/spec_unstable_logs_enabled" ]; - "tracing-opentelemetry" = [ "dep:tracing-opentelemetry" ]; }; resolvedDefaultFeatures = [ "default" ]; }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0pc5nw1ds8v8w0nvyall39m92v8m1xl1p3vwvxk6nkhrffdd19np"; + sha256 = "0ca3drvm4fx5nskl7yn42dimy3bg35ppzc85y1p27pz215fh30sn"; libName = "opentelemetry_http"; dependencies = [ { @@ -6189,16 +6195,16 @@ rec { "internal-logs" = [ "opentelemetry/internal-logs" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking" = [ "dep:reqwest" "reqwest/blocking" ]; - "reqwest-rustls" = [ "dep:reqwest" "reqwest/rustls-tls-native-roots" ]; - "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/rustls-tls-webpki-roots" ]; + "reqwest-rustls" = [ "dep:reqwest" "reqwest/default-tls" ]; + "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/default-tls" "reqwest/webpki-roots" ]; }; - resolvedDefaultFeatures = [ "internal-logs" "reqwest" "reqwest-blocking" ]; + resolvedDefaultFeatures = [ "reqwest" "reqwest-blocking" ]; }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "07zp0b62b9dajnvvcd6j2ppw5zg7wp4ixka9z6fr3bxrrdmcss8z"; + sha256 = "0d9cys2flpidfxbr6h1103hjc633cax47ihnqgbj0xnicscr4rlr"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6259,10 +6265,9 @@ rec { usesDefaultFeatures = false; } { - name = "tracing"; - packageId = "tracing"; + name = "tonic-types"; + packageId = "tonic-types"; optional = true; - usesDefaultFeatures = false; } ]; devDependencies = [ @@ -6287,16 +6292,19 @@ rec { ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; + "experimental-grpc-retry" = [ "grpc-tonic" "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" ]; + "experimental-http-retry" = [ "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" "tokio" "httpdate" ]; "flate2" = [ "dep:flate2" ]; - "grpc-tonic" = [ "tonic" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; + "grpc-tonic" = [ "tonic" "tonic-types" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; "gzip-http" = [ "flate2" ]; "gzip-tonic" = [ "tonic/gzip" ]; "http" = [ "dep:http" ]; "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "trace" "metrics" ]; "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; + "httpdate" = [ "dep:httpdate" ]; "hyper-client" = [ "opentelemetry-http/hyper" ]; "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" "logs" ]; - "internal-logs" = [ "tracing" "opentelemetry_sdk/internal-logs" "opentelemetry-http/internal-logs" ]; + "internal-logs" = [ "opentelemetry_sdk/internal-logs" "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "opentelemetry-proto/metrics" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; @@ -6309,27 +6317,27 @@ rec { "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "serialize" = [ "serde" "serde_json" ]; - "tls" = [ "tonic/tls-ring" ]; + "tls" = [ "tls-ring" ]; "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; "tls-provider-agnostic" = [ "tonic/_tls-any" ]; "tls-ring" = [ "tonic/tls-ring" ]; - "tls-roots" = [ "tls" "tonic/tls-native-roots" ]; - "tls-webpki-roots" = [ "tls" "tonic/tls-webpki-roots" ]; + "tls-roots" = [ "tonic/tls-native-roots" ]; + "tls-webpki-roots" = [ "tonic/tls-webpki-roots" ]; "tokio" = [ "dep:tokio" ]; "tonic" = [ "dep:tonic" ]; + "tonic-types" = [ "dep:tonic-types" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" "opentelemetry-proto/trace" ]; - "tracing" = [ "dep:tracing" ]; "zstd" = [ "dep:zstd" ]; "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "03xkjsjrsm7zkkx5gascqd9bg2z20wymm06l16cyxsp5dpq5s5x7"; + sha256 = "0f5ny4rpnpq6q5q34b8k2q548rf31rpbxkwjqjwzfqxg3yx5imjn"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6373,30 +6381,29 @@ rec { "const-hex" = [ "dep:const-hex" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; - "gen-tonic-messages" = [ "tonic" "tonic-prost" "prost" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic" "tonic-prost" "tonic/channel" ]; + "gen-tonic-messages" = [ "prost" ]; "internal-logs" = [ "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" ]; "prost" = [ "dep:prost" ]; "schemars" = [ "dep:schemars" ]; "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; "testing" = [ "opentelemetry/testing" ]; "tonic" = [ "dep:tonic" ]; "tonic-prost" = [ "dep:tonic-prost" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" ]; "with-schemars" = [ "schemars" ]; - "with-serde" = [ "serde" "const-hex" "base64" "serde_json" ]; + "with-serde" = [ "serde" "const-hex" "base64" ]; "zpages" = [ "trace" ]; }; resolvedDefaultFeatures = [ "gen-tonic" "gen-tonic-messages" "logs" "metrics" "prost" "tonic" "tonic-prost" "trace" ]; }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0in8plv2l2ar7anzi7lrbll0fjfvaymkg5vc5bnvibs1w3gjjbp6"; + sha256 = "0s1x4h1cgmhkxb7i5g02la2vhkf4lg5g26cgn2s2gd1p0j5gk8kc"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -6404,9 +6411,9 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.31.0"; + version = "0.32.1"; edition = "2021"; - sha256 = "1gbjsggdxfpjbanjvaxa3nq32vfa37i3v13dvx4gsxhrk7sy8jp1"; + sha256 = "1ycl11syranrinhgn4c2hlzhyzyvpa06ryxq5mxgzmf4387ghncv"; dependencies = [ { name = "futures-channel"; @@ -6432,6 +6439,13 @@ rec { packageId = "percent-encoding"; optional = true; } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + usesDefaultFeatures = false; + target = { target, features }: (!("64" == target."has_atomic" or null)); + features = [ "fallback" ]; + } { name = "rand"; packageId = "rand 0.9.4"; @@ -6456,10 +6470,18 @@ rec { optional = true; } ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "macros" "rt-multi-thread" ]; + } + ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" "experimental_async_runtime" ]; - "experimental_logs_concurrent_log_processor" = [ "logs" ]; + "experimental_metrics_bound_instruments" = [ "metrics" "opentelemetry/experimental_metrics_bound_instruments" ]; "experimental_metrics_custom_reader" = [ "metrics" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" "experimental_async_runtime" ]; @@ -6476,15 +6498,14 @@ rec { "rt-tokio-current-thread" = [ "tokio/rt" "tokio/time" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; - "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "tokio/sync" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -6993,7 +7014,7 @@ rec { "default" = [ "fallback" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "require-cas" ]; + resolvedDefaultFeatures = [ "fallback" "require-cas" ]; }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; @@ -7261,6 +7282,34 @@ rec { ]; }; + "prost-types" = rec { + crateName = "prost-types"; + version = "0.14.3"; + edition = "2021"; + sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + libName = "prost_types"; + authors = [ + "Dan Burkert " + "Lucio Franco " + "Casper Meijn " + "Tokio Contributors " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "chrono" = [ "dep:chrono" ]; + "default" = [ "std" ]; + "std" = [ "prost/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "quote" = rec { crateName = "quote"; version = "1.0.45"; @@ -7690,9 +7739,9 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.12.28"; + version = "0.13.4"; edition = "2021"; - sha256 = "0iqidijghgqbzl3bjg5hb4zmigwa4r612bgi0yiq0c90b6jkrpgd"; + sha256 = "1hy1plns9krbh3h1dy2sdjygsfkdcnxm6pbxdi0ya9b5vq8mi711"; authors = [ "Sean McArthur " ]; @@ -7709,7 +7758,7 @@ rec { name = "futures-channel"; packageId = "futures-channel"; optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "futures-core"; @@ -7729,62 +7778,44 @@ rec { { name = "http-body"; packageId = "http-body"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "http-body-util"; packageId = "http-body-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "hyper"; packageId = "hyper"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" "client-legacy" "client-proxy" "tokio" ]; } { name = "js-sys"; packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "log"; packageId = "log"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "percent-encoding"; packageId = "percent-encoding"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "pin-project-lite"; packageId = "pin-project-lite"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "serde_urlencoded"; - packageId = "serde_urlencoded"; + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "sync_wrapper"; @@ -7795,27 +7826,27 @@ rec { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "net" "time" ]; } { name = "tower"; packageId = "tower"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "retry" "timeout" "util" ]; } { name = "tower-http"; packageId = "tower-http"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "follow-redirect" ]; } { name = "tower-service"; packageId = "tower-service"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "url"; @@ -7824,17 +7855,17 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "wasm-bindgen-futures"; packageId = "wasm-bindgen-futures"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "web-sys"; packageId = "web-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "RequestCache" ]; } ]; @@ -7843,33 +7874,27 @@ rec { name = "futures-util"; packageId = "futures-util"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "std" "alloc" ]; } { name = "hyper"; packageId = "hyper"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "server" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "client-legacy" "server-auto" "server-graceful" "tokio" ]; } - { - name = "serde"; - packageId = "serde"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "derive" ]; - } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "macros" "rt-multi-thread" ]; } { @@ -7881,40 +7906,37 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "serde-serialize" ]; } ]; features = { + "__native-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "__native-tls-alpn" = [ "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" ]; - "__rustls-ring" = [ "hyper-rustls?/ring" "tokio-rustls?/ring" "rustls?/ring" "quinn?/ring" ]; + "__rustls-aws-lc-rs" = [ "hyper-rustls?/aws-lc-rs" "tokio-rustls?/aws-lc-rs" "rustls?/aws-lc-rs" "quinn?/rustls-aws-lc-rs" ]; "__tls" = [ "dep:rustls-pki-types" "tokio/io-util" ]; "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "dep:futures-util" "futures-util?/io" "futures-util?/sink" "tokio/sync" ]; "brotli" = [ "tower-http/decompression-br" ]; "charset" = [ "dep:encoding_rs" "dep:mime" ]; "cookies" = [ "dep:cookie_crate" "dep:cookie_store" ]; "default" = [ "default-tls" "charset" "http2" "system-proxy" ]; - "default-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "default-tls" = [ "rustls" ]; "deflate" = [ "tower-http/decompression-deflate" ]; + "form" = [ "dep:serde" "dep:serde_urlencoded" ]; "gzip" = [ "tower-http/decompression-gzip" ]; - "h2" = [ "dep:h2" ]; "hickory-dns" = [ "dep:hickory-resolver" "dep:once_cell" ]; - "http2" = [ "h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; - "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; - "json" = [ "dep:serde_json" ]; - "macos-system-configuration" = [ "system-proxy" ]; + "http2" = [ "dep:h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; + "http3" = [ "rustls" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; + "json" = [ "dep:serde" "dep:serde_json" ]; "multipart" = [ "dep:mime_guess" "dep:futures-util" ]; - "native-tls" = [ "default-tls" ]; - "native-tls-alpn" = [ "native-tls" "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; - "native-tls-vendored" = [ "native-tls" "native-tls-crate?/vendored" ]; - "rustls-tls" = [ "rustls-tls-webpki-roots" ]; - "rustls-tls-manual-roots" = [ "rustls-tls-manual-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-manual-roots-no-provider" = [ "__rustls" ]; - "rustls-tls-native-roots" = [ "rustls-tls-native-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-native-roots-no-provider" = [ "dep:rustls-native-certs" "hyper-rustls?/native-tokio" "__rustls" ]; - "rustls-tls-no-provider" = [ "rustls-tls-manual-roots-no-provider" ]; - "rustls-tls-webpki-roots" = [ "rustls-tls-webpki-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-webpki-roots-no-provider" = [ "dep:webpki-roots" "hyper-rustls?/webpki-tokio" "__rustls" ]; + "native-tls" = [ "__native-tls" "__native-tls-alpn" ]; + "native-tls-no-alpn" = [ "__native-tls" ]; + "native-tls-vendored" = [ "__native-tls" "native-tls-crate?/vendored" "__native-tls-alpn" ]; + "native-tls-vendored-no-alpn" = [ "__native-tls" "native-tls-crate?/vendored" ]; + "query" = [ "dep:serde" "dep:serde_urlencoded" ]; + "rustls" = [ "__rustls-aws-lc-rs" "dep:rustls-platform-verifier" "__rustls" ]; + "rustls-no-provider" = [ "dep:rustls-platform-verifier" "__rustls" ]; "stream" = [ "tokio/fs" "dep:futures-util" "dep:tokio-util" "dep:wasm-streams" ]; "system-proxy" = [ "hyper-util/client-proxy-system" ]; "zstd" = [ "tower-http/decompression-zstd" ]; @@ -9288,29 +9310,25 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "rust_1_61" "rust_1_65" "std" ]; }; - "snafu 0.9.0" = rec { + "snafu 0.9.1" = rec { crateName = "snafu"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "1ii9r99x5qcn754m624yzgb9hzvkqkrcygf0aqh0pyb9dbnvrm6i"; + sha256 = "08k5yfydxdlshivfhrdq9km8qn02r93q28gkyvazbqz2icr1586i"; authors = [ "Jake Goulding " ]; dependencies = [ { name = "snafu-derive"; - packageId = "snafu-derive 0.9.0"; + packageId = "snafu-derive 0.9.1"; } ]; features = { - "backtrace" = [ "dep:backtrace" ]; - "backtraces-impl-backtrace-crate" = [ "backtrace" ]; + "backtraces-impl-backtrace-crate" = [ "dep:backtrace" ]; "default" = [ "std" "rust_1_81" ]; - "futures" = [ "futures-core-crate" "pin-project" ]; - "futures-core-crate" = [ "dep:futures-core-crate" ]; - "futures-crate" = [ "dep:futures-crate" ]; - "internal-dev-dependencies" = [ "futures-crate" ]; - "pin-project" = [ "dep:pin-project" ]; + "futures" = [ "dep:futures-core" "dep:pin-project" ]; + "internal-dev-dependencies" = [ "dep:futures" ]; "std" = [ "alloc" ]; "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; }; @@ -9378,11 +9396,11 @@ rec { }; resolvedDefaultFeatures = [ "rust_1_61" ]; }; - "snafu-derive 0.9.0" = rec { + "snafu-derive 0.9.1" = rec { crateName = "snafu-derive"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "0h0x61kyj4fvilcr2nj02l85shw1ika64vq9brf2gyna662ln9al"; + sha256 = "1nkfi7bis72pz3w7vb64m79w49qsv20sbf19jkd471vbhr83q42z"; procMacro = true; libName = "snafu_derive"; authors = [ @@ -9408,7 +9426,7 @@ rec { name = "syn"; packageId = "syn 2.0.117"; usesDefaultFeatures = false; - features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" ]; + features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } ]; features = { @@ -9516,9 +9534,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_certs"; authors = [ @@ -9576,7 +9594,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-shared"; @@ -9649,10 +9667,6 @@ rec { name = "indoc"; packageId = "indoc"; } - { - name = "product-config"; - packageId = "product-config"; - } { name = "serde"; packageId = "serde"; @@ -9664,7 +9678,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator"; @@ -9711,9 +9725,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_operator"; authors = [ @@ -9763,6 +9777,10 @@ rec { name = "indexmap"; packageId = "indexmap"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "jiff"; packageId = "jiff"; @@ -9770,6 +9788,7 @@ rec { { name = "json-patch"; packageId = "json-patch"; + features = [ "schemars" ]; } { name = "k8s-openapi"; @@ -9819,7 +9838,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator-derive"; @@ -9873,12 +9892,21 @@ rec { packageId = "url"; features = [ "serde" ]; } + { + name = "uuid"; + packageId = "uuid"; + } + { + name = "xml"; + packageId = "xml"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; + "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "crds" "certs" "time" "webhook" "kube-ws" ]; + "full" = [ "client-feature-gates" "crds" "certs" "time" "webhook" "kube-ws" ]; "kube-ws" = [ "kube/ws" ]; "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; @@ -9891,9 +9919,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -9922,13 +9950,13 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.1.0"; + version = "0.1.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_shared"; authors = [ @@ -9972,7 +10000,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10003,13 +10031,13 @@ rec { }; "stackable-telemetry" = rec { crateName = "stackable-telemetry"; - version = "0.6.3"; + version = "0.6.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_telemetry"; authors = [ @@ -10052,7 +10080,7 @@ rec { { name = "opentelemetry_sdk"; packageId = "opentelemetry_sdk"; - features = [ "rt-tokio" "logs" "rt-tokio" "spec_unstable_logs_enabled" ]; + features = [ "rt-tokio" "logs" "rt-tokio" ]; } { name = "pin-project"; @@ -10060,7 +10088,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10117,9 +10145,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_versioned"; authors = [ @@ -10152,7 +10180,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-versioned-macros"; @@ -10167,9 +10195,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10235,9 +10263,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_webhook"; authors = [ @@ -10309,7 +10337,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-certs"; @@ -11400,6 +11428,33 @@ rec { } ]; + }; + "tonic-types" = rec { + crateName = "tonic-types"; + version = "0.14.5"; + edition = "2021"; + sha256 = "16bk1cxi2m0xgaabf98nnj7dn9j16ymkh27jq4s3shjm4a85m1ra"; + libName = "tonic_types"; + authors = [ + "Lucio Franco " + "Rafael Lemos " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + } + ]; + }; "tower" = rec { crateName = "tower"; @@ -11856,9 +11911,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "1z2jjmxbkm1qawlb3bm99x8xwf4g8wjkbcknm9z4fv1w14nqzhhs"; + sha256 = "09nvxy5m7nxmifz4b6szdcyczapp2jcgxcac0jw4ax8klz5n9g5d"; libName = "tracing_opentelemetry"; dependencies = [ { @@ -12260,6 +12315,66 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "uuid" = rec { + crateName = "uuid"; + version = "1.23.2"; + edition = "2021"; + sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + authors = [ + "Ashley Mannix" + "Dylan DPC" + "Hunar Roop Kahlon" + ]; + dependencies = [ + { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + devDependencies = [ + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "atomic" = [ "dep:atomic" ]; + "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "default" = [ "std" ]; + "fast-rng" = [ "rng" "dep:rand" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "md5" = [ "dep:md-5" ]; + "rng" = [ "dep:getrandom" ]; + "rng-getrandom" = [ "rng" "dep:getrandom" "uuid-rng-internal-lib" "uuid-rng-internal-lib/getrandom" ]; + "rng-rand" = [ "rng" "dep:rand" "uuid-rng-internal-lib" "uuid-rng-internal-lib/rand" ]; + "serde" = [ "dep:serde_core" ]; + "sha1" = [ "dep:sha1_smol" ]; + "slog" = [ "dep:slog" ]; + "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; + "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; + "v1" = [ "atomic" ]; + "v3" = [ "md5" ]; + "v4" = [ "rng" ]; + "v5" = [ "sha1" ]; + "v6" = [ "atomic" ]; + "v7" = [ "rng" ]; + "zerocopy" = [ "dep:zerocopy" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "valuable" = rec { crateName = "valuable"; version = "0.1.1"; @@ -13918,9 +14033,9 @@ rec { }; "xml" = rec { crateName = "xml"; - version = "1.2.1"; + version = "1.3.0"; edition = "2021"; - sha256 = "0ak4k990faralbli5a0rb8kvwihccb2rp0r94d4azfy94a6lkamq"; + sha256 = "128s58qhq8whrx90zbw8r5algr7lakgbf7mn05jfk234rbjqavv3"; authors = [ "Vladimir Matveev " "Kornel (https://github.com/kornelski)" diff --git a/Cargo.toml b/Cargo.toml index 3647c4ba..d15d6664 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ edition = "2024" repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.111.1", features = ["crds", "webhook"] } anyhow = "1.0" @@ -30,5 +29,6 @@ tokio = { version = "1.40", features = ["full"] } tracing = "0.1" [patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } diff --git a/crate-hashes.json b/crate-hashes.json index 86f2b840..c9a6e6a9 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#k8s-version@0.1.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-certs@0.4.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-operator-derive@0.3.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-operator@0.111.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-shared@0.1.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-telemetry@0.6.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-versioned-macros@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-versioned@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-webhook@0.9.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index b6f80cdd..9bd8c3b2 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -1,152 +1,5 @@ --- version: 0.1.0 spec: - units: - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" - - - unit: &unitUrl - name: "url" - regex: "^((https?|ftp|file)://)?[-a-zA-Z0-9+&@#}/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]" - examples: - - "https://www.stackable.de/blog/" - - - unit: &unitCapacity - name: "capacity" - regex: "^[1-9]\\d*$" - - - unit: &unitMilliseconds - name: "milliseconds" - regex: "^[1-9]\\d*$" - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for domain names that cannot be resolved." - description: "TTL for domain names that cannot be resolved." - - - property: &opaAuthorizerClassName - propertyNames: - - name: "authorizer.class.name" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "com.bisnode.kafka.authorization.OpaAuthorizer" - - fromVersion: "3.0.0" - value: "org.openpolicyagent.kafka.OpaAuthorizer" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer class name" - - - property: &opaAuthorizerUrl - propertyNames: - - name: "opa.authorizer.url" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - unit: *unitUrl - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer URL" - - - property: &opaAuthorizerInitialCacheCapacity - propertyNames: - - name: "opa.authorizer.cache.initial.capacity" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer initial cache capacity" - - - property: &opaAuthorizerMaxCacheSize - propertyNames: - - name: "opa.authorizer.cache.maximum.size" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA authorizer max cache size" - - - property: &opaAuthorizerCacheExpireAfterSeconds - propertyNames: - - name: "opa.authorizer.cache.expire.after.seconds" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "The number of seconds after which the OPA authorizer cache expires" + units: [] +properties: [] diff --git a/deploy/helm/kafka-operator/configs/properties.yaml b/deploy/helm/kafka-operator/configs/properties.yaml index b6f80cdd..9bd8c3b2 100644 --- a/deploy/helm/kafka-operator/configs/properties.yaml +++ b/deploy/helm/kafka-operator/configs/properties.yaml @@ -1,152 +1,5 @@ --- version: 0.1.0 spec: - units: - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" - - - unit: &unitUrl - name: "url" - regex: "^((https?|ftp|file)://)?[-a-zA-Z0-9+&@#}/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]" - examples: - - "https://www.stackable.de/blog/" - - - unit: &unitCapacity - name: "capacity" - regex: "^[1-9]\\d*$" - - - unit: &unitMilliseconds - name: "milliseconds" - regex: "^[1-9]\\d*$" - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for domain names that cannot be resolved." - description: "TTL for domain names that cannot be resolved." - - - property: &opaAuthorizerClassName - propertyNames: - - name: "authorizer.class.name" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "com.bisnode.kafka.authorization.OpaAuthorizer" - - fromVersion: "3.0.0" - value: "org.openpolicyagent.kafka.OpaAuthorizer" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer class name" - - - property: &opaAuthorizerUrl - propertyNames: - - name: "opa.authorizer.url" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - unit: *unitUrl - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer URL" - - - property: &opaAuthorizerInitialCacheCapacity - propertyNames: - - name: "opa.authorizer.cache.initial.capacity" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer initial cache capacity" - - - property: &opaAuthorizerMaxCacheSize - propertyNames: - - name: "opa.authorizer.cache.maximum.size" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA authorizer max cache size" - - - property: &opaAuthorizerCacheExpireAfterSeconds - propertyNames: - - name: "opa.authorizer.cache.expire.after.seconds" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "The number of seconds after which the OPA authorizer cache expires" + units: [] +properties: [] diff --git a/docs/modules/kafka/pages/reference/commandline-parameters.adoc b/docs/modules/kafka/pages/reference/commandline-parameters.adoc index 9059a960..3c66dc41 100644 --- a/docs/modules/kafka/pages/reference/commandline-parameters.adoc +++ b/docs/modules/kafka/pages/reference/commandline-parameters.adoc @@ -2,19 +2,6 @@ This operator accepts the following command line parameters: -== product-config - -*Default value*: `/etc/stackable/kafka-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values:* false - -[source] ----- -stackable-kafka-operator run --product-config /foo/bar/properties.yaml ----- - == watch-namespace *Default value*: All namespaces diff --git a/docs/modules/kafka/pages/reference/environment-variables.adoc b/docs/modules/kafka/pages/reference/environment-variables.adoc index cc7dd3a2..d2271300 100644 --- a/docs/modules/kafka/pages/reference/environment-variables.adoc +++ b/docs/modules/kafka/pages/reference/environment-variables.adoc @@ -33,32 +33,6 @@ docker run \ oci.stackable.tech/sdp/kafka-operator:0.0.0-dev ---- -== PRODUCT_CONFIG - -*Default value*: `/etc/stackable/kafka-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values*: false - -[source] ----- -export PRODUCT_CONFIG=/foo/bar/properties.yaml -stackable-kafka-operator run ----- - -or via docker: - ----- -docker run \ - --name kafka-operator \ - --network host \ - --env KUBECONFIG=/home/stackable/.kube/config \ - --env PRODUCT_CONFIG=/my/product/config.yaml \ - --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ - oci.stackable.tech/sdp/kafka-operator:0.0.0-dev ----- - == WATCH_NAMESPACE *Default value*: All namespaces diff --git a/extra/crds.yaml b/extra/crds.yaml index a2c3b7d9..6177a6e5 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -98,86 +98,6 @@ spec: containers: description: Log configuration per container. properties: - get-service: - anyOf: - - required: - - custom - - {} - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object kafka: anyOf: - required: @@ -542,22 +462,14 @@ spec: broker.properties: additionalProperties: type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -715,86 +627,6 @@ spec: containers: description: Log configuration per container. properties: - get-service: - anyOf: - - required: - - custom - - {} - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object kafka: anyOf: - required: @@ -1159,22 +991,14 @@ spec: broker.properties: additionalProperties: type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -1787,22 +1611,14 @@ spec: controller.properties: additionalProperties: type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -2236,22 +2052,14 @@ spec: controller.properties: additionalProperties: type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index f2903572..b9d9d40f 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,6 @@ repository.workspace = true publish = false [dependencies] -product-config.workspace = true stackable-operator.workspace = true indoc.workspace = true diff --git a/rust/operator-binary/src/config/command.rs b/rust/operator-binary/src/config/command.rs index a4540001..7cdc5738 100644 --- a/rust/operator-binary/src/config/command.rs +++ b/rust/operator-binary/src/config/command.rs @@ -6,15 +6,33 @@ use stackable_operator::{ utils::COMMON_BASH_TRAP_FUNCTIONS, }; -use crate::{ - crd::{ - KafkaPodDescriptor, STACKABLE_CONFIG_DIR, STACKABLE_KERBEROS_KRB5_PATH, - role::{broker::BROKER_PROPERTIES_FILE, controller::CONTROLLER_PROPERTIES_FILE}, - security::KafkaTlsSecurity, - }, - product_logging::{BROKER_ID_POD_MAP_DIR, STACKABLE_LOG_DIR}, +use crate::crd::{ + BROKER_ID_POD_MAP_DIR, ConfigFileName, KafkaPodDescriptor, STACKABLE_CONFIG_DIR, + STACKABLE_KERBEROS_KRB5_PATH, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, + security::KafkaTlsSecurity, }; +/// The JVM options selecting the Kafka log4j/log4j2 config file. Kafka 3.x uses log4j, +/// Kafka 4.0 and higher use log4j2. +pub fn kafka_log_opts(product_version: &str) -> String { + if product_version.starts_with("3.") { + format!( + "-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{log4j}", + log4j = ConfigFileName::Log4j + ) + } else { + format!( + "-Dlog4j2.configurationFile=file:{STACKABLE_LOG_CONFIG_DIR}/{log4j2}", + log4j2 = ConfigFileName::Log4j2 + ) + } +} + +/// The env var carrying the Kafka log4j options (see [`kafka_log_opts`]). +pub fn kafka_log_opts_env_var() -> String { + "KAFKA_LOG4J_OPTS".to_string() +} + /// Returns the commands to start the main Kafka container pub fn broker_kafka_container_commands( kraft_mode: bool, @@ -63,12 +81,13 @@ fn broker_start_command( cp {config_dir}/{properties_file} /tmp/{properties_file} config-utils template /tmp/{properties_file} - cp {config_dir}/jaas.properties /tmp/jaas.properties - config-utils template /tmp/jaas.properties + cp {config_dir}/{jaas_file} /tmp/{jaas_file} + config-utils template /tmp/{jaas_file} ", broker_id_pod_map_dir = BROKER_ID_POD_MAP_DIR, config_dir = STACKABLE_CONFIG_DIR, - properties_file = BROKER_PROPERTIES_FILE, + properties_file = ConfigFileName::BrokerProperties, + jaas_file = ConfigFileName::Jaas, }; if kraft_mode { @@ -78,7 +97,7 @@ fn broker_start_command( bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted {initial_controller_command} bin/kafka-server-start.sh /tmp/{properties_file} & ", - properties_file = BROKER_PROPERTIES_FILE, + properties_file = ConfigFileName::BrokerProperties, initial_controller_command = initial_controllers_command(&controller_descriptors, product_version), } } else { @@ -86,7 +105,7 @@ fn broker_start_command( {common_command} bin/kafka-server-start.sh /tmp/{properties_file} &", - properties_file = BROKER_PROPERTIES_FILE, + properties_file = ConfigFileName::BrokerProperties, } } } @@ -158,7 +177,7 @@ pub fn controller_kafka_container_command( ", remove_vector_shutdown_file_command = remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), config_dir = STACKABLE_CONFIG_DIR, - properties_file = CONTROLLER_PROPERTIES_FILE, + properties_file = ConfigFileName::ControllerProperties, initial_controller_command = initial_controllers_command(&controller_descriptors, product_version), create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR) } diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index 79023618..f0233b66 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -6,9 +6,7 @@ use stackable_operator::{ schemars::JsonSchema, }; -use crate::crd::{ - JVM_SECURITY_PROPERTIES_FILE, METRICS_PORT, STACKABLE_CONFIG_DIR, role::AnyConfig, -}; +use crate::crd::{ConfigFileName, METRICS_PORT, STACKABLE_CONFIG_DIR, role::AnyConfig}; const JAVA_HEAP_FACTOR: f32 = 0.8; @@ -54,7 +52,10 @@ where // Heap settings format!("-Xmx{java_heap}"), format!("-Xms{java_heap}"), - format!("-Djava.security.properties={STACKABLE_CONFIG_DIR}/{JVM_SECURITY_PROPERTIES_FILE}"), + format!( + "-Djava.security.properties={STACKABLE_CONFIG_DIR}/{security}", + security = ConfigFileName::Security + ), format!( "-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/server.yaml" ), @@ -110,8 +111,17 @@ fn is_heap_jvm_argument(jvm_argument: &str) -> bool { #[cfg(test)] mod tests { + use stackable_operator::kube::ResourceExt; + use super::*; - use crate::crd::{BrokerRole, role::KafkaRole, v1alpha1}; + use crate::{ + crd::{ + BrokerRole, + role::{KafkaRole, broker::BrokerConfig}, + v1alpha1, + }, + framework::role_utils::with_validated_config, + }; #[test] fn test_construct_jvm_arguments_defaults() { @@ -197,12 +207,12 @@ mod tests { let kafka: v1alpha1::KafkaCluster = serde_yaml::from_str(kafka_cluster).expect("illegal test input"); - let kafka_role = KafkaRole::Broker; - let rolegroup_ref = kafka.rolegroup_ref(&kafka_role, "default"); - let merged_config = kafka_role - .merged_config(&kafka, &rolegroup_ref.role_group) - .unwrap(); - let role = kafka.spec.brokers.unwrap(); + let role = kafka.spec.brokers.clone().unwrap(); + let role_group = role.role_groups.get("default").unwrap(); + let default_config = + BrokerConfig::default_config(&kafka.name_any(), &KafkaRole::Broker.to_string()); + let validated = with_validated_config(role_group, &role, &default_config).unwrap(); + let merged_config = AnyConfig::Broker(validated.config); (merged_config, role, "default".to_owned()) } diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs new file mode 100644 index 00000000..3db8337e --- /dev/null +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -0,0 +1,205 @@ +use indoc::formatdoc; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + k8s_openapi::api::core::v1::ConfigMap, + role_utils::RoleGroupRef, + v2::config_file_writer::{PropertiesWriterError, to_java_properties_string}, +}; + +use crate::{ + controller::{ + ValidatedCluster, ValidatedRoleGroupConfig, + build::properties::logging::role_group_config_map_data, + }, + crd::{ + ConfigFileName, STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, + listener::{KafkaListenerConfig, node_address_cmd}, + role::AnyConfig, + v1alpha1, + }, + kafka_controller::{KAFKA_CONTROLLER_NAME, build_recommended_labels}, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build ConfigMap for {}", rolegroup))] + BuildRoleGroupConfig { + source: stackable_operator::builder::configmap::Error, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("failed to serialize [{}] for {rolegroup}", ConfigFileName::Security))] + JvmSecurityProperties { + source: PropertiesWriterError, + rolegroup: String, + }, + + #[snafu(display("failed to build Metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to serialize config for {rolegroup}"))] + SerializeConfig { + source: PropertiesWriterError, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("no Kraft controllers found to build"))] + NoKraftControllersFound, +} + +/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator +pub fn build_rolegroup_config_map( + validated_cluster: &ValidatedCluster, + rolegroup: &RoleGroupRef, + validated_rg: &ValidatedRoleGroupConfig, + listener_config: &KafkaListenerConfig, +) -> Result { + let cluster_config = &validated_cluster.cluster_config; + let kafka_security = &cluster_config.kafka_security; + let resolved_product_image = &validated_cluster.image; + let kafka_config_file_name = validated_rg.config.config_file_name().to_string(); + let config_overrides = validated_rg + .config_overrides + .config_file_overrides() + .overrides + .clone(); + + if cluster_config.is_kraft_mode() && cluster_config.pod_descriptors.is_empty() { + return NoKraftControllersFoundSnafu.fail(); + } + + let kafka_config = match &validated_rg.config { + AnyConfig::Broker(_) => crate::controller::build::properties::broker_properties::build( + cluster_config, + listener_config, + config_overrides, + ), + AnyConfig::Controller(_) => { + crate::controller::build::properties::controller_properties::build( + cluster_config, + listener_config, + config_overrides, + ) + } + }; + + let jvm_sec_props = &validated_rg + .config_overrides + .security_properties() + .overrides; + + let mut cm_builder = ConfigMapBuilder::new(); + cm_builder + .metadata( + ObjectMetaBuilder::new() + .name_and_namespace(validated_cluster) + .name(rolegroup.object_name()) + .ownerreference_from_resource(validated_cluster, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&build_recommended_labels( + validated_cluster, + KAFKA_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .build(), + ) + .add_data( + kafka_config_file_name, + to_java_properties_string(kafka_config.iter()).with_context(|_| { + SerializeConfigSnafu { + rolegroup: rolegroup.clone(), + } + })?, + ) + .add_data( + ConfigFileName::Security.to_string(), + to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { + JvmSecurityPropertiesSnafu { + rolegroup: rolegroup.role_group.clone(), + } + })?, + ) + .add_data( + ConfigFileName::Client.to_string(), + to_java_properties_string( + kafka_security + .client_properties() + .iter() + .filter_map(|(k, v)| v.as_ref().map(|v| (k, v))), + ) + .with_context(|_| JvmSecurityPropertiesSnafu { + rolegroup: rolegroup.role_group.clone(), + })?, + ) + // This file contains the JAAS configuration for Kerberos authentication + // It has the ".properties" extension but is not a Java properties file. + // It is processed by `config-utils` to substitute "env:" and "file:" variables + // and this tool currently doesn't support the JAAS login configuration format. + .add_data( + ConfigFileName::Jaas.to_string(), + jaas_config_file(kafka_security.has_kerberos_enabled()), + ); + + tracing::debug!(?kafka_config, "Applied kafka config"); + tracing::debug!(?jvm_sec_props, "Applied JVM config"); + + let config_data = role_group_config_map_data( + &resolved_product_image.product_version, + rolegroup, + &validated_rg.config, + ); + for (file_name, data) in config_data { + if let Some(data) = data { + cm_builder.add_data(file_name, data); + } + } + + cm_builder + .build() + .with_context(|_| BuildRoleGroupConfigSnafu { + rolegroup: rolegroup.clone(), + }) +} + +// Generate JAAS configuration file for Kerberos authentication +// or an empty string if Kerberos is not enabled. +// See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html +fn jaas_config_file(is_kerberos_enabled: bool) -> String { + match is_kerberos_enabled { + false => String::new(), + true => formatdoc! {" + bootstrap.KafkaServer {{ + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + isInitiator=false + keyTab=\"/stackable/kerberos/keytab\" + principal=\"kafka/{bootstrap_address}@${{env:KERBEROS_REALM}}\"; + }}; + + client.KafkaServer {{ + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + isInitiator=false + keyTab=\"/stackable/kerberos/keytab\" + principal=\"kafka/{broker_address}@${{env:KERBEROS_REALM}}\"; + }}; + + ", + bootstrap_address = node_address_cmd(STACKABLE_LISTENER_BOOTSTRAP_DIR), + broker_address = node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), + }, + } +} diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs similarity index 78% rename from rust/operator-binary/src/discovery.rs rename to rust/operator-binary/src/controller/build/discovery.rs index 3730de33..ef78a7e8 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -3,16 +3,15 @@ use std::num::TryFromIntError; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, crd::listener, k8s_openapi::api::core::v1::ConfigMap, - kube::{Resource, ResourceExt, runtime::reflector::ObjectRef}, + kube::runtime::reflector::ObjectRef, }; use crate::{ - controller::KAFKA_CONTROLLER_NAME, - crd::{role::KafkaRole, security::KafkaTlsSecurity, v1alpha1}, - utils::build_recommended_labels, + controller::ValidatedCluster, + crd::{role::KafkaRole, v1alpha1}, + kafka_controller::{KAFKA_CONTROLLER_NAME, build_recommended_labels}, }; #[derive(Snafu, Debug)] @@ -23,9 +22,6 @@ pub enum Error { kafka: ObjectRef, }, - #[snafu(display("object has no name associated"))] - NoName, - #[snafu(display("could not find service port with name {}", port_name))] NoServicePort { port_name: String }, @@ -46,12 +42,12 @@ pub enum Error { /// Build a discovery [`ConfigMap`] containing information about how to connect to a certain /// [`v1alpha1::KafkaCluster`]. pub fn build_discovery_configmap( - kafka: &v1alpha1::KafkaCluster, - owner: &impl Resource, - resolved_product_image: &ResolvedProductImage, - kafka_security: &KafkaTlsSecurity, + validated_cluster: &ValidatedCluster, listeners: &[listener::v1alpha1::Listener], ) -> Result { + let kafka_security = &validated_cluster.cluster_config.kafka_security; + let resolved_product_image = &validated_cluster.image; + let port_name = if kafka_security.has_kerberos_enabled() { kafka_security.bootstrap_port_name() } else { @@ -68,14 +64,13 @@ pub fn build_discovery_configmap( ConfigMapBuilder::new() .metadata( ObjectMetaBuilder::new() - .name_and_namespace(kafka) - .name(owner.name_unchecked()) - .ownerreference_from_resource(owner, None, Some(true)) + .name_and_namespace(validated_cluster) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { - kafka: ObjectRef::from_obj(kafka), + kafka: cluster_object_ref(validated_cluster), })? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.product_version, &KafkaRole::Broker.to_string(), @@ -89,6 +84,12 @@ pub fn build_discovery_configmap( .context(BuildConfigMapSnafu) } +/// An [`ObjectRef`] to the owning cluster, built from the validated identity — used only for +/// error context. +fn cluster_object_ref(cluster: &ValidatedCluster) -> ObjectRef { + ObjectRef::new(cluster.name.as_ref()).within(cluster.namespace.as_ref()) +} + fn listener_hosts( listeners: &[listener::v1alpha1::Listener], port_name: &str, diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs new file mode 100644 index 00000000..0cbab809 --- /dev/null +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -0,0 +1,5 @@ +//! Builders that assemble Kubernetes resources for kafka rolegroups. + +pub mod config_map; +pub mod discovery; +pub mod properties; diff --git a/rust/operator-binary/src/controller/build/properties/broker_properties.rs b/rust/operator-binary/src/controller/build/properties/broker_properties.rs new file mode 100644 index 00000000..700e0822 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/broker_properties.rs @@ -0,0 +1,111 @@ +use std::collections::BTreeMap; + +use super::kraft_controllers; +use crate::{ + controller::ValidatedClusterConfig, + crd::{ + listener::{KafkaListenerConfig, KafkaListenerName}, + role::{ + KAFKA_ADVERTISED_LISTENERS, KAFKA_BROKER_ID, KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, + KAFKA_PROCESS_ROLES, KafkaRole, + }, + }, + operations::graceful_shutdown::graceful_shutdown_config_properties, +}; + +pub fn build( + cluster_config: &ValidatedClusterConfig, + listener_config: &KafkaListenerConfig, + overrides: BTreeMap, +) -> BTreeMap { + let kraft_controllers = kraft_controllers(&cluster_config.pod_descriptors); + + let mut result = BTreeMap::from([ + ( + KAFKA_LOG_DIRS.to_string(), + "/stackable/data/topicdata".to_string(), + ), + (KAFKA_LISTENERS.to_string(), listener_config.listeners()), + ( + KAFKA_ADVERTISED_LISTENERS.to_string(), + listener_config.advertised_listeners(), + ), + ( + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), + listener_config.listener_security_protocol_map(), + ), + ( + "inter.broker.listener.name".to_string(), + KafkaListenerName::Internal.to_string(), + ), + ]); + + if cluster_config.is_kraft_mode() { + let kraft_controllers = kraft_controllers.join(","); + + // Running in KRaft mode + result.extend([ + ( + "broker.id.generation.enable".to_string(), + "false".to_string(), + ), + (KAFKA_NODE_ID.to_string(), "${env:REPLICA_ID}".to_string()), + ( + KAFKA_PROCESS_ROLES.to_string(), + KafkaRole::Broker.to_string(), + ), + ( + "controller.listener.names".to_string(), + KafkaListenerName::Controller.to_string(), + ), + ( + KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), + kraft_controllers.clone(), + ), + ]); + } else { + // Running with ZooKeeper enabled + result.extend([( + "zookeeper.connect".to_string(), + "${env:ZOOKEEPER}".to_string(), + )]); + // We are in zookeeper mode and the user has defined a broker id mapping + // so we disable automatic id generation. + // This check ensures that existing clusters running in ZooKeeper mode do not + // suddenly break after the introduction of this change. + if cluster_config.disable_broker_id_generation { + result.extend([ + ( + "broker.id.generation.enable".to_string(), + "false".to_string(), + ), + (KAFKA_BROKER_ID.to_string(), "${env:REPLICA_ID}".to_string()), + ]); + } + } + + // Enable OPA authorization + if let Some(opa_connect_string) = cluster_config.opa_connect() { + result.extend([ + ( + "authorizer.class.name".to_string(), + "org.openpolicyagent.kafka.OpaAuthorizer".to_string(), + ), + ( + "opa.authorizer.metrics.enabled".to_string(), + "true".to_string(), + ), + ( + "opa.authorizer.url".to_string(), + opa_connect_string.to_string(), + ), + ]); + } + + result.extend(cluster_config.kafka_security.broker_config_settings()); + result.extend(graceful_shutdown_config_properties()); + result.extend(overrides); + + result +} diff --git a/rust/operator-binary/src/controller/build/properties/controller_properties.rs b/rust/operator-binary/src/controller/build/properties/controller_properties.rs new file mode 100644 index 00000000..289d0cb2 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/controller_properties.rs @@ -0,0 +1,70 @@ +use std::collections::BTreeMap; + +use super::kraft_controllers; +use crate::{ + controller::ValidatedClusterConfig, + crd::{ + listener::{KafkaListenerConfig, KafkaListenerName}, + role::{ + KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, + KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, KAFKA_PROCESS_ROLES, KafkaRole, + }, + }, + operations::graceful_shutdown::graceful_shutdown_config_properties, +}; + +pub fn build( + cluster_config: &ValidatedClusterConfig, + listener_config: &KafkaListenerConfig, + overrides: BTreeMap, +) -> BTreeMap { + let kraft_controllers = kraft_controllers(&cluster_config.pod_descriptors).join(","); + + let mut result = BTreeMap::from([ + ( + KAFKA_LOG_DIRS.to_string(), + "/stackable/data/kraft".to_string(), + ), + (KAFKA_PROCESS_ROLES.to_string(), KafkaRole::Controller.to_string()), + ( + "controller.listener.names".to_string(), + KafkaListenerName::Controller.to_string(), + ), + ( + KAFKA_NODE_ID.to_string(), + "${env:REPLICA_ID}".to_string(), + ), + ( + KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), + kraft_controllers.clone(), + ), + ( + KAFKA_LISTENERS.to_string(), + "CONTROLLER://${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}:${env:KAFKA_CLIENT_PORT}".to_string(), + ), + ( + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), + listener_config + .listener_security_protocol_map_for_controller()), + ]); + + result.insert( + "inter.broker.listener.name".to_string(), + KafkaListenerName::Internal.to_string(), + ); + + // The ZooKeeper connection is needed for migration from ZooKeeper to KRaft mode. + // It is not needed once the controller is fully running in KRaft mode. + if !cluster_config.is_kraft_mode() { + result.insert( + "zookeeper.connect".to_string(), + "${env:ZOOKEEPER}".to_string(), + ); + } + + result.extend(cluster_config.kafka_security.controller_config_settings()); + result.extend(graceful_shutdown_config_properties()); + result.extend(overrides); + + result +} diff --git a/rust/operator-binary/src/controller/build/properties/logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs new file mode 100644 index 00000000..da198541 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/logging.rs @@ -0,0 +1,139 @@ +//! Renders the logging config files (`log4j.properties` / `log4j2.properties` and the +//! Vector agent config) assembled into the rolegroup `ConfigMap`. + +use std::{borrow::Cow, collections::BTreeMap, fmt::Display}; + +use stackable_operator::{ + memory::{BinaryMultiple, MemoryQuantity}, + product_logging::{ + self, + spec::{ContainerLogConfig, ContainerLogConfigChoice}, + }, + role_utils::RoleGroupRef, +}; + +use crate::{ + crd::{ + ConfigFileName, STACKABLE_LOG_DIR, + role::{AnyConfig, broker::BrokerContainer, controller::ControllerContainer}, + v1alpha1, + }, + kafka_controller::MAX_KAFKA_LOG_FILES_SIZE, +}; + +const KAFKA_LOG4J_FILE: &str = "kafka.log4j.xml"; +const KAFKA_LOG4J2_FILE: &str = "kafka.log4j2.xml"; + +const CONSOLE_CONVERSION_PATTERN_LOG4J: &str = "[%d] %p %m (%c)%n"; +const CONSOLE_CONVERSION_PATTERN_LOG4J2: &str = "%d{ISO8601} %p [%t] %c - %m%n"; + +/// Get the role group ConfigMap data with logging and Vector configurations +pub fn role_group_config_map_data( + product_version: &str, + rolegroup: &RoleGroupRef, + merged_config: &AnyConfig, +) -> BTreeMap> { + let container_name = match merged_config { + AnyConfig::Broker(_) => BrokerContainer::Kafka.to_string(), + AnyConfig::Controller(_) => ControllerContainer::Kafka.to_string(), + }; + + let mut configs: BTreeMap> = BTreeMap::new(); + + // Starting with Kafka 4.0, log4j2 is used instead of log4j. + match product_version.starts_with("3.") { + true => { + configs.insert( + ConfigFileName::Log4j.to_string(), + log4j_config_if_automatic( + Some(merged_config.kafka_logging()), + container_name, + KAFKA_LOG4J_FILE, + MAX_KAFKA_LOG_FILES_SIZE, + ), + ); + } + false => { + configs.insert( + ConfigFileName::Log4j2.to_string(), + log4j2_config_if_automatic( + Some(merged_config.kafka_logging()), + container_name, + KAFKA_LOG4J2_FILE, + MAX_KAFKA_LOG_FILES_SIZE, + ), + ); + } + } + + let vector_log_config = merged_config.vector_logging(); + let vector_log_config = if let ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + } = &*vector_log_config + { + Some(log_config) + } else { + None + }; + + if merged_config.vector_logging_enabled() { + configs.insert( + product_logging::framework::VECTOR_CONFIG_FILE.to_string(), + Some(product_logging::framework::create_vector_config( + rolegroup, + vector_log_config, + )), + ); + } + configs +} + +fn log4j_config_if_automatic( + log_config: Option>, + container_name: impl Display, + log_file: &str, + max_log_file_size: MemoryQuantity, +) -> Option { + if let Some(ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + }) = log_config.as_deref() + { + Some(product_logging::framework::create_log4j_config( + &format!("{STACKABLE_LOG_DIR}/{container_name}"), + log_file, + max_log_file_size + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, + CONSOLE_CONVERSION_PATTERN_LOG4J, + log_config, + )) + } else { + None + } +} + +fn log4j2_config_if_automatic( + log_config: Option>, + container_name: impl Display, + log_file: &str, + max_log_file_size: MemoryQuantity, +) -> Option { + if let Some(ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + }) = log_config.as_deref() + { + Some(product_logging::framework::create_log4j2_config( + &format!("{STACKABLE_LOG_DIR}/{container_name}",), + log_file, + max_log_file_size + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, + CONSOLE_CONVERSION_PATTERN_LOG4J2, + log_config, + )) + } else { + None + } +} diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs new file mode 100644 index 00000000..4f83b22c --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -0,0 +1,21 @@ +//! Property-file builders for Kafka rolegroup ConfigMaps. + +pub mod broker_properties; +pub mod controller_properties; +pub mod logging; + +use crate::crd::{KafkaPodDescriptor, role::KafkaRole}; + +pub(crate) fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Vec { + pod_descriptors + .iter() + .filter(|pd| pd.role == KafkaRole::Controller.to_string()) + .map(|desc| { + format!( + "{fqdn}:{client_port}", + fqdn = desc.fqdn(), + client_port = desc.client_port + ) + }) + .collect::>() +} diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs index b7a22107..c90d1258 100644 --- a/rust/operator-binary/src/controller/dereference.rs +++ b/rust/operator-binary/src/controller/dereference.rs @@ -9,7 +9,7 @@ //! and stays here as-is. use snafu::{ResultExt, Snafu}; -use stackable_operator::client::Client; +use stackable_operator::{client::Client, utils::cluster_info::KubernetesClusterInfo}; use crate::crd::{ authentication::{self, ResolvedAuthenticationClasses}, @@ -33,6 +33,7 @@ type Result = std::result::Result; pub struct DereferencedObjects { pub authentication_classes: ResolvedAuthenticationClasses, pub authorization_config: Option, + pub kubernetes_cluster_info: KubernetesClusterInfo, } /// Fetches all Kubernetes objects referenced from the [`v1alpha1::KafkaCluster`] spec. @@ -59,5 +60,6 @@ pub async fn dereference( Ok(DereferencedObjects { authentication_classes, authorization_config, + kubernetes_cluster_info: client.kubernetes_cluster_info.clone(), }) } diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs new file mode 100644 index 00000000..2c907057 --- /dev/null +++ b/rust/operator-binary/src/controller/mod.rs @@ -0,0 +1,151 @@ +//! The validated cluster model and the steps that produce it. +//! +//! [`ValidatedCluster`] carries everything the build steps need, resolved once during +//! [`validate`] (after [`dereference`]) so downstream code never re-derives it or +//! touches the raw [`v1alpha1::KafkaCluster`] spec. The reconcile loop that consumes +//! it lives in [`crate::kafka_controller`]. + +use std::{borrow::Cow, collections::BTreeMap}; + +use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, + kube::{Resource, api::ObjectMeta}, + v2::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, +}; + +pub(crate) mod build; +pub(crate) mod dereference; +pub(crate) mod validate; + +use crate::{ + crd::{ + KafkaPodDescriptor, MetadataManager, + authorization::KafkaAuthorizationConfig, + role::{AnyConfig, AnyConfigOverrides, KafkaRole}, + security::KafkaTlsSecurity, + v1alpha1, + }, + framework::role_utils::RoleGroupConfig, +}; + +pub type RoleGroupName = String; + +/// The validated cluster. Carries everything the build steps need, resolved once +/// here so downstream code never re-derives it or touches the raw spec. +/// +/// The cluster identity (`name`, `namespace`, `uid`) is captured here so that owner +/// references for child objects can be built straight from this struct (via its +/// [`Resource`] impl) without threading the raw [`v1alpha1::KafkaCluster`] around. +/// This mirrors the hive-/opensearch-operator's `ValidatedCluster`. +pub struct ValidatedCluster { + /// `ObjectMeta` carrying `name`, `namespace` and `uid`, so this struct can act as the + /// owner [`Resource`] for child objects. + metadata: ObjectMeta, + pub name: ClusterName, + pub namespace: NamespaceName, + pub image: ResolvedProductImage, + pub cluster_config: ValidatedClusterConfig, + pub role_group_configs: BTreeMap>, +} + +impl ValidatedCluster { + pub fn new( + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + image: ResolvedProductImage, + cluster_config: ValidatedClusterConfig, + role_group_configs: BTreeMap>, + ) -> Self { + Self { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }, + name, + namespace, + image, + cluster_config, + role_group_configs, + } + } +} + +/// Cluster-wide settings resolved during validation and dereferencing. +/// +/// Everything the build steps need is resolved here so they never have to read the +/// raw [`v1alpha1::KafkaCluster`] spec. +pub struct ValidatedClusterConfig { + pub kafka_security: KafkaTlsSecurity, + pub authorization_config: Option, + pub pod_descriptors: Vec, + pub metadata_manager: MetadataManager, + + /// Whether the operator must not generate broker ids itself, because the user + /// supplied a `broker_id_pod_config_map_name`. Resolved from the raw spec during + /// validation so the config-map builder never has to read it. + pub disable_broker_id_generation: bool, +} + +impl ValidatedClusterConfig { + /// Whether the cluster runs in KRaft mode (as opposed to ZooKeeper mode). + pub fn is_kraft_mode(&self) -> bool { + self.metadata_manager == MetadataManager::KRaft + } + + /// The OPA connect string, if OPA authorization is configured. + pub fn opa_connect(&self) -> Option<&str> { + self.authorization_config + .as_ref() + .map(|auth_config| auth_config.opa_connect.as_str()) + } +} + +/// Lets [`ValidatedCluster`] act as the owner [`Resource`] for child objects, so owner +/// references are built from it (via the captured `metadata`) rather than the raw CR. +impl Resource for ValidatedCluster { + type DynamicType = ::DynamicType; + type Scope = ::Scope; + + fn kind(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::kind(dt) + } + + fn group(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::group(dt) + } + + fn version(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::version(dt) + } + + fn plural(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::KafkaCluster::plural(dt) + } + + fn meta(&self) -> &ObjectMeta { + &self.metadata + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} + +/// A validated, merged Kafka role-group config. +/// +/// The merged config fragment is wrapped in [`AnyConfig`] and the merged +/// `configOverrides` in [`AnyConfigOverrides`], so a single role-agnostic type +/// carries both broker and controller role groups (their concrete config and +/// override types differ). Produced via the local-`framework` +/// [`with_validated_config`](crate::framework::role_utils::with_validated_config). +pub type ValidatedRoleGroupConfig = RoleGroupConfig< + AnyConfig, + stackable_operator::role_utils::JavaCommonConfig, + AnyConfigOverrides, +>; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index a3e441a0..973a8ce1 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,33 +1,49 @@ //! The validate step in the KafkaCluster controller. //! //! Synchronously validates inputs that don't require a Kubernetes client. Produces -//! [`ValidatedInputs`], consumed by the rest of `reconcile_kafka`. +//! [`ValidatedCluster`], consumed by the rest of `reconcile_kafka`. -use std::collections::HashMap; +use std::{collections::BTreeMap, str::FromStr}; -use product_config::{ProductConfigManager, types::PropertyNameKind}; -use snafu::{ResultExt, Snafu}; +use serde::Serialize; +use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, - commons::product_image_selection::{self, ResolvedProductImage}, - product_config_utils::{ - ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, - validate_all_roles_and_groups_config, + commons::product_image_selection, + config::{fragment::FromFragment, merge::Merge}, + kube::ResourceExt, + role_utils::{GenericRoleConfig, JavaCommonConfig, Role}, + schemars::JsonSchema, + v2::{ + builder::pod::container::{self, EnvVarName, EnvVarSet}, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, }, }; use crate::{ - controller::dereference::DereferencedObjects, + controller::{ + RoleGroupName, ValidatedCluster, ValidatedClusterConfig, ValidatedRoleGroupConfig, + dereference::DereferencedObjects, + }, crd::{ - self, CONTAINER_IMAGE_BASE_NAME, JVM_SECURITY_PROPERTIES_FILE, + self, CONTAINER_IMAGE_BASE_NAME, authentication::{self}, - authorization::KafkaAuthorizationConfig, - role::{KafkaRole, broker::BROKER_PROPERTIES_FILE, controller::CONTROLLER_PROPERTIES_FILE}, + role::{ + AnyConfig, AnyConfigOverrides, KafkaRole, broker::BrokerConfig, + controller::ControllerConfig, + }, security::{self, KafkaTlsSecurity}, v1alpha1, }, + framework::role_utils::with_validated_config, }; +/// The operator-managed env var carrying the Kafka cluster id. +const KAFKA_CLUSTER_ID_ENV: &str = "KAFKA_CLUSTER_ID"; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to resolve product image"))] @@ -44,34 +60,50 @@ pub enum Error { #[snafu(display("cluster object defines no '{role}' role"))] MissingKafkaRole { source: crd::Error, role: KafkaRole }, - #[snafu(display("failed to generate product config"))] - GenerateProductConfig { - source: stackable_operator::product_config_utils::Error, + #[snafu(display("failed to merge and validate the role group config"))] + ValidateRoleGroupConfig { + source: crate::framework::role_utils::Error, + }, + + #[snafu(display("invalid environment variable name"))] + InvalidEnvVarName { source: container::Error }, + + #[snafu(display("failed to build pod descriptors"))] + BuildPodDescriptors { source: crate::crd::Error }, + + #[snafu(display("invalid metadata manager"))] + InvalidMetadataManager { source: crate::crd::Error }, + + #[snafu(display("invalid cluster name"))] + InvalidClusterName { + source: stackable_operator::v2::macros::attributed_string_type::Error, }, - #[snafu(display("invalid product config"))] - InvalidProductConfig { - source: stackable_operator::product_config_utils::Error, + #[snafu(display("object defines no namespace"))] + ObjectHasNoNamespace, + + #[snafu(display("invalid cluster namespace"))] + InvalidNamespace { + source: stackable_operator::v2::macros::attributed_string_type::Error, }, -} -type Result = std::result::Result; + #[snafu(display("object has no uid"))] + ObjectHasNoUid, -/// Synchronous inputs the rest of `reconcile_kafka` needs after dereferencing. -pub struct ValidatedInputs { - pub authorization_config: Option, - pub image: ResolvedProductImage, - pub kafka_security: KafkaTlsSecurity, - pub role_config: ValidatedRoleConfigByPropertyKind, + #[snafu(display("invalid cluster uid"))] + InvalidUid { + source: stackable_operator::v2::macros::attributed_string_type::Error, + }, } +type Result = std::result::Result; + /// Validates the cluster spec and the dereferenced inputs. pub fn validate( kafka: &v1alpha1::KafkaCluster, dereferenced_objects: DereferencedObjects, operator_environment: &OperatorEnvironmentOptions, - product_config: &ProductConfigManager, -) -> Result { +) -> Result { let image = kafka .spec .image @@ -99,81 +131,175 @@ pub fn validate( .validate_authentication_methods() .context(FailedToValidateAuthenticationMethodSnafu)?; - let role_config = validated_product_config(kafka, &image.product_version, product_config)?; + let cluster_id = kafka.cluster_id(); + + let mut role_group_configs: BTreeMap< + KafkaRole, + BTreeMap, + > = BTreeMap::new(); + + // Brokers always exist. + let broker_role = kafka.broker_role().context(MissingKafkaRoleSnafu { + role: KafkaRole::Broker, + })?; + let broker_groups = validate_role_group_configs( + broker_role, + BrokerConfig::default_config(&kafka.name_any(), &KafkaRole::Broker.to_string()), + cluster_id, + AnyConfig::Broker, + AnyConfigOverrides::Broker, + )?; + role_group_configs.insert(KafkaRole::Broker, broker_groups); + + // Controllers are optional: ZooKeeper-mode clusters have none, and `controller_role()` + // errors when `controllers` is unset, which would stop their reconciliation. + if let Some(controller_role) = kafka.spec.controllers.as_ref() { + let controller_groups = validate_role_group_configs( + controller_role, + ControllerConfig::default_config(&kafka.name_any(), &KafkaRole::Controller.to_string()), + cluster_id, + AnyConfig::Controller, + AnyConfigOverrides::Controller, + )?; + role_group_configs.insert(KafkaRole::Controller, controller_groups); + } + + let pod_descriptors = kafka + .pod_descriptors( + None, + &dereferenced_objects.kubernetes_cluster_info, + kafka_security.client_port(), + ) + .context(BuildPodDescriptorsSnafu)?; + + let metadata_manager = kafka + .effective_metadata_manager() + .context(InvalidMetadataManagerSnafu)?; + + let name = ClusterName::from_str(&kafka.name_any()).context(InvalidClusterNameSnafu)?; + let namespace = NamespaceName::from_str(&kafka.namespace().context(ObjectHasNoNamespaceSnafu)?) + .context(InvalidNamespaceSnafu)?; + let uid = Uid::from_str(&kafka.uid().context(ObjectHasNoUidSnafu)?).context(InvalidUidSnafu)?; - Ok(ValidatedInputs { - authorization_config: dereferenced_objects.authorization_config, + Ok(ValidatedCluster::new( + name, + namespace, + uid, image, - kafka_security, - role_config, - }) + ValidatedClusterConfig { + kafka_security, + authorization_config: dereferenced_objects.authorization_config, + pod_descriptors, + metadata_manager, + disable_broker_id_generation: kafka + .spec + .cluster_config + .broker_id_pod_config_map_name + .is_some(), + }, + role_group_configs, + )) } -fn validated_product_config( - kafka: &v1alpha1::KafkaCluster, - product_version: &str, - product_config: &ProductConfigManager, -) -> Result { - let mut role_config = HashMap::new(); - - let broker_role = [( - KafkaRole::Broker.to_string(), - ( - vec![ - PropertyNameKind::File(BROKER_PROPERTIES_FILE.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - PropertyNameKind::Env, - ], - kafka - .broker_role() - .cloned() - .context(MissingKafkaRoleSnafu { - role: KafkaRole::Broker, - })? - .erase(), - ), - )] - .into(); - - let broker_role_config = - transform_all_roles_to_config(kafka, &broker_role).context(GenerateProductConfigSnafu)?; - - role_config.extend(broker_role_config); - - // We need this because controller_role() raises an error if non-existent, - // which would stop reconciliation. - if kafka.spec.controllers.is_some() { - let controller_role = [( - KafkaRole::Controller.to_string(), - ( - vec![ - PropertyNameKind::File(CONTROLLER_PROPERTIES_FILE.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - PropertyNameKind::Env, - ], - kafka - .controller_role() - .cloned() - .context(MissingKafkaRoleSnafu { - role: KafkaRole::Controller, - })? - .erase(), - ), - )] - .into(); - - let controller_role_config = transform_all_roles_to_config(kafka, &controller_role) - .context(GenerateProductConfigSnafu)?; - - role_config.extend(controller_role_config); +/// Validates every role group of a role into a map keyed by role group name. +/// +/// Each role group is merged and validated via the local-`framework` +/// [`with_validated_config`], which folds the config fragment (default <- role <- +/// role group) plus the `configOverrides`, `envOverrides`, `cliOverrides` and +/// `podOverrides` (role group wins) into a single +/// [`RoleGroupConfig`](crate::framework::role_utils::RoleGroupConfig). The concrete +/// per-role validated config and overrides are wrapped into the role-agnostic +/// [`AnyConfig`]/[`AnyConfigOverrides`] via `wrap_config`/`wrap_overrides`, and the +/// operator-managed `KAFKA_CLUSTER_ID` is injected into the env overrides. +fn validate_role_group_configs( + role: &Role, + default_config: Config, + cluster_id: Option<&str>, + wrap_config: fn(ValidatedConfig) -> AnyConfig, + wrap_overrides: fn(ConfigOverrides) -> AnyConfigOverrides, +) -> Result> +where + Config: Clone + Merge, + ValidatedConfig: FromFragment, + ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, +{ + role.role_groups + .iter() + .map(|(role_group_name, role_group)| { + let validated = with_validated_config::< + ValidatedConfig, + JavaCommonConfig, + Config, + GenericRoleConfig, + ConfigOverrides, + >(role_group, role, &default_config) + .context(ValidateRoleGroupConfigSnafu)?; + + // Re-wrap the per-role validated config and overrides into the role-agnostic + // enums; the merged env/cli/pod overrides carry over unchanged, except that + // `KAFKA_CLUSTER_ID` is injected into the env overrides. + let validated = ValidatedRoleGroupConfig { + replicas: validated.replicas, + config: wrap_config(validated.config), + config_overrides: wrap_overrides(validated.config_overrides), + env_overrides: inject_cluster_id(validated.env_overrides, cluster_id)?, + cli_overrides: validated.cli_overrides, + pod_overrides: validated.pod_overrides, + product_specific_common_config: validated.product_specific_common_config, + }; + Ok((role_group_name.clone(), validated)) + }) + .collect() +} + +/// Injects the operator-managed `KAFKA_CLUSTER_ID` into the merged env overrides, +/// but only when the user has not already set it via `envOverrides` (user value +/// wins). +fn inject_cluster_id(env_overrides: EnvVarSet, cluster_id: Option<&str>) -> Result { + let Some(cluster_id) = cluster_id else { + return Ok(env_overrides); + }; + let name = EnvVarName::from_str(KAFKA_CLUSTER_ID_ENV).context(InvalidEnvVarNameSnafu)?; + if env_overrides.get(&name).is_some() { + // The user set `KAFKA_CLUSTER_ID` via envOverrides; their value wins. + Ok(env_overrides) + } else { + Ok(env_overrides.with_value(&name, cluster_id)) } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use stackable_operator::v2::builder::pod::container::{EnvVarName, EnvVarSet}; + + use super::{KAFKA_CLUSTER_ID_ENV, inject_cluster_id}; - validate_all_roles_and_groups_config( - product_version, - &role_config, - product_config, - false, - false, - ) - .context(InvalidProductConfigSnafu) + fn cluster_id_value(env: &EnvVarSet) -> Option { + let name = EnvVarName::from_str(KAFKA_CLUSTER_ID_ENV).unwrap(); + env.get(&name).and_then(|var| var.value.clone()) + } + + #[test] + fn injects_cluster_id_when_absent() { + let env = inject_cluster_id(EnvVarSet::new(), Some("my-id")).unwrap(); + assert_eq!(cluster_id_value(&env), Some("my-id".to_string())); + } + + #[test] + fn user_cluster_id_override_wins() { + let name = EnvVarName::from_str(KAFKA_CLUSTER_ID_ENV).unwrap(); + let env = EnvVarSet::new().with_value(&name, "user-value"); + + let env = inject_cluster_id(env, Some("operator-value")).unwrap(); + + assert_eq!(cluster_id_value(&env), Some("user-value".to_string())); + } + + #[test] + fn without_cluster_id_nothing_is_injected() { + let env = inject_cluster_id(EnvVarSet::new(), None).unwrap(); + assert_eq!(cluster_id_value(&env), None); + } } diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index da01acca..51e0ae6b 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -30,9 +30,17 @@ mod tests { api::core::v1::{PodAffinityTerm, PodAntiAffinity, WeightedPodAffinityTerm}, apimachinery::pkg::apis::meta::v1::LabelSelector, }, + kube::ResourceExt, }; - use crate::crd::{KafkaRole, v1alpha1}; + use crate::{ + crd::{ + KafkaRole, + role::{AnyConfig, broker::BrokerConfig}, + v1alpha1, + }, + framework::role_utils::with_validated_config, + }; #[rstest] #[case(KafkaRole::Broker)] @@ -55,7 +63,11 @@ mod tests { let kafka: v1alpha1::KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); - let merged_config = role.merged_config(&kafka, "default").unwrap(); + let broker_role = kafka.spec.brokers.clone().unwrap(); + let role_group = broker_role.role_groups.get("default").unwrap(); + let default_config = BrokerConfig::default_config(&kafka.name_any(), &role.to_string()); + let validated = with_validated_config(role_group, &broker_role, &default_config).unwrap(); + let merged_config = AnyConfig::Broker(validated.config); assert_eq!( merged_config.affinity, diff --git a/rust/operator-binary/src/crd/config_file.rs b/rust/operator-binary/src/crd/config_file.rs new file mode 100644 index 00000000..aec47d39 --- /dev/null +++ b/rust/operator-binary/src/crd/config_file.rs @@ -0,0 +1,50 @@ +//! The names of the config files assembled into the rolegroup `ConfigMap`. +//! +//! A single source of truth for the on-disk file names, used by the config-map +//! builder, the per-file property builders, the JVM/command builders and +//! [`AnyConfig::config_file_name`](crate::crd::role::AnyConfig::config_file_name). + +/// The names of the Kafka config files assembled into the rolegroup `ConfigMap`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] +pub enum ConfigFileName { + #[strum(serialize = "broker.properties")] + BrokerProperties, + #[strum(serialize = "controller.properties")] + ControllerProperties, + #[strum(serialize = "security.properties")] + Security, + #[strum(serialize = "client.properties")] + Client, + /// JAAS configuration for Kerberos authentication. It has the `.properties` + /// extension but is not a Java properties file. + #[strum(serialize = "jaas.properties")] + Jaas, + /// Used by Kafka 3.x. + #[strum(serialize = "log4j.properties")] + Log4j, + /// Used by Kafka 4.0 and later. + #[strum(serialize = "log4j2.properties")] + Log4j2, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_names_match_the_kafka_on_disk_names() { + assert_eq!( + ConfigFileName::BrokerProperties.to_string(), + "broker.properties" + ); + assert_eq!( + ConfigFileName::ControllerProperties.to_string(), + "controller.properties" + ); + assert_eq!(ConfigFileName::Security.to_string(), "security.properties"); + assert_eq!(ConfigFileName::Client.to_string(), "client.properties"); + assert_eq!(ConfigFileName::Jaas.to_string(), "jaas.properties"); + assert_eq!(ConfigFileName::Log4j.to_string(), "log4j.properties"); + assert_eq!(ConfigFileName::Log4j2.to_string(), "log4j2.properties"); + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index d662de30..630595f6 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,6 +1,7 @@ pub mod affinity; pub mod authentication; pub mod authorization; +pub mod config_file; pub mod listener; pub mod role; pub mod security; @@ -9,6 +10,7 @@ pub mod tls; use std::collections::{BTreeMap, HashMap}; use authentication::KafkaAuthentication; +pub use config_file::ConfigFileName; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, Snafu}; use stackable_operator::{ @@ -16,13 +18,14 @@ use stackable_operator::{ cluster_operation::ClusterOperation, networking::DomainName, product_image_selection::ProductImage, }, - config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider}, + config::merge::Merge, deep_merger::ObjectOverrides, kube::{CustomResource, runtime::reflector::ObjectRef}, role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, utils::cluster_info::KubernetesClusterInfo, + v2::config_overrides::KeyValueConfigOverrides, versioned::versioned, }; use strum::{Display, EnumIter, EnumString}; @@ -43,8 +46,6 @@ pub const FIELD_MANAGER: &str = "kafka-operator"; // metrics pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 9606; -// config files -pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; // env vars pub const KAFKA_HEAP_OPTS: &str = "KAFKA_HEAP_OPTS"; // server_properties @@ -59,6 +60,10 @@ pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; // kerberos pub const STACKABLE_KERBEROS_DIR: &str = "/stackable/kerberos"; pub const STACKABLE_KERBEROS_KRB5_PATH: &str = "/stackable/kerberos/krb5.conf"; +// logging +pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; +pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config"; +pub const BROKER_ID_POD_MAP_DIR: &str = "/stackable/broker-id-pod-map"; #[derive(Snafu, Debug)] pub enum Error { @@ -240,40 +245,24 @@ pub mod versioned { pub broker_id_pod_config_map_name: Option, } - #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Merge, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KafkaBrokerConfigOverrides { - #[serde( - default, - rename = "broker.properties", - skip_serializing_if = "Option::is_none" - )] - pub broker_properties: Option, + #[serde(default, rename = "broker.properties")] + pub broker_properties: KeyValueConfigOverrides, - #[serde( - default, - rename = "security.properties", - skip_serializing_if = "Option::is_none" - )] - pub security_properties: Option, + #[serde(default, rename = "security.properties")] + pub security_properties: KeyValueConfigOverrides, } - #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Merge, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KafkaControllerConfigOverrides { - #[serde( - default, - rename = "controller.properties", - skip_serializing_if = "Option::is_none" - )] - pub controller_properties: Option, + #[serde(default, rename = "controller.properties")] + pub controller_properties: KeyValueConfigOverrides, - #[serde( - default, - rename = "security.properties", - skip_serializing_if = "Option::is_none" - )] - pub security_properties: Option, + #[serde(default, rename = "security.properties")] + pub security_properties: KeyValueConfigOverrides, } } @@ -291,32 +280,6 @@ impl Default for v1alpha1::KafkaClusterConfig { } } -impl KeyValueOverridesProvider for v1alpha1::KafkaBrokerConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - let field = match file { - role::broker::BROKER_PROPERTIES_FILE => self.broker_properties.as_ref(), - JVM_SECURITY_PROPERTIES_FILE => self.security_properties.as_ref(), - _ => None, - }; - field - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default() - } -} - -impl KeyValueOverridesProvider for v1alpha1::KafkaControllerConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - let field = match file { - role::controller::CONTROLLER_PROPERTIES_FILE => self.controller_properties.as_ref(), - JVM_SECURITY_PROPERTIES_FILE => self.security_properties.as_ref(), - _ => None, - }; - field - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default() - } -} - impl HasStatusCondition for v1alpha1::KafkaCluster { fn conditions(&self) -> Vec { match &self.status { diff --git a/rust/operator-binary/src/crd/role/broker.rs b/rust/operator-binary/src/crd/role/broker.rs index 70ac85d0..2024d519 100644 --- a/rust/operator-binary/src/crd/role/broker.rs +++ b/rust/operator-binary/src/crd/role/broker.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::resources::{ @@ -8,18 +6,12 @@ use stackable_operator::{ }, config::{fragment::Fragment, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, - product_config_utils::Configuration, product_logging::{self, spec::Logging}, schemars::{self, JsonSchema}, }; use strum::{Display, EnumIter}; -use crate::crd::{ - role::commons::{CommonConfig, Storage, StorageFragment}, - v1alpha1, -}; - -pub const BROKER_PROPERTIES_FILE: &str = "broker.properties"; +use crate::crd::role::commons::{CommonConfig, Storage, StorageFragment}; #[derive( Clone, @@ -39,7 +31,6 @@ pub const BROKER_PROPERTIES_FILE: &str = "broker.properties"; pub enum BrokerContainer { Vector, KcatProber, - GetService, Kafka, } @@ -101,39 +92,3 @@ impl BrokerConfig { } } } - -impl Configuration for BrokerConfigFragment { - type Configurable = v1alpha1::KafkaCluster; - - fn compute_env( - &self, - resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - let mut result = BTreeMap::new(); - if let Some(cluster_id) = resource.cluster_id() { - result.insert("KAFKA_CLUSTER_ID".to_string(), Some(cluster_id.to_string())); - } - Ok(result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _resource: &Self::Configurable, - _role_name: &str, - _file: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } -} diff --git a/rust/operator-binary/src/crd/role/controller.rs b/rust/operator-binary/src/crd/role/controller.rs index bf1468b6..fbaf898c 100644 --- a/rust/operator-binary/src/crd/role/controller.rs +++ b/rust/operator-binary/src/crd/role/controller.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::resources::{ @@ -8,18 +6,12 @@ use stackable_operator::{ }, config::{fragment::Fragment, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, - product_config_utils::Configuration, product_logging::{self, spec::Logging}, schemars::{self, JsonSchema}, }; use strum::{Display, EnumIter}; -use crate::crd::{ - role::commons::{CommonConfig, Storage, StorageFragment}, - v1alpha1, -}; - -pub const CONTROLLER_PROPERTIES_FILE: &str = "controller.properties"; +use crate::crd::role::commons::{CommonConfig, Storage, StorageFragment}; #[derive( Clone, @@ -91,39 +83,3 @@ impl ControllerConfig { } } } - -impl Configuration for ControllerConfigFragment { - type Configurable = v1alpha1::KafkaCluster; - - fn compute_env( - &self, - resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - let mut result = BTreeMap::new(); - if let Some(cluster_id) = resource.cluster_id() { - result.insert("KAFKA_CLUSTER_ID".to_string(), Some(cluster_id.to_string())); - } - Ok(result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _resource: &Self::Configurable, - _role_name: &str, - _file: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } -} diff --git a/rust/operator-binary/src/crd/role/mod.rs b/rust/operator-binary/src/crd/role/mod.rs index e08474ed..41cdcf1f 100644 --- a/rust/operator-binary/src/crd/role/mod.rs +++ b/rust/operator-binary/src/crd/role/mod.rs @@ -8,24 +8,24 @@ use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ commons::resources::{NoRuntimeLimits, Resources}, - config::{ - fragment::{self, ValidationError}, - merge::Merge, - }, k8s_openapi::api::core::v1::PodTemplateSpec, - kube::{ResourceExt, runtime::reflector::ObjectRef}, + kube::runtime::reflector::ObjectRef, product_logging::spec::ContainerLogConfig, role_utils::RoleGroupRef, schemars::{self, JsonSchema}, + v2::config_overrides::KeyValueConfigOverrides, }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ config::jvm::{construct_heap_jvm_args, construct_non_heap_jvm_args}, - crd::role::{ - broker::{BROKER_PROPERTIES_FILE, BrokerConfig}, - commons::{CommonConfig, Storage}, - controller::{CONTROLLER_PROPERTIES_FILE, ControllerConfig}, + crd::{ + ConfigFileName, + role::{ + broker::BrokerConfig, + commons::{CommonConfig, Storage}, + controller::ControllerConfig, + }, }, v1alpha1, }; @@ -73,9 +73,6 @@ pub const KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS: &str = "controller.quorum.b #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("fragment validation failure"))] - FragmentValidationFailure { source: ValidationError }, - #[snafu(display("the Kafka role [{role}] is missing from spec"))] MissingRole { source: crate::crd::Error, @@ -98,7 +95,9 @@ pub enum Error { Eq, Hash, JsonSchema, + Ord, PartialEq, + PartialOrd, Serialize, EnumString, )] @@ -138,87 +137,6 @@ impl KafkaRole { "kafka" } - /// Merge the [Broker|Controller]ConfigFragment defaults, role and role group settings. - /// The priority is: default < role config < role_group config - pub fn merged_config( - &self, - kafka: &v1alpha1::KafkaCluster, - rolegroup: &str, - ) -> Result { - match self { - Self::Broker => { - // Initialize the result with all default values as baseline - let default_config = - BrokerConfig::default_config(&kafka.name_any(), &self.to_string()); - - // Retrieve role resource config - let role = kafka.broker_role().with_context(|_| MissingRoleSnafu { - role: self.to_string(), - })?; - - let mut role_config = role.config.config.clone(); - // Retrieve rolegroup specific resource config - let mut role_group_config = role - .role_groups - .get(rolegroup) - .with_context(|| MissingRoleGroupSnafu { - role: self.to_string(), - rolegroup: rolegroup.to_string(), - })? - .config - .config - .clone(); - - // Merge more specific configs into default config - // Hierarchy is: - // 1. RoleGroup - // 2. Role - // 3. Default - role_config.merge(&default_config); - role_group_config.merge(&role_config); - Ok(AnyConfig::Broker( - fragment::validate::(role_group_config) - .context(FragmentValidationFailureSnafu)?, - )) - } - Self::Controller => { - // Initialize the result with all default values as baseline - let default_config = - ControllerConfig::default_config(&kafka.name_any(), &self.to_string()); - - // Retrieve role resource config - let role = kafka.controller_role().with_context(|_| MissingRoleSnafu { - role: self.to_string(), - })?; - - let mut role_config = role.config.config.clone(); - // Retrieve rolegroup specific resource config - let mut role_group_config = role - .role_groups - .get(rolegroup) - .with_context(|| MissingRoleGroupSnafu { - role: self.to_string(), - rolegroup: rolegroup.to_string(), - })? - .config - .config - .clone(); - - // Merge more specific configs into default config - // Hierarchy is: - // 1. RoleGroup - // 2. Role - // 3. Default - role_config.merge(&default_config); - role_group_config.merge(&role_config); - Ok(AnyConfig::Controller( - fragment::validate::(role_group_config) - .context(FragmentValidationFailureSnafu)?, - )) - } - } - } - pub fn construct_non_heap_jvm_args( &self, merged_config: &AnyConfig, @@ -440,10 +358,40 @@ impl AnyConfig { } } - pub fn config_file_name(&self) -> &str { + pub fn config_file_name(&self) -> ConfigFileName { + match self { + AnyConfig::Broker(_) => ConfigFileName::BrokerProperties, + AnyConfig::Controller(_) => ConfigFileName::ControllerProperties, + } + } +} + +/// Merged role/role-group `configOverrides` for a role group of an unknown type. +/// +/// Mirrors [`AnyConfig`] for the override side: broker and controller use distinct +/// override structs, so this enum lets the build layer carry the typed, merged +/// overrides through a single role-agnostic `RoleGroupConfig`. +#[derive(Clone, Debug, PartialEq)] +pub enum AnyConfigOverrides { + Broker(v1alpha1::KafkaBrokerConfigOverrides), + Controller(v1alpha1::KafkaControllerConfigOverrides), +} + +impl AnyConfigOverrides { + /// The merged product config-file overrides (`broker.properties` for brokers, + /// `controller.properties` for controllers). + pub fn config_file_overrides(&self) -> &KeyValueConfigOverrides { + match self { + AnyConfigOverrides::Broker(o) => &o.broker_properties, + AnyConfigOverrides::Controller(o) => &o.controller_properties, + } + } + + /// The merged `security.properties` overrides (shared by both roles). + pub fn security_properties(&self) -> &KeyValueConfigOverrides { match self { - AnyConfig::Broker(_) => BROKER_PROPERTIES_FILE, - AnyConfig::Controller(_) => CONTROLLER_PROPERTIES_FILE, + AnyConfigOverrides::Broker(o) => &o.security_properties, + AnyConfigOverrides::Controller(o) => &o.security_properties, } } } diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs new file mode 100644 index 00000000..6e28e88a --- /dev/null +++ b/rust/operator-binary/src/framework.rs @@ -0,0 +1,11 @@ +//! Local framework helpers that mirror the work-in-progress upstream +//! `stackable_operator::v2::*` modules. +//! +//! We vendor `role_utils` because the upstream `v2::role_utils` requires +//! `CommonConfig: Merge`. Kafka (like hdfs and trino) uses `JavaCommonConfig`, +//! whose JVM-argument merge is fallible and so does not implement `Merge`. +//! +//! Follow-up: replace with `stackable_operator::v2::role_utils::*` once upstream +//! relaxes the `Merge` bound. + +pub mod role_utils; diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs new file mode 100644 index 00000000..b4bc9a8b --- /dev/null +++ b/rust/operator-binary/src/framework/role_utils.rs @@ -0,0 +1,152 @@ +//! Vendored variant of `stackable_operator::v2::role_utils` from the +//! `smooth-operator` branch, with simplifications appropriate for kafka-operator. +//! +//! Differences from upstream: +//! - No `cli_overrides_to_vec` helper, `ResourceNames`, or service-account helpers. +//! - The `CommonConfig` (a.k.a. `product_specific_common_config`) does NOT need to +//! implement `Merge`. Kafka uses `JavaCommonConfig`, which intentionally does not +//! implement `Merge` because its inner `JvmArgumentOverrides::try_merge` is +//! fallible (regex validation). The `RoleGroupConfig::product_specific_common_config` +//! field here simply carries the role-group level value through. +//! +//! Replace with `stackable_operator::v2::role_utils::*` once upstream relaxes the +//! `Merge` bound. + +use std::{ + collections::{BTreeMap, HashMap}, + str::FromStr, +}; + +use serde::Serialize; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + config::{ + fragment::{self, FromFragment}, + merge::{Merge, merge}, + }, + k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, + role_utils::{Role, RoleGroup}, + schemars::JsonSchema, + v2::builder::pod::container::{self, EnvVarName, EnvVarSet}, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to validate the role group config"))] + ValidateConfig { source: fragment::ValidationError }, + + #[snafu(display("invalid environment variable override name"))] + ParseEnvVarName { source: container::Error }, +} + +/// Kafka-friendly view of a validated, merged `RoleGroup`. +#[derive(Clone, Debug, PartialEq)] +pub struct RoleGroupConfig { + pub replicas: u16, + pub config: Config, + pub config_overrides: ConfigOverrides, + pub env_overrides: EnvVarSet, + pub cli_overrides: BTreeMap, + pub pod_overrides: PodTemplateSpec, + pub product_specific_common_config: CommonConfig, +} + +/// Merges and validates the `RoleGroup` with the given `role` and `default_config`. +pub fn with_validated_config( + role_group: &RoleGroup, + role: &Role, + default_config: &Config, +) -> Result, Error> +where + ValidatedConfig: FromFragment, + CommonConfig: Clone + Default + JsonSchema + Serialize, + Config: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, +{ + let validated_config = + validate_config(role_group, role, default_config).context(ValidateConfigSnafu)?; + Ok(RoleGroupConfig { + replicas: role_group.replicas.unwrap_or(1), + config: validated_config, + config_overrides: merged_config_overrides( + &role.config.config_overrides, + role_group.config.config_overrides.clone(), + ), + env_overrides: merged_env_overrides( + &role.config.env_overrides, + &role_group.config.env_overrides, + )?, + cli_overrides: merged_cli_overrides( + role.config.cli_overrides.clone(), + role_group.config.cli_overrides.clone(), + ), + pod_overrides: merged_pod_overrides( + role.config.pod_overrides.clone(), + role_group.config.pod_overrides.clone(), + ), + product_specific_common_config: role_group.config.product_specific_common_config.clone(), + }) +} + +fn validate_config( + role_group: &RoleGroup, + role: &Role, + default_config: &Config, +) -> Result +where + ValidatedConfig: FromFragment, + CommonConfig: Default + JsonSchema + Serialize, + Config: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, +{ + role_group.validate_config(role, default_config) +} + +fn merged_config_overrides( + role_config_overrides: &ConfigOverrides, + role_group_config_overrides: ConfigOverrides, +) -> ConfigOverrides +where + ConfigOverrides: Merge, +{ + merge(role_group_config_overrides, role_config_overrides) +} + +fn merged_env_overrides( + role_env_overrides: &HashMap, + role_group_env_overrides: &HashMap, +) -> Result { + // Process the role first, then the role group, so that role-group overrides win on key + // collisions (`EnvVarSet::with_value` overrides earlier entries with the same name). + let mut env_overrides = EnvVarSet::new(); + for (name, value) in role_env_overrides + .iter() + .chain(role_group_env_overrides.iter()) + { + env_overrides = env_overrides.with_value( + &EnvVarName::from_str(name).context(ParseEnvVarNameSnafu)?, + value.clone(), + ); + } + Ok(env_overrides) +} + +fn merged_cli_overrides( + role_cli_overrides: BTreeMap, + role_group_cli_overrides: BTreeMap, +) -> BTreeMap { + let mut merged = role_cli_overrides; + merged.extend(role_group_cli_overrides); + merged +} + +fn merged_pod_overrides( + role_pod_overrides: PodTemplateSpec, + role_group_pod_overrides: PodTemplateSpec, +) -> PodTemplateSpec { + let mut merged = role_pod_overrides; + merged.merge_from(role_group_pod_overrides); + merged +} diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/kafka_controller.rs similarity index 74% rename from rust/operator-binary/src/controller.rs rename to rust/operator-binary/src/kafka_controller.rs index a064c1ac..e1186e10 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/kafka_controller.rs @@ -1,9 +1,8 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha1::KafkaCluster`]. -use std::{str::FromStr, sync::Arc}; +use std::sync::Arc; use const_format::concatcp; -use product_config::ProductConfigManager; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, @@ -16,7 +15,9 @@ use stackable_operator::{ core::{DeserializeGuard, error_boundary}, runtime::{controller::Action, reflector::ObjectRef}, }, + kvp::ObjectLabels, logging::controller::ReconcilerError, + memory::{BinaryMultiple, MemoryQuantity}, role_utils::{GenericRoleConfig, RoleGroupRef}, shared::time::Duration, status::condition::{ @@ -26,20 +27,16 @@ use stackable_operator::{ }; use strum::{EnumDiscriminants, IntoStaticStr}; -mod dereference; -mod validate; - use crate::{ + controller::{build, dereference, validate}, crd::{ - self, APP_NAME, KafkaClusterStatus, OPERATOR_NAME, + APP_NAME, KafkaClusterStatus, OPERATOR_NAME, listener::get_kafka_listener_config, role::{AnyConfig, KafkaRole}, v1alpha1, }, - discovery::{self, build_discovery_configmap}, operations::pdb::add_pdbs, resource::{ - configmap::build_rolegroup_config_map, listener::build_broker_rolegroup_bootstrap_listener, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, statefulset::{build_broker_rolegroup_statefulset, build_controller_rolegroup_statefulset}, @@ -49,9 +46,36 @@ use crate::{ pub const KAFKA_CONTROLLER_NAME: &str = "kafkacluster"; pub const KAFKA_FULL_CONTROLLER_NAME: &str = concatcp!(KAFKA_CONTROLLER_NAME, '.', OPERATOR_NAME); +/// The maximum size of a single Kafka log file before it is rotated. +pub const MAX_KAFKA_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 10.0, + unit: BinaryMultiple::Mebi, +}; + +/// Build recommended values for labels. +/// +/// Generic over the owner `T` so the owner can be either the raw `KafkaCluster` or the +/// `ValidatedCluster` (which also implements `Resource`). +pub fn build_recommended_labels<'a, T>( + owner: &'a T, + controller_name: &'a str, + app_version: &'a str, + role: &'a str, + role_group: &'a str, +) -> ObjectLabels<'a, T> { + ObjectLabels { + owner, + app_name: APP_NAME, + app_version, + operator_name: OPERATOR_NAME, + controller_name, + role, + role_group, + } +} + pub struct Ctx { pub client: stackable_operator::client::Client, - pub product_config: ProductConfigManager, pub operator_environment: OperatorEnvironmentOptions, } @@ -65,16 +89,13 @@ pub enum Error { #[snafu(display("failed to validate cluster"))] ValidateCluster { source: validate::Error }, - #[snafu(display("failed to build pod descriptors"))] - BuildPodDescriptors { source: crate::crd::Error }, - #[snafu(display("invalid kafka listeners"))] InvalidKafkaListeners { source: crate::crd::listener::KafkaListenerError, }, - #[snafu(display("failed to apply role Service"))] - ApplyRoleService { + #[snafu(display("failed to apply bootstrap Listener"))] + ApplyBootstrapListener { source: stackable_operator::cluster_resources::Error, }, @@ -97,7 +118,7 @@ pub enum Error { }, #[snafu(display("failed to build discovery ConfigMap"))] - BuildDiscoveryConfig { source: discovery::Error }, + BuildDiscoveryConfig { source: build::discovery::Error }, #[snafu(display("failed to apply discovery ConfigMap"))] ApplyDiscoveryConfig { @@ -114,9 +135,6 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::role::Error }, - #[snafu(display("failed to patch service account"))] ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, @@ -153,12 +171,6 @@ pub enum Error { source: error_boundary::InvalidObject, }, - #[snafu(display("KafkaCluster object is misconfigured"))] - MisconfiguredKafkaCluster { source: crd::Error }, - - #[snafu(display("failed to parse role: {source}"))] - ParseRole { source: strum::ParseError }, - #[snafu(display("failed to build statefulset"))] BuildStatefulset { source: crate::resource::statefulset::Error, @@ -166,7 +178,7 @@ pub enum Error { #[snafu(display("failed to build configmap"))] BuildConfigMap { - source: crate::resource::configmap::Error, + source: crate::controller::build::config_map::Error, }, #[snafu(display("failed to build service"))] @@ -190,7 +202,7 @@ impl ReconcilerError for Error { match self { Error::Dereference { .. } => None, Error::ValidateCluster { .. } => None, - Error::ApplyRoleService { .. } => None, + Error::ApplyBootstrapListener { .. } => None, Error::ApplyRoleGroupService { .. } => None, Error::ApplyRoleGroupConfig { .. } => None, Error::ApplyRoleGroupStatefulSet { .. } => None, @@ -198,7 +210,6 @@ impl ReconcilerError for Error { Error::ApplyDiscoveryConfig { .. } => None, Error::DeleteOrphans { .. } => None, Error::CreateClusterResources { .. } => None, - Error::FailedToResolveConfig { .. } => None, Error::ApplyServiceAccount { .. } => None, Error::ApplyRoleBinding { .. } => None, Error::ApplyStatus { .. } => None, @@ -206,14 +217,11 @@ impl ReconcilerError for Error { Error::FailedToCreatePdb { .. } => None, Error::GetRequiredLabels { .. } => None, Error::InvalidKafkaCluster { .. } => None, - Error::MisconfiguredKafkaCluster { .. } => None, - Error::ParseRole { .. } => None, Error::BuildStatefulset { .. } => None, Error::BuildConfigMap { .. } => None, Error::BuildService { .. } => None, Error::BuildListener { .. } => None, Error::InvalidKafkaListeners { .. } => None, - Error::BuildPodDescriptors { .. } => None, } } } @@ -238,38 +246,25 @@ pub async fn reconcile_kafka( .context(DereferenceSnafu)?; // validate (no client required) - let validate::ValidatedInputs { - authorization_config, - image, - kafka_security, - role_config: validated_config, - } = validate::validate( - kafka, - dereferenced_objects, - &ctx.operator_environment, - &ctx.product_config, - ) - .context(ValidateClusterSnafu)?; - - let opa_connect = authorization_config - .as_ref() - .map(|auth_config| auth_config.opa_connect.clone()); + let validated_cluster = + validate::validate(kafka, dereferenced_objects, &ctx.operator_environment) + .context(ValidateClusterSnafu)?; let mut cluster_resources = ClusterResources::new( APP_NAME, OPERATOR_NAME, KAFKA_CONTROLLER_NAME, - &kafka.object_ref(&()), + &validated_cluster.object_ref(&()), ClusterResourceApplyStrategy::from(&kafka.spec.cluster_operation), &kafka.spec.object_overrides, ) .context(CreateClusterResourcesSnafu)?; tracing::debug!( - kerberos_enabled = kafka_security.has_kerberos_enabled(), - kerberos_secret_class = ?kafka_security.kerberos_secret_class(), - tls_enabled = kafka_security.tls_enabled(), - tls_client_authentication_class = ?kafka_security.tls_client_authentication_class(), + kerberos_enabled = validated_cluster.cluster_config.kafka_security.has_kerberos_enabled(), + kerberos_secret_class = ?validated_cluster.cluster_config.kafka_security.kerberos_secret_class(), + tls_enabled = validated_cluster.cluster_config.kafka_security.tls_enabled(), + tls_client_authentication_class = ?validated_cluster.cluster_config.kafka_security.tls_client_authentication_class(), "The following security settings are used" ); @@ -295,93 +290,77 @@ pub async fn reconcile_kafka( let mut bootstrap_listeners = Vec::::new(); - for (kafka_role_str, role_config) in &validated_config { - let kafka_role = KafkaRole::from_str(kafka_role_str).context(ParseRoleSnafu)?; - - for (rolegroup_name, rolegroup_config) in role_config.iter() { - let rolegroup_ref = kafka.rolegroup_ref(&kafka_role, rolegroup_name); + for (kafka_role, rg_map) in &validated_cluster.role_group_configs { + for (rolegroup_name, validated_rg) in rg_map { + let rolegroup_ref = kafka.rolegroup_ref(kafka_role, rolegroup_name); - let merged_config = kafka_role - .merged_config(kafka, &rolegroup_ref.role_group) - .context(FailedToResolveConfigSnafu)?; - - let rg_headless_service = - build_rolegroup_headless_service(kafka, &image, &rolegroup_ref, &kafka_security) - .context(BuildServiceSnafu)?; + let rg_headless_service = build_rolegroup_headless_service( + &validated_cluster, + &validated_cluster.image, + &rolegroup_ref, + &validated_cluster.cluster_config.kafka_security, + ) + .context(BuildServiceSnafu)?; - let rg_metrics_service = build_rolegroup_metrics_service(kafka, &image, &rolegroup_ref) - .context(BuildServiceSnafu)?; + let rg_metrics_service = build_rolegroup_metrics_service( + &validated_cluster, + &validated_cluster.image, + &rolegroup_ref, + ) + .context(BuildServiceSnafu)?; let kafka_listeners = get_kafka_listener_config( kafka, - &kafka_security, + &validated_cluster.cluster_config.kafka_security, &rolegroup_ref, &client.kubernetes_cluster_info, ) .context(InvalidKafkaListenersSnafu)?; - let pod_descriptors = kafka - .pod_descriptors( - None, - &client.kubernetes_cluster_info, - kafka_security.client_port(), - ) - .context(BuildPodDescriptorsSnafu)?; - - let rg_configmap = build_rolegroup_config_map( - kafka, - &image, - &kafka_security, + let rg_configmap = build::config_map::build_rolegroup_config_map( + &validated_cluster, &rolegroup_ref, - rolegroup_config, - &merged_config, + validated_rg, &kafka_listeners, - &pod_descriptors, - opa_connect.as_deref(), ) .context(BuildConfigMapSnafu)?; let rg_statefulset = match kafka_role { KafkaRole::Broker => build_broker_rolegroup_statefulset( kafka, - &kafka_role, - &image, + kafka_role, + &validated_cluster, &rolegroup_ref, - rolegroup_config, - &kafka_security, - &merged_config, + validated_rg, &rbac_sa, &client.kubernetes_cluster_info, ) .context(BuildStatefulsetSnafu)?, KafkaRole::Controller => build_controller_rolegroup_statefulset( kafka, - &kafka_role, - &image, + kafka_role, + &validated_cluster, &rolegroup_ref, - rolegroup_config, - &kafka_security, - &merged_config, + validated_rg, &rbac_sa, &client.kubernetes_cluster_info, ) .context(BuildStatefulsetSnafu)?, }; - if let AnyConfig::Broker(broker_config) = merged_config { + if let AnyConfig::Broker(broker_config) = &validated_rg.config { let rg_bootstrap_listener = build_broker_rolegroup_bootstrap_listener( kafka, - &image, - &kafka_security, + &validated_cluster, &rolegroup_ref, - &broker_config, + broker_config, ) .context(BuildListenerSnafu)?; bootstrap_listeners.push( cluster_resources .add(client, rg_bootstrap_listener) .await - .context(ApplyRoleServiceSnafu)?, + .context(ApplyBootstrapListenerSnafu)?, ); } @@ -417,19 +396,19 @@ pub async fn reconcile_kafka( ); } - let role_config = kafka.role_config(&kafka_role); + let role_cfg = kafka.role_config(kafka_role); if let Some(GenericRoleConfig { pod_disruption_budget: pdb, - }) = role_config + }) = role_cfg { - add_pdbs(pdb, kafka, &kafka_role, client, &mut cluster_resources) + add_pdbs(pdb, kafka, kafka_role, client, &mut cluster_resources) .await .context(FailedToCreatePdbSnafu)?; } } let discovery_cm = - build_discovery_configmap(kafka, kafka, &image, &kafka_security, &bootstrap_listeners) + build::discovery::build_discovery_configmap(&validated_cluster, &bootstrap_listeners) .context(BuildDiscoveryConfigSnafu)?; cluster_resources diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index c074f8af..fd1e1b9a 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -35,20 +35,19 @@ use stackable_operator::{ }; use crate::{ - controller::KAFKA_FULL_CONTROLLER_NAME, crd::{KafkaCluster, KafkaClusterVersion, OPERATOR_NAME, v1alpha1}, + kafka_controller::KAFKA_FULL_CONTROLLER_NAME, webhooks::conversion::create_webhook_server, }; mod config; mod controller; mod crd; -mod discovery; +mod framework; +mod kafka_controller; mod kerberos; mod operations; -mod product_logging; mod resource; -mod utils; mod webhooks; mod built_info { @@ -80,9 +79,9 @@ async fn main() -> anyhow::Result<()> { RunArguments { operator_environment, watch_namespace, - product_config, maintenance, common, + .. }, .. }) => { @@ -127,11 +126,6 @@ async fn main() -> anyhow::Result<()> { .run(sigterm_watcher.handle()) .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let product_config = product_config.load(&[ - "deploy/config-spec/properties.yaml", - "/etc/stackable/kafka-operator/config-spec/properties.yaml", - ])?; - let event_recorder = Arc::new(Recorder::new( client.as_kube_client(), Reporter { @@ -183,12 +177,11 @@ async fn main() -> anyhow::Result<()> { ) .graceful_shutdown_on(sigterm_watcher.handle()) .run( - controller::reconcile_kafka, - controller::error_policy, - Arc::new(controller::Ctx { + kafka_controller::reconcile_kafka, + kafka_controller::error_policy, + Arc::new(kafka_controller::Ctx { client: client.clone(), operator_environment, - product_config, }), ) // We can let the reporting happen in the background diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index e42888ca..18f46dc7 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -5,8 +5,8 @@ use stackable_operator::{ }; use crate::{ - controller::KAFKA_CONTROLLER_NAME, crd::{APP_NAME, OPERATOR_NAME, role::KafkaRole, v1alpha1}, + kafka_controller::KAFKA_CONTROLLER_NAME, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs deleted file mode 100644 index 8336f5f7..00000000 --- a/rust/operator-binary/src/product_logging.rs +++ /dev/null @@ -1,152 +0,0 @@ -use std::{borrow::Cow, fmt::Display}; - -use stackable_operator::{ - builder::configmap::ConfigMapBuilder, - memory::{BinaryMultiple, MemoryQuantity}, - product_logging::{ - self, - spec::{ContainerLogConfig, ContainerLogConfigChoice}, - }, - role_utils::RoleGroupRef, -}; - -use crate::crd::{ - role::{AnyConfig, broker::BrokerContainer, controller::ControllerContainer}, - v1alpha1, -}; - -pub const BROKER_ID_POD_MAP_DIR: &str = "/stackable/broker-id-pod-map"; -pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config"; -pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; -// log4j -pub const LOG4J_CONFIG_FILE: &str = "log4j.properties"; -pub const KAFKA_LOG4J_FILE: &str = "kafka.log4j.xml"; -// log4j2 -pub const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; -pub const KAFKA_LOG4J2_FILE: &str = "kafka.log4j2.xml"; -// max size -pub const MAX_KAFKA_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { - value: 10.0, - unit: BinaryMultiple::Mebi, -}; - -const CONSOLE_CONVERSION_PATTERN_LOG4J: &str = "[%d] %p %m (%c)%n"; -const CONSOLE_CONVERSION_PATTERN_LOG4J2: &str = "%d{ISO8601} %p [%t] %c - %m%n"; - -pub fn kafka_log_opts(product_version: &str) -> String { - if product_version.starts_with("3.") { - format!("-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J_CONFIG_FILE}") - } else { - format!("-Dlog4j2.configurationFile=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J2_CONFIG_FILE}") - } -} - -pub fn kafka_log_opts_env_var() -> String { - "KAFKA_LOG4J_OPTS".to_string() -} - -/// Extend the role group ConfigMap with logging and Vector configurations -pub fn extend_role_group_config_map( - product_version: &str, - rolegroup: &RoleGroupRef, - merged_config: &AnyConfig, - cm_builder: &mut ConfigMapBuilder, -) { - let container_name = match merged_config { - AnyConfig::Broker(_) => BrokerContainer::Kafka.to_string(), - AnyConfig::Controller(_) => ControllerContainer::Kafka.to_string(), - }; - - // Starting with Kafka 4.0, log4j2 is used instead of log4j. - match product_version.starts_with("3.") { - true => add_log4j_config_if_automatic( - cm_builder, - Some(merged_config.kafka_logging()), - LOG4J_CONFIG_FILE, - container_name, - KAFKA_LOG4J_FILE, - MAX_KAFKA_LOG_FILES_SIZE, - ), - false => add_log4j2_config_if_automatic( - cm_builder, - Some(merged_config.kafka_logging()), - LOG4J2_CONFIG_FILE, - container_name, - KAFKA_LOG4J2_FILE, - MAX_KAFKA_LOG_FILES_SIZE, - ), - } - - let vector_log_config = merged_config.vector_logging(); - let vector_log_config = if let ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - } = &*vector_log_config - { - Some(log_config) - } else { - None - }; - - if merged_config.vector_logging_enabled() { - cm_builder.add_data( - product_logging::framework::VECTOR_CONFIG_FILE, - product_logging::framework::create_vector_config(rolegroup, vector_log_config), - ); - } -} - -fn add_log4j_config_if_automatic( - cm_builder: &mut ConfigMapBuilder, - log_config: Option>, - log_config_file: &str, - container_name: impl Display, - log_file: &str, - max_log_file_size: MemoryQuantity, -) { - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = log_config.as_deref() - { - cm_builder.add_data( - log_config_file, - product_logging::framework::create_log4j_config( - &format!("{STACKABLE_LOG_DIR}/{container_name}"), - log_file, - max_log_file_size - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32, - CONSOLE_CONVERSION_PATTERN_LOG4J, - log_config, - ), - ); - } -} - -fn add_log4j2_config_if_automatic( - cm_builder: &mut ConfigMapBuilder, - log_config: Option>, - log_config_file: &str, - container_name: impl Display, - log_file: &str, - max_log_file_size: MemoryQuantity, -) { - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = log_config.as_deref() - { - cm_builder.add_data( - log_config_file, - product_logging::framework::create_log4j2_config( - &format!("{STACKABLE_LOG_DIR}/{container_name}",), - log_file, - max_log_file_size - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32, - CONSOLE_CONVERSION_PATTERN_LOG4J2, - log_config, - ), - ); - } -} diff --git a/rust/operator-binary/src/resource/configmap.rs b/rust/operator-binary/src/resource/configmap.rs deleted file mode 100644 index 473120b3..00000000 --- a/rust/operator-binary/src/resource/configmap.rs +++ /dev/null @@ -1,421 +0,0 @@ -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, -}; - -use indoc::formatdoc; -use product_config::{types::PropertyNameKind, writer::to_java_properties_string}; -use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_operator::{ - builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, - k8s_openapi::api::core::v1::ConfigMap, - role_utils::RoleGroupRef, -}; - -use crate::{ - controller::KAFKA_CONTROLLER_NAME, - crd::{ - JVM_SECURITY_PROPERTIES_FILE, KafkaPodDescriptor, MetadataManager, - STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, - listener::{KafkaListenerConfig, KafkaListenerName, node_address_cmd}, - role::{ - AnyConfig, KAFKA_ADVERTISED_LISTENERS, KAFKA_BROKER_ID, - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, - KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, KAFKA_PROCESS_ROLES, KafkaRole, - }, - security::KafkaTlsSecurity, - v1alpha1, - }, - operations::graceful_shutdown::graceful_shutdown_config_properties, - product_logging::extend_role_group_config_map, - utils::build_recommended_labels, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("invalid metadata manager"))] - InvalidMetadataManager { source: crate::crd::Error }, - - #[snafu(display("failed to build ConfigMap for {}", rolegroup))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, - }, - - #[snafu(display( - "failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", - rolegroup - ))] - JvmSecurityProperties { - source: product_config::writer::PropertiesWriterError, - rolegroup: String, - }, - - #[snafu(display("failed to build Metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to serialize config for {rolegroup}"))] - SerializeConfig { - source: product_config::writer::PropertiesWriterError, - rolegroup: RoleGroupRef, - }, - - #[snafu(display("no Kraft controllers found to build"))] - NoKraftControllersFound, - - #[snafu(display("unknown Kafka role [{name}]"))] - UnknownKafkaRole { - source: strum::ParseError, - name: String, - }, - - #[snafu(display("failed to build jaas configuration file for {rolegroup}"))] - BuildJaasConfig { rolegroup: String }, -} - -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] -pub fn build_rolegroup_config_map( - kafka: &v1alpha1::KafkaCluster, - resolved_product_image: &ResolvedProductImage, - kafka_security: &KafkaTlsSecurity, - rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, - merged_config: &AnyConfig, - listener_config: &KafkaListenerConfig, - pod_descriptors: &[KafkaPodDescriptor], - opa_connect_string: Option<&str>, -) -> Result { - let kafka_config_file_name = merged_config.config_file_name(); - - let metadata_manager = kafka - .effective_metadata_manager() - .context(InvalidMetadataManagerSnafu)?; - - let mut kafka_config = server_properties_file( - metadata_manager == MetadataManager::KRaft, - &rolegroup.role, - pod_descriptors, - listener_config, - opa_connect_string, - kafka - .spec - .cluster_config - .broker_id_pod_config_map_name - .is_some(), - )?; - - match merged_config { - AnyConfig::Broker(_) => kafka_config.extend(kafka_security.broker_config_settings()), - AnyConfig::Controller(_) => { - kafka_config.extend(kafka_security.controller_config_settings()) - } - } - - kafka_config.extend(graceful_shutdown_config_properties()); - - // Need to call this to get configOverrides :( - kafka_config.extend( - rolegroup_config - .get(&PropertyNameKind::File(kafka_config_file_name.to_string())) - .cloned() - .unwrap_or_default(), - ); - - let kafka_config = kafka_config - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect::>(); - - let jvm_sec_props: BTreeMap> = rolegroup_config - .get(&PropertyNameKind::File( - JVM_SECURITY_PROPERTIES_FILE.to_string(), - )) - .cloned() - .unwrap_or_default() - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect(); - - let mut cm_builder = ConfigMapBuilder::new(); - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(kafka) - .name(rolegroup.object_name()) - .ownerreference_from_resource(kafka, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - kafka, - KAFKA_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .build(), - ) - .add_data( - kafka_config_file_name, - to_java_properties_string(kafka_config.iter().map(|(k, v)| (k, v))).with_context( - |_| SerializeConfigSnafu { - rolegroup: rolegroup.clone(), - }, - )?, - ) - .add_data( - JVM_SECURITY_PROPERTIES_FILE, - to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { - JvmSecurityPropertiesSnafu { - rolegroup: rolegroup.role_group.clone(), - } - })?, - ) - .add_data( - "client.properties", - to_java_properties_string( - kafka_security - .client_properties() - .iter() - .map(|(k, v)| (k, v)), - ) - .with_context(|_| JvmSecurityPropertiesSnafu { - rolegroup: rolegroup.role_group.clone(), - })?, - ) - // This file contains the JAAS configuration for Kerberos authentication - // It has the ".properties" extension but is not a Java properties file. - // It is processed by `config-utils` to substitute "env:" and "file:" variables - // and this tool currently doesn't support the JAAS login configuration format. - .add_data( - "jaas.properties", - jaas_config_file(kafka_security.has_kerberos_enabled()), - ); - - tracing::debug!(?kafka_config, "Applied kafka config"); - tracing::debug!(?jvm_sec_props, "Applied JVM config"); - - extend_role_group_config_map( - &resolved_product_image.product_version, - rolegroup, - merged_config, - &mut cm_builder, - ); - - cm_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) -} - -// Generate the content of both broker.properties and controller.properties files. -fn server_properties_file( - kraft_mode: bool, - role: &str, - pod_descriptors: &[KafkaPodDescriptor], - listener_config: &KafkaListenerConfig, - opa_connect_string: Option<&str>, - disable_broker_id_generation: bool, -) -> Result, Error> { - let kraft_controllers = kraft_controllers(pod_descriptors); - - let role = KafkaRole::from_str(role).context(UnknownKafkaRoleSnafu { - name: role.to_string(), - })?; - - match role { - KafkaRole::Controller => { - let kraft_controllers = kraft_controllers.context(NoKraftControllersFoundSnafu)?; - - let mut result = BTreeMap::from([ - ( - KAFKA_LOG_DIRS.to_string(), - "/stackable/data/kraft".to_string(), - ), - (KAFKA_PROCESS_ROLES.to_string(), role.to_string()), - ( - "controller.listener.names".to_string(), - KafkaListenerName::Controller.to_string(), - ), - ( - KAFKA_NODE_ID.to_string(), - "${env:REPLICA_ID}".to_string(), - ), - ( - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), - kraft_controllers.clone(), - ), - ( - KAFKA_LISTENERS.to_string(), - "CONTROLLER://${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}:${env:KAFKA_CLIENT_PORT}".to_string(), - ), - ( - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), - listener_config - .listener_security_protocol_map_for_controller()), - ]); - - result.insert( - "inter.broker.listener.name".to_string(), - KafkaListenerName::Internal.to_string(), - ); - - // The ZooKeeper connection is needed for migration from ZooKeeper to KRaft mode. - // It is not needed once the controller is fully running in KRaft mode. - if !kraft_mode { - result.insert( - "zookeeper.connect".to_string(), - "${env:ZOOKEEPER}".to_string(), - ); - } - Ok(result) - } - KafkaRole::Broker => { - let mut result = BTreeMap::from([ - ( - KAFKA_LOG_DIRS.to_string(), - "/stackable/data/topicdata".to_string(), - ), - (KAFKA_LISTENERS.to_string(), listener_config.listeners()), - ( - KAFKA_ADVERTISED_LISTENERS.to_string(), - listener_config.advertised_listeners(), - ), - ( - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), - listener_config.listener_security_protocol_map(), - ), - ( - "inter.broker.listener.name".to_string(), - KafkaListenerName::Internal.to_string(), - ), - ]); - - if kraft_mode { - let kraft_controllers = kraft_controllers.context(NoKraftControllersFoundSnafu)?; - - // Running in KRaft mode - result.extend([ - ( - "broker.id.generation.enable".to_string(), - "false".to_string(), - ), - (KAFKA_NODE_ID.to_string(), "${env:REPLICA_ID}".to_string()), - ( - KAFKA_PROCESS_ROLES.to_string(), - KafkaRole::Broker.to_string(), - ), - ( - "controller.listener.names".to_string(), - KafkaListenerName::Controller.to_string(), - ), - ( - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), - kraft_controllers.clone(), - ), - ]); - } else { - // Running with ZooKeeper enabled - result.extend([( - "zookeeper.connect".to_string(), - "${env:ZOOKEEPER}".to_string(), - )]); - // We are in zookeeper mode and the user has defined a broker id mapping - // so we disable automatic id generation. - // This check ensures that existing clusters running in ZooKeeper mode do not - // suddenly break after the introduction of this change. - if disable_broker_id_generation { - result.extend([ - ( - "broker.id.generation.enable".to_string(), - "false".to_string(), - ), - (KAFKA_BROKER_ID.to_string(), "${env:REPLICA_ID}".to_string()), - ]); - } - } - - // Enable OPA authorization - if opa_connect_string.is_some() { - result.extend([ - ( - "authorizer.class.name".to_string(), - "org.openpolicyagent.kafka.OpaAuthorizer".to_string(), - ), - ( - "opa.authorizer.metrics.enabled".to_string(), - "true".to_string(), - ), - ( - "opa.authorizer.url".to_string(), - opa_connect_string.unwrap_or_default().to_string(), - ), - ]); - } - - Ok(result) - } - } -} - -fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Option { - let result = pod_descriptors - .iter() - .filter(|pd| pd.role == KafkaRole::Controller.to_string()) - .map(|desc| { - format!( - "{fqdn}:{client_port}", - fqdn = desc.fqdn(), - client_port = desc.client_port - ) - }) - .collect::>() - .join(","); - - if result.is_empty() { - None - } else { - Some(result) - } -} - -// Generate JAAS configuration file for Kerberos authentication -// or an empty string if Kerberos is not enabled. -// See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html -fn jaas_config_file(is_kerberos_enabled: bool) -> String { - match is_kerberos_enabled { - false => String::new(), - true => formatdoc! {" - bootstrap.KafkaServer {{ - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - isInitiator=false - keyTab=\"/stackable/kerberos/keytab\" - principal=\"kafka/{bootstrap_address}@${{env:KERBEROS_REALM}}\"; - }}; - - client.KafkaServer {{ - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - isInitiator=false - keyTab=\"/stackable/kerberos/keytab\" - principal=\"kafka/{broker_address}@${{env:KERBEROS_REALM}}\"; - }}; - - ", - bootstrap_address = node_address_cmd(STACKABLE_LISTENER_BOOTSTRAP_DIR), - broker_address = node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), - }, - } -} diff --git a/rust/operator-binary/src/resource/listener.rs b/rust/operator-binary/src/resource/listener.rs index 4afde134..d22a2a96 100644 --- a/rust/operator-binary/src/resource/listener.rs +++ b/rust/operator-binary/src/resource/listener.rs @@ -1,13 +1,12 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::meta::ObjectMetaBuilder, commons::product_image_selection::ResolvedProductImage, - crd::listener, role_utils::RoleGroupRef, + builder::meta::ObjectMetaBuilder, crd::listener, role_utils::RoleGroupRef, }; use crate::{ - controller::KAFKA_CONTROLLER_NAME, + controller::ValidatedCluster, crd::{role::broker::BrokerConfig, security::KafkaTlsSecurity, v1alpha1}, - utils::build_recommended_labels, + kafka_controller::{KAFKA_CONTROLLER_NAME, build_recommended_labels}, }; #[derive(Snafu, Debug)] @@ -28,19 +27,21 @@ pub enum Error { // TODO (@NickLarsenNZ): Move shared functionality to stackable-operator pub fn build_broker_rolegroup_bootstrap_listener( kafka: &v1alpha1::KafkaCluster, - resolved_product_image: &ResolvedProductImage, - kafka_security: &KafkaTlsSecurity, + validated_cluster: &ValidatedCluster, rolegroup: &RoleGroupRef, merged_config: &BrokerConfig, ) -> Result { + let kafka_security = &validated_cluster.cluster_config.kafka_security; + let resolved_product_image = &validated_cluster.image; + Ok(listener::v1alpha1::Listener { metadata: ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(kafka.bootstrap_service_name(rolegroup)) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup.role, diff --git a/rust/operator-binary/src/resource/mod.rs b/rust/operator-binary/src/resource/mod.rs index a79483f8..514d0adb 100644 --- a/rust/operator-binary/src/resource/mod.rs +++ b/rust/operator-binary/src/resource/mod.rs @@ -1,4 +1,3 @@ -pub mod configmap; pub mod listener; pub mod service; pub mod statefulset; diff --git a/rust/operator-binary/src/resource/service.rs b/rust/operator-binary/src/resource/service.rs index f430fc8e..8f4fa0e3 100644 --- a/rust/operator-binary/src/resource/service.rs +++ b/rust/operator-binary/src/resource/service.rs @@ -8,9 +8,9 @@ use stackable_operator::{ }; use crate::{ - controller::KAFKA_CONTROLLER_NAME, + controller::ValidatedCluster, crd::{APP_NAME, METRICS_PORT, METRICS_PORT_NAME, security::KafkaTlsSecurity, v1alpha1}, - utils::build_recommended_labels, + kafka_controller::{KAFKA_CONTROLLER_NAME, build_recommended_labels}, }; #[derive(Snafu, Debug)] @@ -35,19 +35,19 @@ pub enum Error { /// /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. pub fn build_rolegroup_headless_service( - kafka: &v1alpha1::KafkaCluster, + validated_cluster: &ValidatedCluster, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, kafka_security: &KafkaTlsSecurity, ) -> Result { Ok(Service { metadata: ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(rolegroup.rolegroup_headless_service_name()) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup.role, @@ -60,7 +60,7 @@ pub fn build_rolegroup_headless_service( ports: Some(headless_ports(kafka_security)), selector: Some( Labels::role_group_selector( - kafka, + validated_cluster, APP_NAME, &rolegroup.role, &rolegroup.role_group, @@ -77,18 +77,18 @@ pub fn build_rolegroup_headless_service( /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label pub fn build_rolegroup_metrics_service( - kafka: &v1alpha1::KafkaCluster, + validated_cluster: &ValidatedCluster, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, ) -> Result { let metrics_service = Service { metadata: ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(rolegroup.rolegroup_metrics_service_name()) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup.role, @@ -105,7 +105,7 @@ pub fn build_rolegroup_metrics_service( ports: Some(metrics_ports()), selector: Some( Labels::role_group_selector( - kafka, + validated_cluster, APP_NAME, &rolegroup.role, &rolegroup.role_group, diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index 5cb262f0..3df261a2 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -1,9 +1,5 @@ -use std::{ - collections::{BTreeMap, HashMap}, - ops::Deref, -}; +use std::ops::Deref; -use product_config::types::PropertyNameKind; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -16,7 +12,6 @@ use stackable_operator::{ volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference, VolumeBuilder}, }, }, - commons::product_image_selection::ResolvedProductImage, constants::RESTART_CONTROLLER_ENABLED_LABEL, k8s_openapi::{ DeepMerge, @@ -45,29 +40,29 @@ use stackable_operator::{ use crate::{ config::{ - command::{broker_kafka_container_commands, controller_kafka_container_command}, + command::{ + broker_kafka_container_commands, controller_kafka_container_command, kafka_log_opts, + kafka_log_opts_env_var, + }, node_id_hasher::node_id_hash32_offset, }, - controller::KAFKA_CONTROLLER_NAME, + controller::{ValidatedCluster, ValidatedRoleGroupConfig}, crd::{ - self, APP_NAME, KAFKA_HEAP_OPTS, LISTENER_BOOTSTRAP_VOLUME_NAME, + self, APP_NAME, BROKER_ID_POD_MAP_DIR, KAFKA_HEAP_OPTS, LISTENER_BOOTSTRAP_VOLUME_NAME, LISTENER_BROKER_VOLUME_NAME, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, MetadataManager, STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, - STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, + STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, STACKABLE_LOG_CONFIG_DIR, + STACKABLE_LOG_DIR, role::{ - AnyConfig, KAFKA_NODE_ID_OFFSET, KafkaRole, broker::BrokerContainer, + KAFKA_NODE_ID_OFFSET, KafkaRole, broker::BrokerContainer, controller::ControllerContainer, }, security::KafkaTlsSecurity, v1alpha1, }, + kafka_controller::{KAFKA_CONTROLLER_NAME, MAX_KAFKA_LOG_FILES_SIZE, build_recommended_labels}, kerberos::add_kerberos_pod_config, operations::graceful_shutdown::add_graceful_shutdown_config, - product_logging::{ - BROKER_ID_POD_MAP_DIR, MAX_KAFKA_LOG_FILES_SIZE, STACKABLE_LOG_CONFIG_DIR, - STACKABLE_LOG_DIR, kafka_log_opts, kafka_log_opts_env_var, - }, - utils::build_recommended_labels, }; #[derive(Snafu, Debug)] @@ -123,11 +118,6 @@ pub enum Error { source: stackable_operator::builder::pod::container::Error, }, - #[snafu(display("invalid kafka listeners"))] - InvalidKafkaListeners { - source: crate::crd::listener::KafkaListenerError, - }, - #[snafu(display("failed to build Labels"))] LabelBuild { source: stackable_operator::kvp::LabelError, @@ -152,11 +142,6 @@ pub enum Error { #[snafu(display("failed to retrieve rolegroup replicas"))] RoleGroupReplicas { source: crd::role::Error }, - #[snafu(display( - "cluster does not define 'metadata.name' which is required for the Kafka cluster id" - ))] - ClusterIdMissing, - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] VectorAggregatorConfigMapMissing, } @@ -164,21 +149,21 @@ pub enum Error { /// The broker rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding -/// [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) from [`build_rolegroup_service`](`crate::resource::service::build_rolegroup_headless_service`). -#[allow(clippy::too_many_arguments)] +/// [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) from [`build_rolegroup_headless_service`](`crate::resource::service::build_rolegroup_headless_service`). pub fn build_broker_rolegroup_statefulset( kafka: &v1alpha1::KafkaCluster, kafka_role: &KafkaRole, - resolved_product_image: &ResolvedProductImage, + validated_cluster: &ValidatedCluster, rolegroup_ref: &RoleGroupRef, - broker_config: &HashMap>, - kafka_security: &KafkaTlsSecurity, - merged_config: &AnyConfig, + validated_rg: &ValidatedRoleGroupConfig, service_account: &ServiceAccount, cluster_info: &KubernetesClusterInfo, ) -> Result { + let kafka_security = &validated_cluster.cluster_config.kafka_security; + let resolved_product_image = &validated_cluster.image; + let merged_config = &validated_rg.config; let recommended_object_labels = build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup_ref.role, @@ -188,7 +173,7 @@ pub fn build_broker_rolegroup_statefulset( Labels::recommended(&recommended_object_labels).context(LabelBuildSnafu)?; // Used for PVC templates that cannot be modified once they are deployed let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, // A version value is required, and we do want to use the "recommended" format for the other desired labels "none", @@ -249,16 +234,7 @@ pub fn build_broker_rolegroup_statefulset( .context(AddKerberosConfigSnafu)?; } - let mut env = broker_config - .get(&PropertyNameKind::Env) - .into_iter() - .flatten() - .map(|(k, v)| EnvVar { - name: k.clone(), - value: Some(v.clone()), - ..EnvVar::default() - }) - .collect::>(); + let mut env = Vec::::from(validated_rg.env_overrides.clone()); if let Some(zookeeper_config_map_name) = &kafka.spec.cluster_config.zookeeper_config_map_name { env.push(EnvVar { @@ -532,12 +508,12 @@ pub fn build_broker_rolegroup_statefulset( Ok(StatefulSet { metadata: ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup_ref.role, @@ -575,20 +551,20 @@ pub fn build_broker_rolegroup_statefulset( } /// The controller rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -#[allow(clippy::too_many_arguments)] pub fn build_controller_rolegroup_statefulset( kafka: &v1alpha1::KafkaCluster, kafka_role: &KafkaRole, - resolved_product_image: &ResolvedProductImage, + validated_cluster: &ValidatedCluster, rolegroup_ref: &RoleGroupRef, - controller_config: &HashMap>, - kafka_security: &KafkaTlsSecurity, - merged_config: &AnyConfig, + validated_rg: &ValidatedRoleGroupConfig, service_account: &ServiceAccount, cluster_info: &KubernetesClusterInfo, ) -> Result { + let kafka_security = &validated_cluster.cluster_config.kafka_security; + let resolved_product_image = &validated_cluster.image; + let merged_config = &validated_rg.config; let recommended_object_labels = build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup_ref.role, @@ -603,16 +579,7 @@ pub fn build_controller_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); - let mut env = controller_config - .get(&PropertyNameKind::Env) - .into_iter() - .flatten() - .map(|(k, v)| EnvVar { - name: k.clone(), - value: Some(v.clone()), - ..EnvVar::default() - }) - .collect::>(); + let mut env = Vec::::from(validated_rg.env_overrides.clone()); env.push(EnvVar { name: "NAMESPACE".to_string(), @@ -859,12 +826,12 @@ pub fn build_controller_rolegroup_statefulset( Ok(StatefulSet { metadata: ObjectMetaBuilder::new() - .name_and_namespace(kafka) + .name_and_namespace(validated_cluster) .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(kafka, None, Some(true)) + .ownerreference_from_resource(validated_cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - kafka, + validated_cluster, KAFKA_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &rolegroup_ref.role, diff --git a/rust/operator-binary/src/utils.rs b/rust/operator-binary/src/utils.rs deleted file mode 100644 index 7abbafff..00000000 --- a/rust/operator-binary/src/utils.rs +++ /dev/null @@ -1,22 +0,0 @@ -use stackable_operator::kvp::ObjectLabels; - -use crate::crd::{APP_NAME, OPERATOR_NAME, v1alpha1}; - -/// Build recommended values for labels -pub fn build_recommended_labels<'a>( - owner: &'a v1alpha1::KafkaCluster, - controller_name: &'a str, - app_version: &'a str, - role: &'a str, - role_group: &'a str, -) -> ObjectLabels<'a, v1alpha1::KafkaCluster> { - ObjectLabels { - owner, - app_name: APP_NAME, - app_version, - operator_name: OPERATOR_NAME, - controller_name, - role, - role_group, - } -}