operator: KBS API for LUKS key registration

Makefile

@@ -30,8 +30,10 @@ OPERATOR_IMAGE=$(REGISTRY)/trusted-cluster-operator:$(TAG)
 COMPUTE_PCRS_IMAGE=$(REGISTRY)/compute-pcrs:$(TAG)
 REG_SERVER_IMAGE=$(REGISTRY)/registration-server:$(TAG)
 ATTESTATION_KEY_REGISTER_IMAGE=$(REGISTRY)/attestation-key-register:$(TAG)
-TRUSTEE_IMAGE ?= quay.io/trusted-execution-clusters/key-broker-service:v0.17.0
+
+TRUSTEE_IMAGE ?= quay.io/trusted-execution-clusters/key-broker-service:v0.20.0
 TEST_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:42.20260622
+
 # tagged as 42.20251012.2.0
 APPROVED_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos@sha256:6997f51fd27d1be1b5fc2e6cc3ebf16c17eb94d819b5d44ea8d6cf5f826ee773
 

operator/Cargo.toml

@@ -31,6 +31,9 @@ serde_json.workspace = true
 thiserror = "2.0.18"
 tokio.workspace = true
 toml = "1.1.2"
+kbs-client = {git = "https://github.com/confidential-containers/trustee.git", rev = "e65897a9ad4eb3ac69fa2ec75ed831200eb2acd7", default-features = false, features = ["native-tls"] }
+jsonwebtoken = { version = "10.4.0", default-features = false, features = ["use_pem"] }
+jsonwebtoken-openssl = "1.0.0"
 
 [dev-dependencies]
 http.workspace = true

operator/src/attestation_key_register.rs

@@ -331,7 +331,7 @@ async fn secret_reconcile(
         match ev {
             Event::Apply(_secret) => {
                 // On creation/update, just update the trustee deployment volumes
-                trustee::update_attestation_keys(&ctx)
+                trustee::update_attestation_keys(ctx.client.clone())
                     .await
                     .map(|_| Action::await_change())
                     .map_err(|e| {
@@ -345,7 +345,7 @@ async fn secret_reconcile(
                     "AttestationKey secret {secret_name} is being deleted, updating trustee deployment volumes"
                 );
                 // Update trustee deployment - secrets with deletion_timestamp will be filtered out
-                trustee::update_attestation_keys(&ctx)
+                trustee::update_attestation_keys(ctx.client.clone())
                     .await
                     .map(|_| Action::await_change())
                     .map_err(|e| {

operator/src/kbs-config.toml

@@ -2,36 +2,35 @@
 sockets = ["0.0.0.0:8080"]
 
 [admin]
-type = "DenyAll"
+authorization_mode = "AuthenticatedAuthorization"
+
+  [admin.authentication.bearer_jwt]
+  identity_providers = [
+    { public_key_uri = "/opt/trustee/keys/public.pub" }
+  ]
+
+  [admin.authorization.regex_acl]
+  acls = [{ role = "admin", allowed_endpoints = "^/kbs/.+$" }]
 
 [attestation_token]
-insecure_key = true
-attestation_token_type = "CoCo"
+insecure_header_jwk = true
 
 [attestation_service]
 type = "coco_as_builtin"
-work_dir = "/opt/trustee"
-policy_engine = "opa"
+timeout = 5
 
   [attestation_service.attestation_token_broker]
-  type = "Ear"
-  policy_dir = "/opt/trustee/policies"
-
-  [attestation_service.attestation_token_config]
   duration_min = 5
 
   [attestation_service.rvps_config]
   type = "BuiltIn"
 
-    [attestation_service.rvps_config.storage]
-    type = "LocalJson"
-    file_path = "/opt/trustee/reference-values.json"
-
-
 [[plugins]]
 name = "resource"
-type = "LocalFs"
-dir_path = "/opt/trustee/kbs-repository"
+storage_backend_type = "kvstorage"
+
+[storage_backend]
+storage_type = "LocalJson"
 
-[policy_engine]
-policy_path = "/opt/trustee/policy.rego"
+  [storage_backend.backends.local_json]
+  file_dir_path = "/opt/trustee/storage"

operator/src/main.rs

@@ -28,7 +28,6 @@ mod register_server;
 #[cfg(test)]
 mod test_utils;
 mod trustee;
-
 use crate::conditions::*;
 use operator::*;
 
@@ -134,8 +133,7 @@ async fn reconcile(
 
     install_trustee_configuration(kube_client.clone(), &cluster).await?;
     install_register_server(kube_client.clone(), &cluster).await?;
-    install_attestation_key_register(kube_client.clone(), &cluster).await?;
-    reference_values::adopt_approved_images(kube_client, &cluster).await?;
+    install_attestation_key_register(kube_client, &cluster).await?;
 
     let installed_condition = installed_condition(INSTALLED_REASON, generation, existing_status);
     let changed = upsert_condition(&mut conditions, installed_condition);
@@ -160,9 +158,20 @@ async fn install_trustee_configuration(
         Err(e) => error!("Failed to create the KBS configuration configmap: {e}"),
     }
 
-    match trustee::generate_attestation_policy(client.clone(), owner_reference.clone()).await {
-        Ok(_) => info!("Generate configmap for the attestation policy",),
-        Err(e) => error!("Failed to create the attestation policy configmap: {e}"),
+    match reference_values::create_pcrs_config_map(client.clone()).await {
+        Ok(_) => info!("Created bare configmap for PCRs"),
+        Err(e) => error!("Failed to create the PCRs configmap: {e}"),
+    }
+
+    match trustee::generate_trustee_auth_keys_secret(client.clone(), owner_reference.clone()).await
+    {
+        Ok(_) => info!("Generate auth keys for the KBS API",),
+        Err(e) => error!("Failed to create the auth keys: {e}"),
+    }
+
+    match trustee::generate_rv_data(client.clone(), owner_reference.clone()).await {
+        Ok(_) => info!("Created configmap for reference values"),
+        Err(e) => error!("Failed to create the reference values configmap: {e}"),
     }
 
     let kbs_port = cluster.spec.trustee_kbs_port;
@@ -251,7 +260,7 @@ async fn install_attestation_key_register(
 #[tokio::main]
 async fn main() -> Result<()> {
     env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
-
+    let _ = jsonwebtoken_openssl::install_default();
     let kube_client = Client::try_default().await?;
     info!("trusted execution clusters operator",);
 
@@ -282,9 +291,9 @@ async fn main() -> Result<()> {
     attestation_key_register::launch_ak_controller(ak_ctx.clone()).await;
     attestation_key_register::launch_machine_ak_controller(ak_ctx.clone()).await;
     attestation_key_register::launch_secret_ak_controller(ak_ctx).await;
-    reference_values::create_pcrs_config_map(kube_client.clone()).await?;
     reference_values::launch_rv_image_controller(kube_client.clone()).await;
     reference_values::launch_rv_job_controller(kube_client.clone()).await;
+    trustee::launch_trustee_sync_controller(kube_client.clone()).await;
 
     Controller::new(cl, watcher::Config::default())
         .run(reconcile, controller_error_policy, ctx)
@@ -298,9 +307,7 @@ async fn main() -> Result<()> {
 mod tests {
     use http::{Method, Request, StatusCode};
     use k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp};
-    use kube::api::ObjectList;
     use kube::client::Body;
-    use trusted_cluster_operator_lib::ApprovedImage;
 
     use super::*;
     use trusted_cluster_operator_test_utils::mock_client::*;
@@ -438,16 +445,9 @@ mod tests {
         };
 
         let clos = async |req: Request<Body>, ctr| {
-            if ctr < 8 && req.method() == Method::POST {
+            if ctr < 10 && req.method() == Method::POST {
                 Ok(serde_json::to_string(&dummy_cluster()).unwrap())
-            } else if ctr == 8 && req.method() == Method::GET {
-                let object_list = ObjectList::<ApprovedImage> {
-                    items: Vec::new(),
-                    types: Default::default(),
-                    metadata: Default::default(),
-                };
-                Ok(serde_json::to_string(&object_list).unwrap())
-            } else if ctr == 9 && req.method() == Method::PATCH {
+            } else if ctr == 10 && req.method() == Method::PATCH {
                 let body = req.into_body().collect_bytes().await.unwrap().to_vec();
                 let body = String::from_utf8_lossy(&body);
                 assert!(body.contains("ForeignCondition"),);
@@ -478,7 +478,7 @@ mod tests {
         cluster.status = Some(TrustedExecutionClusterStatus {
             conditions: Some(vec![pre_existing_installed, foreign_condition]),
         });
-        count_check!(10, clos, |client| {
+        count_check!(11, clos, |client| {
             let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await;
             assert_eq!(result.unwrap(), Action::await_change());
         });

operator/src/reference_values.rs

@@ -220,22 +220,6 @@ async fn adopt_approved_image(
     Ok(())
 }
 
-pub async fn adopt_approved_images(
-    client: Client,
-    cluster: &TrustedExecutionCluster,
-) -> Result<()> {
-    let images: Api<ApprovedImage> = Api::default_namespaced(client.clone());
-    let images_list = images.list(&Default::default()).await?;
-    for image in images_list.items.iter() {
-        if image.metadata.deletion_timestamp.is_none()
-            && let Some(name) = image.metadata.name.as_ref()
-        {
-            adopt_approved_image(client.clone(), name, cluster).await?;
-        }
-    }
-    Ok(())
-}
-
 async fn image_reconcile(
     image: Arc<ApprovedImage>,
     client: Arc<Client>,
@@ -247,6 +231,12 @@ async fn image_reconcile(
         .await
         .map_err(|e| -> ControllerError { e.into() })?;
 
+    if let Some(ref cluster) = cluster {
+        adopt_approved_image(kube_client.clone(), &name, cluster)
+            .await
+            .map_err(|e| -> ControllerError { e.into() })?;
+    }
+
     let images: Api<ApprovedImage> = Api::default_namespaced(kube_client.clone());
     finalizer(&images, APPROVED_IMAGE_FINALIZER, image, |ev| async {
         match ev {
@@ -277,19 +267,6 @@ async fn image_add_reconcile(
         info!("TrustedExecutionCluster is being deleted, deferring image processing for {name}");
         return Ok(Action::requeue(Duration::from_secs(5)));
     }
-    let uid_owns = |uid: &String| {
-        let refs = image.metadata.owner_references.as_ref();
-        refs.map(|os| os.iter().any(|o| o.uid == *uid))
-    };
-    let cluster_owns = |cluster: &TrustedExecutionCluster| {
-        let uid = cluster.metadata.uid.as_ref();
-        uid.and_then(uid_owns).unwrap_or(false)
-    };
-    // Adopt the image by adding TEC as owner reference if not already owned
-    if !cluster_owns(&cluster) {
-        adopt_approved_image(client.clone(), name, &cluster).await?;
-    }
-
     let (action, reason) = match handle_new_image(client.clone(), image).await {
         Ok(reason) => (Action::await_change(), reason),
         Err(e) => {
@@ -428,7 +405,6 @@ mod tests {
     use http::{Method, Request, StatusCode};
     use k8s_openapi::api::batch::v1::JobStatus;
     use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
-    use kube::api::ObjectList;
     use kube::client::Body;
     use trusted_cluster_operator_test_utils::mock_client::*;
     use trusted_cluster_operator_test_utils::test_error_method;
@@ -491,12 +467,13 @@ mod tests {
                 Ok(serde_json::to_string(&dummy_pcrs_map()).unwrap())
             }
             (2, &Method::GET) | (3, &Method::PUT) => {
-                assert!(req.uri().path().contains(trustee::TRUSTEE_DATA_MAP));
+                assert!(req.uri().path().contains(trustee::TRUSTEE_RV_MAP));
                 Ok(serde_json::to_string(&dummy_trustee_map()).unwrap())
             }
+            (4, &Method::GET) => Err(StatusCode::NOT_FOUND),
             _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"),
         };
-        count_check!(4, clos, |client| {
+        count_check!(5, clos, |client| {
             let job = Arc::new(dummy_job());
             let result = job_reconcile(job, Arc::new(client)).await.unwrap();
             assert_eq!(result, Action::await_change());
@@ -563,37 +540,6 @@ mod tests {
         test_error_method!(clos, Method::PATCH);
     }
 
-    #[tokio::test]
-    async fn test_adopt_approved_images() {
-        let cluster = dummy_cluster();
-        let clos = async |req: Request<_>, ctr| {
-            if ctr == 0 && req.method() == Method::GET {
-                let mut deleted = dummy_image();
-                deleted.metadata.deletion_timestamp = Some(Time(Timestamp::now()));
-                let list = ObjectList {
-                    items: vec![dummy_image(), deleted, dummy_image()],
-                    types: Default::default(),
-                    metadata: Default::default(),
-                };
-                Ok(serde_json::to_string(&list).unwrap())
-            } else if ctr < 3 && req.method() == Method::PATCH {
-                Ok(serde_json::to_string(&dummy_image()).unwrap())
-            } else {
-                panic!("unexpected API interaction: {req:?}, counter {ctr}")
-            }
-        };
-        count_check!(3, clos, |client| {
-            assert!(adopt_approved_images(client, &cluster).await.is_ok());
-        });
-    }
-
-    #[tokio::test]
-    async fn test_adopt_approved_images_error() {
-        let cluster = dummy_cluster();
-        let clos = |client| adopt_approved_images(client, &cluster);
-        test_error_method!(clos, Method::GET);
-    }
-
     // handle_new_image and its caller image_add_reconcile are
     // inherently online functions and not tested here
 
@@ -608,12 +554,13 @@ mod tests {
                 Ok(serde_json::to_string(&dummy_pcrs_map()).unwrap())
             }
             (3, &Method::GET) | (4, &Method::PUT) => {
-                assert!(req.uri().path().contains(trustee::TRUSTEE_DATA_MAP));
+                assert!(req.uri().path().contains(trustee::TRUSTEE_RV_MAP));
                 Ok(serde_json::to_string(&dummy_trustee_map()).unwrap())
             }
+            (5, &Method::GET) => Err(StatusCode::NOT_FOUND),
             _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"),
         };
-        count_check!(5, clos, |client| {
+        count_check!(6, clos, |client| {
             assert!(image_remove_reconcile(client, image, cluster).await.is_ok());
         });
     }

operator/src/register_server.rs

@@ -139,7 +139,7 @@ async fn keygen_reconcile(
                 async {
                     let owner_reference = generate_owner_reference(&Arc::unwrap_or_clone(machine))?;
                     trustee::generate_secret(kube_client.clone(), id, owner_reference).await?;
-                    trustee::mount_secret(kube_client, id).await
+                    trustee::send_secret(kube_client, id).await
                 }
                 .await
                 .map(|_| Action::await_change())
@@ -185,10 +185,12 @@ async fn keygen_reconcile(
                     }
                 }
 
-                trustee::unmount_secret(kube_client, id)
-                    .await
-                    .map(|_| Action::await_change())
-                    .map_err(|e| finalizer::Error::<ControllerError>::CleanupFailed(e.into()))
+                trustee::delete_secret(kube_client, id).await.map_err(|e| {
+                    finalizer::Error::<ControllerError>::CleanupFailed(
+                        anyhow!("failed to delete secret for machine {id}: {e}").into(),
+                    )
+                })?;
+                Ok(Action::await_change())
             }
         }
     })

operator/src/tpm.rego

@@ -9,6 +9,7 @@ executables := 3 if {
   input.tpm.pcr04 in query_reference_value("tpm_pcr4")
   input.tpm.pcr14 in query_reference_value("tpm_pcr14")
 
+  input.tpm.ak_public in query_reference_value("trusted_aks")
 }
 # Azure SNP vTPM validation
 executables := 3 if {