From b096dea59ef5b81e725262fdd095c43bcb4d3d0e Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 16 Jun 2026 12:08:35 +0200 Subject: [PATCH 01/14] make: Do not track Kind Azure host Kind CI, if we make one, will not run on Azure. This partially reverts commit cc6bf855dc7122a9e6fcb0d72732636c4d239213. Signed-off-by: Jakob Naucke --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 72d647e8..6331de99 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,6 @@ PLATFORM ?= kind KUBECTL=kubectl INTEGRATION_TEST_THREADS ?= 1 -# Azure CI only: which image to use as Kind host -KIND_HOST_URN = RedHat:RHEL:10-lvm-gen2:10.1.2026022409 LOCALBIN ?= $(shell pwd)/bin CONTROLLER_TOOLS_VERSION ?= $(shell go list -m -f '{{.Version}}' sigs.k8s.io/controller-tools) From 3e1608ea8ac3c51d1bfd042f71a47e4e51829d90 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 16 Jun 2026 12:08:46 +0200 Subject: [PATCH 02/14] Revert "ci: Avoid cargo for retrieving kopium version" This reverts commit 2e4f6d5128c0a5e36f3fffa65d99065615a96f4f. CI requires cargo anyhow. Signed-off-by: Jakob Naucke --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6331de99..ef0966ee 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ CONTROLLER_TOOLS_VERSION ?= $(shell go list -m -f '{{.Version}}' sigs.k8s.io/con CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) YQ_VERSION ?= $(shell go list -m -f '{{.Version}}' github.com/mikefarah/yq/v4) YQ ?= $(LOCALBIN)/yq-$(YQ_VERSION) -KOPIUM_VERSION ?= $(shell grep kopium lib/Cargo.toml | sed -E 's/.*"(.*)"/\1/') +KOPIUM_VERSION ?= $(shell cargo metadata --format-version 1 | jq -r '.resolve.nodes[] | select(.deps[]?.name == "kopium") | .deps[] | select(.name == "kopium") | .pkg | split("@")[1]') KOPIUM ?= $(LOCALBIN)/kopium-$(KOPIUM_VERSION) REGISTRY ?= quay.io/trusted-execution-clusters From c46fc90bb382ac2d451eb812eeb07ed1f5965ff5 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 16 Jun 2026 12:30:20 +0200 Subject: [PATCH 03/14] Fix `v` in image tag version should be v0.2.0, not 0.2.0 Signed-off-by: Jakob Naucke --- operator/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/src/main.rs b/operator/src/main.rs index d6fc9944..63307d37 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -35,7 +35,7 @@ use operator::*; /// Default tag for Trustee image const TRUSTEE_VERSION: &str = "v0.17.0"; /// Default version tag for operator-managed component images -const COMPONENT_VERSION: &str = "0.2.0"; +const COMPONENT_VERSION: &str = "v0.2.0"; /// Default registry const TEC_REGISTRY: &str = "quay.io/trusted-execution-clusters"; From 160eff37e8450e2d7980de3587473d837d6a4231 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Thu, 18 Jun 2026 14:31:26 +0200 Subject: [PATCH 04/14] tests: Fix a missing platform kubectl usage following #266 Signed-off-by: Jakob Naucke --- test_utils/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 85be2a94..b48ac667 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -773,7 +773,8 @@ impl TestContext { self.set_certificates().await?; let tec = "trustedexecutionclusters.trusted-execution-clusters.io"; let args = ["get", "crd", tec]; - let crd_check_output = Command::new("kubectl").args(args).output().await?; + let mut cmd = get_k8s_platform().kubectl(); + let crd_check_output = cmd.args(args).output().await?; if crd_check_output.status.success() { test_info!( From c08d8af81943d718858ea43ead4e7db902ed59c7 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 8 Jun 2026 10:56:21 +0200 Subject: [PATCH 05/14] tests: Use await_condition instead of polling also works for deployments that don't exist yet Signed-off-by: Jakob Naucke --- test_utils/src/lib.rs | 186 +++++------------- test_utils/src/virt/kubevirt.rs | 37 ++-- tests/attestation.rs | 2 +- tests/trusted_execution_cluster.rs | 304 ++++++++--------------------- 4 files changed, 140 insertions(+), 389 deletions(-) diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index b48ac667..f5ac36a2 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -3,15 +3,17 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Result, anyhow}; +use anyhow::{Context, Result, anyhow}; use fs_extra::dir; -use k8s_openapi::api::apps::v1::Deployment; +use k8s_openapi::api::apps::v1::{Deployment, DeploymentCondition, DeploymentStatus}; use k8s_openapi::api::core::v1::{ConfigMap, Namespace, Secret, Service, ServicePort, ServiceSpec}; use kube::api::{DeleteParams, ObjectMeta}; +use kube::runtime::wait::await_condition; use kube::{Api, Client}; use std::path::{Path, PathBuf}; use std::{collections::BTreeMap, env, sync::Once, time::Duration}; use tokio::process::Command; +use tokio::time::timeout; use trusted_cluster_operator_lib::certificates::{ Certificate, CertificateIssuerRef, CertificateSpec, }; @@ -468,7 +470,7 @@ impl TestContext { tec_api.delete(name, &dp).await?; // Wait for the resource to be deleted - wait_for_resource_deleted(&tec_api, name, scaled_timeout(120), 5).await?; + wait_for_resource_deleted(&tec_api, name, scaled_timeout(120)).await?; test_info!( &self.test_name, "TrustedExecutionCluster {} has been deleted", @@ -491,7 +493,7 @@ impl TestContext { "Waiting for Machine {} to be deleted", name ); - wait_for_resource_deleted(&machine_api, name, 120, 5).await?; + wait_for_resource_deleted(&machine_api, name, scaled_timeout(120)).await?; test_info!(&self.test_name, "Machine {} has been deleted", name); } } @@ -506,13 +508,8 @@ impl TestContext { match namespace_api.get(&self.test_namespace).await { Ok(_) => { namespace_api.delete(&self.test_namespace, &dp).await?; - wait_for_resource_deleted( - &namespace_api, - &self.test_namespace, - scaled_timeout(300), - 5, - ) - .await?; + let timeout = scaled_timeout(300); + wait_for_resource_deleted(&namespace_api, &self.test_namespace, timeout).await?; test_info!(&self.test_name, "Deleted namespace {}", self.test_namespace); } Err(kube::Error::Api(ae)) if ae.code == 404 => { @@ -547,49 +544,6 @@ impl TestContext { Ok(()) } - async fn wait_for_deployment_ready( - &self, - deployments_api: &Api, - deployment_name: &str, - timeout_secs: u64, - ) -> Result<()> { - test_info!( - &self.test_name, - "Waiting for deployment {} to be ready", - deployment_name - ); - let poller = Poller::new() - .with_timeout(Duration::from_secs(timeout_secs)) - .with_interval(Duration::from_secs(5)) - .with_error_message(format!( - "{deployment_name} deployment does not have 1 available replica after {timeout_secs} seconds" - )); - - let test_name_owned = self.test_name.clone(); - poller - .poll_async(move || { - let api = deployments_api.clone(); - let name = deployment_name.to_string(); - let tn = test_name_owned.clone(); - async move { - let deployment = api.get(&name).await?; - - if let Some(status) = &deployment.status - && let Some(available_replicas) = status.available_replicas - && available_replicas == 1 - { - test_info!(&tn, "{} deployment has 1 available replica", name); - return Ok(()); - } - - Err(anyhow!( - "{name} deployment does not have 1 available replica yet" - )) - } - }) - .await - } - async fn create_certificate( &self, service_name: &str, @@ -682,9 +636,9 @@ impl TestContext { .await?; let secrets: Api = Api::namespaced(self.client.clone(), &self.test_namespace); - wait_for_resource_created(&secrets, REG_SECRET, scaled_timeout(60), 1).await?; - wait_for_resource_created(&secrets, TRUSTEE_SECRET, scaled_timeout(60), 1).await?; - wait_for_resource_created(&secrets, ATT_REG_SECRET, scaled_timeout(60), 1).await?; + for secret in [REG_SECRET, TRUSTEE_SECRET, ATT_REG_SECRET] { + wait_for_resource_created(&secrets, secret, scaled_timeout(60)).await?; + } Ok(()) } @@ -934,28 +888,27 @@ impl TestContext { "Applying ApprovedImage manifest" ); - let deployments_api: Api = Api::namespaced(self.client.clone(), ns); + let depl_ready = |depl: Option<&Deployment>| { + let chk_cond = |c: &DeploymentCondition| c.type_ == "Available" && c.status == "True"; + let chk_status = + |st: &DeploymentStatus| st.conditions.as_ref().map(|cs| cs.iter().any(chk_cond)); + let chk = |depl: &Deployment| depl.status.as_ref().and_then(chk_status); + depl.and_then(chk).unwrap_or(false) + }; - self.wait_for_deployment_ready( - &deployments_api, + let depls: Api = Api::namespaced(self.client.clone(), ns); + for depl in [ "trusted-cluster-operator", - scaled_timeout(120), - ) - .await?; - self.wait_for_deployment_ready( - &deployments_api, REGISTER_SERVER_DEPLOYMENT, - scaled_timeout(300), - ) - .await?; - self.wait_for_deployment_ready(&deployments_api, TRUSTEE_DEPLOYMENT, scaled_timeout(180)) - .await?; - self.wait_for_deployment_ready( - &deployments_api, + TRUSTEE_DEPLOYMENT, ATTESTATION_KEY_REGISTER_DEPLOYMENT, - scaled_timeout(120), - ) - .await?; + ] { + let info = format!("Waiting for deployment {depl} to be ready"); + test_info!(&self.test_name, "{info}"); + let done = await_condition(depls.clone(), depl, depl_ready); + let ctx = format!("waiting for deployment {depl} to be ready"); + timeout(scaled_duration(300), done).await.context(ctx)??; + } let platform = get_k8s_platform(); let ak_port = ATTESTATION_KEY_REGISTER_PORT; @@ -974,28 +927,7 @@ impl TestContext { "Waiting for image-pcrs ConfigMap to be created" ); let configmap_api: Api = Api::namespaced(self.client.clone(), ns); - - let err = format!("image-pcrs ConfigMap in the namespace {ns} not found"); - let poller = Poller::new() - .with_timeout(scaled_duration(60)) - .with_interval(Duration::from_secs(5)) - .with_error_message(err); - - let test_name_owned = self.test_name.clone(); - let check_fn = move || { - let api = configmap_api.clone(); - let tn = test_name_owned.clone(); - async move { - let result = api.get("image-pcrs").await; - if result.is_ok() { - test_info!(&tn, "image-pcrs ConfigMap created"); - } - result - } - }; - poller.poll_async(check_fn).await?; - - Ok(()) + wait_for_resource_created(&configmap_api, "image-pcrs", scaled_timeout(60)).await } } @@ -1044,60 +976,34 @@ pub async fn wait_for_resource_created( api: &Api, resource_name: &str, timeout_secs: u64, - interval_secs: u64, -) -> anyhow::Result<()> -where - K: kube::Resource + Clone + std::fmt::Debug, - K: k8s_openapi::serde::de::DeserializeOwned, -{ - wait_for_resource_state(api, resource_name, timeout_secs, interval_secs, true).await -} - -pub async fn wait_for_resource_deleted( - api: &Api, - resource_name: &str, - timeout_secs: u64, - interval_secs: u64, ) -> Result<()> where - K: kube::Resource + Clone + std::fmt::Debug, + K: kube::Resource + Clone + std::fmt::Debug + Send + 'static, K: k8s_openapi::serde::de::DeserializeOwned, { - wait_for_resource_state(api, resource_name, timeout_secs, interval_secs, false).await + let created = |r: Option<&K>| r.is_some(); + let done = await_condition(api.clone(), resource_name, created); + let type_ = std::any::type_name::(); + let ctx = format!("waiting {timeout_secs} for {type_} '{resource_name}' creation"); + let duration = Duration::from_secs(timeout_secs); + timeout(duration, done).await.context(ctx)??; + Ok(()) } -async fn wait_for_resource_state( +pub async fn wait_for_resource_deleted( api: &Api, resource_name: &str, timeout_secs: u64, - interval_secs: u64, - state: bool, ) -> Result<()> where - K: kube::Resource + Clone + std::fmt::Debug, + K: kube::Resource + Clone + std::fmt::Debug + Send + 'static, K: k8s_openapi::serde::de::DeserializeOwned, { - let poller = Poller::new() - .with_timeout(Duration::from_secs(timeout_secs)) - .with_interval(Duration::from_secs(interval_secs)) - .with_error_message(format!( - "{resource_name} did not reach state {} after {timeout_secs} seconds", - if state { "created" } else { "deleted" } - )); - - let check = || { - let api = api.clone(); - let name = resource_name.to_string(); - async move { - let result = api.get(&name).await; - if let Err(kube::Error::Api(ae)) = &result - && ae.code != 404 - { - panic!("Unexpected error while fetching {name}: {ae:?}"); - } - let err = anyhow!("{name} not in desired state: {result:?}"); - (result.is_err() ^ state).then_some(()).ok_or(err) - } - }; - poller.poll_async(check).await + let deleted = |r: Option<&K>| r.is_none(); + let done = await_condition(api.clone(), resource_name, deleted); + let type_ = std::any::type_name::(); + let ctx = format!("waiting {timeout_secs} for {type_} '{resource_name}' deletion"); + let duration = Duration::from_secs(timeout_secs); + timeout(duration, done).await.context(ctx)??; + Ok(()) } diff --git a/test_utils/src/virt/kubevirt.rs b/test_utils/src/virt/kubevirt.rs index b4ad9fac..a19d0054 100644 --- a/test_utils/src/virt/kubevirt.rs +++ b/test_utils/src/virt/kubevirt.rs @@ -5,12 +5,13 @@ use anyhow::{Context, Result, anyhow}; use k8s_openapi::{api::core::v1::Secret, apimachinery::pkg::util::intstr::IntOrString}; -use kube::{Api, api::ObjectMeta}; +use kube::{Api, api::ObjectMeta, runtime::wait::await_condition}; use std::{collections::BTreeMap, time::Duration}; +use tokio::time::timeout; use trusted_cluster_operator_lib::virtualmachines::*; use super::{VmBackend, VmConfig, generate_ignition, ssh_exec}; -use crate::{Poller, ensure_command}; +use crate::ensure_command; pub struct KubevirtBackend(pub VmConfig); @@ -141,29 +142,17 @@ impl VmBackend for KubevirtBackend { async fn wait_for_running(&self, timeout_secs: u64) -> Result<()> { let api: Api = Api::namespaced(self.0.client.clone(), &self.0.namespace); - - let poller = Poller::new() - .with_timeout(Duration::from_secs(timeout_secs)) - .with_interval(Duration::from_secs(5)) - .with_error_message(format!( - "VirtualMachine {} did not reach Running phase after {timeout_secs} seconds", - self.0.vm_name - )); - - let check_fn = || { - let api = api.clone(); - async move { - let vm = api.get(&self.0.vm_name).await?; - let status = vm.status.and_then(|p| p.printable_status); - if status.map(|s| s == "Running").unwrap_or(false) { - return Ok(()); - } - let vm_name = &self.0.vm_name; - let err = anyhow!("VirtualMachine {vm_name} is not in Running phase yet"); - Err(err) - } + let machine_running = |m: Option<&VirtualMachine>| { + let status = m.as_ref().and_then(|m| m.status.as_ref()); + let print_status = status.and_then(|s| s.printable_status.as_ref()); + print_status.map(|s| s == "Running").unwrap_or(false) }; - poller.poll_async(check_fn).await + let vm_name = &self.0.vm_name; + let done = await_condition(api, vm_name, machine_running); + let ctx = format!("waiting {timeout_secs} for VirtualMachine {vm_name} to be running"); + let duration = Duration::from_secs(timeout_secs); + timeout(duration, done).await.context(ctx)??; + Ok(()) } async fn ssh_exec(&self, command: &str) -> Result { diff --git a/tests/attestation.rs b/tests/attestation.rs index cb1a2157..c9afcdec 100644 --- a/tests/attestation.rs +++ b/tests/attestation.rs @@ -204,7 +204,7 @@ async fn test_vm_reboot_delete_machine() -> anyhow::Result<()> { let list = machines.list(&Default::default()).await?; let name = list.items[0].metadata.name.as_ref().unwrap(); machines.delete(name, &Default::default()).await?; - wait_for_resource_deleted(&machines, name, scaled_timeout(120), 5).await?; + wait_for_resource_deleted(&machines, name, scaled_timeout(120)).await?; test_ctx.info("Performing reboot, expecting missing resource"); let _reboot_result = att_ctx.backend.ssh_exec("sudo systemctl reboot").await; diff --git a/tests/trusted_execution_cluster.rs b/tests/trusted_execution_cluster.rs index 742ac840..bceb9e16 100644 --- a/tests/trusted_execution_cluster.rs +++ b/tests/trusted_execution_cluster.rs @@ -3,14 +3,18 @@ // // SPDX-License-Identifier: MIT -use anyhow::anyhow; +use anyhow::Context; use compute_pcrs_lib::{Part, Pcr}; use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::{ConfigMap, Secret}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, OwnerReference}; use kube::api::ObjectMeta; +use kube::runtime::wait::await_condition; use kube::{Api, api::DeleteParams}; use std::time::Duration; +use tokio::time::timeout; use trusted_cluster_operator_lib::conditions::NOT_COMMITTED_REASON_PENDING; +use trusted_cluster_operator_lib::endpoints::{REGISTER_SERVER_DEPLOYMENT, TRUSTEE_DEPLOYMENT}; use trusted_cluster_operator_lib::reference_values::ImagePcrs; use trusted_cluster_operator_lib::{ ApprovedImage, AttestationKey, Machine, TrustedExecutionCluster, generate_owner_reference, @@ -23,6 +27,12 @@ const APPROVED_IMAGE_NAME: &str = "coreos"; const TRUSTEE_CONFIG_MAP: &str = "trustee-data"; const RV_JSON_KEY: &str = "reference-values.json"; +fn ak_approved(ak: Option<&AttestationKey>) -> bool { + let is_approved = |c: &Condition| c.type_ == "Approved" && c.status == "True"; + let cs = ak.and_then(|ak| ak.status.as_ref().and_then(|s| s.conditions.as_ref())); + cs.map(|cs| cs.iter().any(is_approved)).unwrap_or(false) +} + named_test!( async fn test_trusted_execution_cluster_uninstall() -> anyhow::Result<()> { let test_ctx = setup!().await?; @@ -82,40 +92,9 @@ named_test!( )); // Wait for the AttestationKey to be approved (operator should match Machine IP and approve it) - let poller = Poller::new() - .with_timeout(scaled_duration(30)) - .with_interval(Duration::from_millis(500)) - .with_error_message("AttestationKey was not approved".to_string()); - - poller - .poll_async(|| { - let ak_api = attestation_keys.clone(); - let ak_name_clone = ak_name.clone(); - async move { - let ak = ak_api.get(&ak_name_clone).await?; - - // Check for Approved condition - let has_approved_condition = ak - .status - .as_ref() - .and_then(|s| s.conditions.as_ref()) - .map(|conditions| { - conditions - .iter() - .any(|c| c.type_ == "Approved" && c.status == "True") - }) - .unwrap_or(false); - - if !has_approved_condition { - return Err(anyhow!( - "AttestationKey does not have Approved condition yet" - )); - } - - Ok(()) - } - }) - .await?; + let done = await_condition(attestation_keys.clone(), &ak_name, ak_approved); + let ctx = format!("waiting for AttestationKey {ak_name} to be approved"); + timeout(scaled_duration(30), done).await.context(ctx)??; test_ctx.info("AttestationKey successfully approved"); @@ -125,26 +104,20 @@ named_test!( api.delete(TEC_NAME, &dp).await?; // Wait until it disappears - wait_for_resource_deleted(&api, TEC_NAME, scaled_timeout(120), 5).await?; + wait_for_resource_deleted(&api, TEC_NAME, scaled_timeout(120)).await?; let deployments_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted( - &deployments_api, - "trustee-deployment", - scaled_timeout(120), - 1, - ) - .await?; - wait_for_resource_deleted(&deployments_api, "register-server", scaled_timeout(120), 1) - .await?; + let timeout = scaled_timeout(120); + wait_for_resource_deleted(&deployments_api, TRUSTEE_DEPLOYMENT, timeout).await?; + wait_for_resource_deleted(&deployments_api, REGISTER_SERVER_DEPLOYMENT, timeout).await?; let images_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted(&images_api, APPROVED_IMAGE_NAME, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&images_api, APPROVED_IMAGE_NAME, scaled_timeout(120)).await?; - wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120), 1).await?; - wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120)).await?; let secrets_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120)).await?; test_ctx.cleanup().await?; @@ -159,30 +132,15 @@ async fn test_image_pcrs_configmap_updates() -> anyhow::Result<()> { let namespace = test_ctx.namespace(); let configmap_api: Api = Api::namespaced(client.clone(), namespace); - - let poller = Poller::new() - .with_timeout(scaled_duration(180)) - .with_interval(Duration::from_secs(5)) - .with_error_message("image-pcrs ConfigMap not populated with data".to_string()); - - poller - .poll_async(|| { - let api = configmap_api.clone(); - async move { - let cm = api.get("image-pcrs").await?; - - if let Some(data) = &cm.data - && let Some(image_pcrs_json) = data.get("image-pcrs.json") - && let Ok(image_pcrs) = serde_json::from_str::(image_pcrs_json) - && !image_pcrs.0.is_empty() - { - return Ok(()); - } - - Err(anyhow!("image-pcrs ConfigMap not yet populated with image-pcrs.json data")) - } - }) - .await?; + let populated = |cm: Option<&ConfigMap>| { + let data = cm.and_then(|cm| cm.data.as_ref()); + let json = data.and_then(|data| data.get("image-pcrs.json")); + let pcrs = json.and_then(|json| serde_json::from_str::(json).ok()); + pcrs.map(|pcrs| !pcrs.0.is_empty()).unwrap_or(false) + }; + let done = await_condition(configmap_api.clone(), "image-pcrs", populated); + let ctx = "waiting for ConfigMap image-pcrs to be populated"; + timeout(scaled_duration(180), done).await.context(ctx)??; let image_pcrs_cm = configmap_api.get("image-pcrs").await?; assert_eq!(image_pcrs_cm.metadata.name.as_deref(), Some("image-pcrs")); @@ -268,23 +226,14 @@ async fn test_image_disallow() -> anyhow::Result<()> { images.delete(APPROVED_IMAGE_NAME, &DeleteParams::default()).await?; let configmap_api: Api = Api::namespaced(client.clone(), namespace); - let poller = Poller::new() - .with_timeout(scaled_duration(180)) - .with_interval(Duration::from_secs(5)) - .with_error_message("Reference value not removed".to_string()); - poller.poll_async(|| { - let api = configmap_api.clone(); - async move { - let cm = api.get(TRUSTEE_CONFIG_MAP).await?; - if let Some(data) = &cm.data - && let Some(reference_values_json) = data.get(RV_JSON_KEY) - && !reference_values_json.contains(EXPECTED_PCR4) - { - return Ok(()); - } - Err(anyhow!("Reference value not yet removed")) - } - }).await?; + let chk_removed = |cm: Option<&ConfigMap>| { + let data = cm.and_then(|cm| cm.data.as_ref()); + let json = data.and_then(|data| data.get(RV_JSON_KEY)); + json.map(|json| !json.contains(EXPECTED_PCR4)).unwrap_or(false) + }; + let rv_removed = await_condition(configmap_api, TRUSTEE_CONFIG_MAP, chk_removed); + let ctx = format!("waiting for ConfigMap {TRUSTEE_CONFIG_MAP} to not contain PCR value"); + timeout(scaled_duration(180), rv_removed).await.context(ctx)??; test_ctx.cleanup().await?; Ok(()) @@ -348,81 +297,27 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { "Created test Machine: {machine_name} with uuid: {machine_uuid}", )); - // Poll for the AttestationKey to be approved, have owner reference, and have a Secret created + // Timeout for the AttestationKey to be approved, have owner reference, and have a Secret created + let approved = await_condition(attestation_keys.clone(), &ak_name, ak_approved); + let ctx = format!("waiting for AttestationKey {ak_name} to be approved"); + timeout(scaled_duration(30), approved).await.context(ctx)??; + let chk_machine_owner = |ak: Option<&AttestationKey>| { + let chk_owner = |owner: &OwnerReference| owner.kind == "Machine" && owner.name == machine_name; + let refs = ak.and_then(|ak| ak.metadata.owner_references.as_ref()); + refs.map(|refs| refs.iter().any(chk_owner)).unwrap_or(false) + }; + let has_machine_owner = await_condition(attestation_keys.clone(), &ak_name, chk_machine_owner); + let ctx = format!("waiting for AttestationKey {ak_name} to be owned by Machine {machine_name}"); + timeout(scaled_duration(30), has_machine_owner).await.context(ctx)??; let secrets_api: Api = Api::namespaced(client.clone(), namespace); - let poller = Poller::new() - .with_timeout(scaled_duration(30)) - .with_interval(Duration::from_millis(500)) - .with_error_message("AttestationKey was not approved with owner reference and secret".to_string()); - - poller - .poll_async(|| { - let ak_api = attestation_keys.clone(); - let secrets = secrets_api.clone(); - let ak_name_clone = ak_name.clone(); - let machine_name_clone = machine_name.clone(); - async move { - let ak = ak_api.get(&ak_name_clone).await?; - - // Check for Approved condition - let has_approved_condition = ak - .status - .as_ref() - .and_then(|s| s.conditions.as_ref()) - .map(|conditions| { - conditions - .iter() - .any(|c| c.type_ == "Approved" && c.status == "True") - }) - .unwrap_or(false); - - if !has_approved_condition { - return Err(anyhow!( - "AttestationKey does not have Approved condition yet" - )); - } - - // Check for owner reference to the Machine - let has_machine_owner_ref = ak - .metadata - .owner_references - .as_ref() - .map(|owner_refs| { - owner_refs.iter().any(|owner_ref| { - owner_ref.kind == "Machine" && owner_ref.name == machine_name_clone - }) - }) - .unwrap_or(false); - - if !has_machine_owner_ref { - return Err(anyhow!( - "AttestationKey does not have owner reference to Machine yet" - )); - } - - // Check that a Secret with the same name exists and has the AttestationKey as owner - let secret = secrets.get(&ak_name_clone).await?; - let has_ak_owner_ref = secret - .metadata - .owner_references - .as_ref() - .map(|owner_refs| { - owner_refs.iter().any(|owner_ref| { - owner_ref.kind == "AttestationKey" && owner_ref.name == ak_name_clone - }) - }) - .unwrap_or(false); - - if !has_ak_owner_ref { - return Err(anyhow!( - "Secret does not have owner reference to AttestationKey yet" - )); - } - - Ok(()) - } - }) - .await?; + let chk_ak_owner = |secret: Option<&Secret>| { + let chk_owner = |owner: &OwnerReference| owner.kind == "AttestationKey" && owner.name == ak_name; + let refs = secret.and_then(|s| s.metadata.owner_references.as_ref()); + refs.map(|refs| refs.iter().any(chk_owner)).unwrap_or(false) + }; + let has_ak_owner = await_condition(secrets_api.clone(), &ak_name, chk_ak_owner); + let ctx = format!("waiting for Secret {ak_name} to be owned by AttestationKey {ak_name}"); + timeout(scaled_duration(30), has_ak_owner).await.context(ctx)??; test_ctx.info(format!( "AttestationKey successfully approved with owner reference to Machine: {machine_name} and Secret created" @@ -433,11 +328,11 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { machines.delete(&machine_name, &dp).await?; test_ctx.info(format!("Deleted Machine: {machine_name}")); - wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120)).await?; test_ctx.info("Machine successfully deleted"); - wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120)).await?; test_ctx.info("AttestationKey successfully deleted"); - wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120)).await?; test_ctx.info("Secret successfully deleted"); test_ctx.cleanup().await?; @@ -465,22 +360,14 @@ async fn test_nonexistent_approved_image() -> anyhow::Result<()> { status: None, }).await?; - let poller = Poller::new() - .with_timeout(scaled_duration(30)) - .with_interval(Duration::from_millis(500)) - .with_error_message("ApprovedImage not created".to_string()); - poller.poll_async(|| { - let api = images.clone(); - async move { - let img = api.get("coreos1").await?; - if img.status.as_ref().and_then(|s| s.conditions.as_ref()).map(|conditions| { - conditions.iter().any(|c| c.reason == NOT_COMMITTED_REASON_PENDING) - }).unwrap_or(false) { - return Ok(()); - } - Err(anyhow::anyhow!("ApprovedImage not yet committed")) - } - }).await?; + let is_pending = |img: Option<&ApprovedImage>| { + let pending = |c: &Condition| c.reason == NOT_COMMITTED_REASON_PENDING; + let cs = img.and_then(|img| img.status.as_ref()).and_then(|s| s.conditions.as_ref()); + cs.map(|cs| cs.iter().any(pending)).unwrap_or(false) + }; + let done = await_condition(images, "coreos1", is_pending); + let ctx = "waiting for ApprovedImage coreos1 to be PodPending"; + timeout(scaled_duration(30), done).await.context(ctx)??; test_ctx.cleanup().await?; Ok(()) @@ -502,28 +389,8 @@ async fn test_approved_image_readoption() -> anyhow::Result<()> { test_ctx.info(format!("Deleting TrustedExecuctionCluster {TEC_NAME}")); clusters.delete(TEC_NAME, &Default::default()).await?; - let removal_poller = Poller::new() - .with_timeout(Duration::from_secs(60)) - .with_interval(Duration::from_secs(5)) - .with_error_message(format!( - "ConfigMap {TRUSTEE_CONFIG_MAP} or ApprovedImage {APPROVED_IMAGE_NAME} not removed" - )); - removal_poller - .poll_async(|| { - let configmaps = configmaps.clone(); - let images = images.clone(); - async move { - if configmaps.get(TRUSTEE_CONFIG_MAP).await.is_ok() { - return Err(anyhow!("ConfigMap {TRUSTEE_CONFIG_MAP} not yet removed")); - } - if images.get(APPROVED_IMAGE_NAME).await.is_ok() { - let err = anyhow!("ApprovedImage {APPROVED_IMAGE_NAME} not yet removed"); - return Err(err); - } - Ok(()) - } - }) - .await?; + wait_for_resource_deleted(&configmaps, TRUSTEE_CONFIG_MAP, scaled_timeout(60)).await?; + wait_for_resource_deleted(&images, APPROVED_IMAGE_NAME, scaled_timeout(60)).await?; test_ctx.info(format!("Configmap {TRUSTEE_CONFIG_MAP} was removed")); let image = ApprovedImage { @@ -548,25 +415,14 @@ async fn test_approved_image_readoption() -> anyhow::Result<()> { // Ensure adoption works even when cluster creation was delayed tokio::time::sleep(Duration::from_secs(5)).await; clusters.create(&Default::default(), &cluster).await?; - let regeneration_poller = Poller::new() - .with_timeout(Duration::from_secs(180)) - .with_interval(Duration::from_secs(5)) - .with_error_message("Reference value not regenerated".to_string()); - regeneration_poller - .poll_async(|| { - let configmaps = configmaps.clone(); - async move { - let configmap = configmaps.get(TRUSTEE_CONFIG_MAP).await?; - if let Some(data) = &configmap.data - && let Some(json) = data.get(RV_JSON_KEY) - && json.contains(EXPECTED_PCR4) - { - return Ok(()); - } - Err(anyhow!("Reference value not yet regenerated")) - } - }) - .await?; + let chk_added = |cm: Option<&ConfigMap>| { + let data = cm.and_then(|cm| cm.data.as_ref()); + let json = data.and_then(|data| data.get(RV_JSON_KEY)); + json.map(|json| json.contains(EXPECTED_PCR4)).unwrap_or(false) + }; + let rv_added = await_condition(configmaps, TRUSTEE_CONFIG_MAP, chk_added); + let ctx = format!("waiting for ConfigMap {TRUSTEE_CONFIG_MAP} to contain PCR value"); + timeout(scaled_duration(180), rv_added).await.context(ctx)??; test_ctx.info("Reference values regenerated"); test_ctx.cleanup().await?; From c4bbd80611c69f701a96c6636acb0a613321d5d3 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 22 Jun 2026 15:11:38 +0200 Subject: [PATCH 06/14] tests: Move kubectl() out of K8sPlatform does not need fields to come or parameters Signed-off-by: Jakob Naucke --- test_utils/src/lib.rs | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index f5ac36a2..72f1c3a0 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -118,8 +118,7 @@ macro_rules! kube_apply { args.extend_from_slice(&["--server-side", "--force-conflicts"]) } )? - let mut cmd = get_k8s_platform().kubectl(); - let apply_output = cmd.args(args).output().await?; + let apply_output = kubectl().args(args).output().await?; if !apply_output.status.success() { let stderr = String::from_utf8_lossy(&apply_output.stderr); return Err(anyhow!("{} failed: {}", $log, stderr)); @@ -159,6 +158,13 @@ pub fn ensure_command(name: &str) -> Result<()> { result.map_err(|_| anyhow!("Command {name} not found. Please install {name} first.")) } +fn kubectl() -> Command { + match env::var(PLATFORM_ENV).as_deref().unwrap_or("kind") { + "openshift" => Command::new("oc"), + _ => Command::new("kubectl"), + } +} + #[async_trait::async_trait] #[auto_impl::auto_impl(Box)] trait K8sPlatform: Send + Sync { @@ -178,7 +184,6 @@ trait K8sPlatform: Send + Sync { service: &str, port: Option, ) -> Result; - fn kubectl(&self) -> Command; } struct Kind { @@ -261,10 +266,6 @@ impl K8sPlatform for Kind { None => url, }) } - - fn kubectl(&self) -> Command { - Command::new("kubectl") - } } #[async_trait::async_trait] @@ -314,10 +315,6 @@ impl K8sPlatform for OpenShift { let domain = ingress.spec.domain.unwrap(); Ok(format!("{service}-{namespace}.{domain}")) } - - fn kubectl(&self) -> Command { - Command::new("oc") - } } #[async_trait::async_trait] @@ -340,10 +337,6 @@ impl K8sPlatform for OtherK8s { ) -> Result { Err(anyhow!(SET_CLUSTER_ERR)) } - - fn kubectl(&self) -> Command { - Command::new("kubectl") - } } pub async fn get_cluster_url( @@ -727,8 +720,7 @@ impl TestContext { self.set_certificates().await?; let tec = "trustedexecutionclusters.trusted-execution-clusters.io"; let args = ["get", "crd", tec]; - let mut cmd = get_k8s_platform().kubectl(); - let crd_check_output = cmd.args(args).output().await?; + let crd_check_output = kubectl().args(args).output().await?; if crd_check_output.status.success() { test_info!( From 07e2b6a3eed18d7405fdef4d06543a42c695a07a Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 22 Jun 2026 15:12:33 +0200 Subject: [PATCH 07/14] tests: Make client & ns part of K8sPlatform to avoid too many parameters Signed-off-by: Jakob Naucke --- test_utils/src/lib.rs | 123 ++++++++++++++----------------------- test_utils/src/virt/mod.rs | 2 +- 2 files changed, 46 insertions(+), 79 deletions(-) diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 72f1c3a0..33da9307 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -169,34 +169,36 @@ fn kubectl() -> Command { #[auto_impl::auto_impl(Box)] trait K8sPlatform: Send + Sync { fn add_scc(&self, kustomization: &mut serde_yaml::Value); - async fn expose( - &self, - client: &Client, - namespace: &str, - service: &str, - test_name: &str, - port: i32, - ) -> Result<()>; - async fn get_cluster_url( - &self, - client: &Client, - namespace: &str, - service: &str, - port: Option, - ) -> Result; + async fn expose(&self, service: &str, test_name: &str, port: i32) -> Result<()>; + async fn get_cluster_url(&self, service: &str, port: Option) -> Result; } struct Kind { public: bool, + client: Client, + namespace: String, +} +struct OpenShift { + client: Client, + namespace: String, } -struct OpenShift {} struct OtherK8s {} -fn get_k8s_platform() -> Box { +fn get_k8s_platform(client: &Client, namespace: &str) -> Box { + let client = client.clone(); + let namespace = namespace.to_string(); match env::var(PLATFORM_ENV).as_deref().unwrap_or("kind") { - "kind" => Box::new(Kind { public: false }), - "kind_public" => Box::new(Kind { public: true }), - "openshift" => Box::new(OpenShift {}), + "kind" => Box::new(Kind { + public: false, + client, + namespace, + }), + "kind_public" => Box::new(Kind { + public: true, + client, + namespace, + }), + "openshift" => Box::new(OpenShift { client, namespace }), _ => Box::new(OtherK8s {}), } } @@ -204,14 +206,7 @@ fn get_k8s_platform() -> Box { #[async_trait::async_trait] impl K8sPlatform for Kind { fn add_scc(&self, _: &mut serde_yaml::Value) {} - async fn expose( - &self, - client: &Client, - namespace: &str, - service: &str, - _: &str, - _: i32, - ) -> Result<()> { + async fn expose(&self, service: &str, _: &str, _: i32) -> Result<()> { if !self.public { return Ok(()); } @@ -235,7 +230,7 @@ impl K8sPlatform for Kind { port, ..Default::default() }; - let services: Api = Api::namespaced(client.clone(), namespace); + let services: Api = Api::namespaced(self.client.clone(), &self.namespace); let service = Service { metadata: ObjectMeta { name: Some(format!("{service}-forward")), @@ -253,14 +248,8 @@ impl K8sPlatform for Kind { Ok(()) } - async fn get_cluster_url( - &self, - _: &Client, - namespace: &str, - service: &str, - port: Option, - ) -> Result { - let url = format!("{service}.{namespace}.svc.cluster.local"); + async fn get_cluster_url(&self, service: &str, port: Option) -> Result { + let url = format!("{service}.{}.svc.cluster.local", self.namespace); Ok(match port { Some(port) => format!("{url}:{port}"), None => url, @@ -277,19 +266,12 @@ impl K8sPlatform for OpenShift { resource_seq.push(serde_yaml::Value::String("scc.yaml".to_string())) } - async fn expose( - &self, - _: &Client, - namespace: &str, - service: &str, - _: &str, - port: i32, - ) -> Result<()> { + async fn expose(&self, service: &str, _: &str, port: i32) -> Result<()> { ensure_command("oc")?; - let mut args = vec!["create", "route", "passthrough", service, "-n", namespace]; + let mut args = vec!["create", "route", "passthrough", service]; let svc = format!("--service={service}"); let port = format!("--port={port}"); - args.extend_from_slice(&[&svc, &port]); + args.extend_from_slice(&["-n", &self.namespace, &svc, &port]); let output = Command::new("oc").args(args).output().await?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); @@ -298,22 +280,16 @@ impl K8sPlatform for OpenShift { Ok(()) } - async fn get_cluster_url( - &self, - client: &Client, - namespace: &str, - service: &str, - _: Option, - ) -> Result { - let routes: Api = Api::namespaced(client.clone(), namespace); + async fn get_cluster_url(&self, service: &str, _: Option) -> Result { + let routes: Api = Api::namespaced(self.client.clone(), &self.namespace); if let Ok(route) = routes.get(service).await { return Ok(route.spec.host.expect("route existed, but had no host")); } // Fallback when route does not exist yet - let ingresses: Api = Api::all(client.clone()); + let ingresses: Api = Api::all(self.client.clone()); let ingress = ingresses.get("cluster").await?; let domain = ingress.spec.domain.unwrap(); - Ok(format!("{service}-{namespace}.{domain}")) + Ok(format!("{service}-{}.{domain}", self.namespace)) } } @@ -321,26 +297,20 @@ impl K8sPlatform for OpenShift { impl K8sPlatform for OtherK8s { fn add_scc(&self, _: &mut serde_yaml::Value) {} - async fn expose(&self, _: &Client, _: &str, _: &str, test_name: &str, _: i32) -> Result<()> { + async fn expose(&self, _: &str, test_name: &str, _: i32) -> Result<()> { let warn = "You appear to be on an environment that is not Kind or OpenShift. \ Ensure operator services are reachable"; test_warn!(test_name, "{warn}"); Ok(()) } - async fn get_cluster_url( - &self, - _: &Client, - _: &str, - _: &str, - _: Option, - ) -> Result { + async fn get_cluster_url(&self, _: &str, _: Option) -> Result { Err(anyhow!(SET_CLUSTER_ERR)) } } pub async fn get_cluster_url( - client: Client, + client: &Client, namespace: &str, service: &str, port: Option, @@ -352,8 +322,8 @@ pub async fn get_cluster_url( None => full_url, }); } - get_k8s_platform() - .get_cluster_url(&client, namespace, service, port) + get_k8s_platform(client, namespace) + .get_cluster_url(service, port) .await } @@ -545,7 +515,7 @@ impl TestContext { issuer_name: &str, ) -> Result<()> { let ns = &self.test_namespace; - let domain = get_cluster_url(self.client.clone(), ns, service_name, None).await?; + let domain = get_cluster_url(&self.client, ns, service_name, None).await?; let certs: Api = Api::namespaced(self.client.clone(), ns); let cert = Certificate { metadata: ObjectMeta { @@ -775,7 +745,7 @@ impl TestContext { std::fs::write(&le_rb_dst, le_rb_content)?; test_info!(&self.test_name, "Preparing RBAC kustomization"); - let platform = get_k8s_platform(); + let platform = get_k8s_platform(&self.client, &self.test_namespace); let kustomization_src = workspace_root.join("config/rbac/kustomization.yaml.in"); let kustomization_content = std::fs::read_to_string(&kustomization_src)?; let mut kustom_value: serde_yaml::Value = serde_yaml::from_str(&kustomization_content)?; @@ -822,7 +792,7 @@ impl TestContext { async fn apply_cr_manifests(&self, manifests_path: &Path) -> Result<()> { let ns = &self.test_namespace; let trustee_addr = - get_cluster_url(self.client.clone(), ns, TRUSTEE_SERVICE, Some(TRUSTEE_PORT)).await?; + get_cluster_url(&self.client, ns, TRUSTEE_SERVICE, Some(TRUSTEE_PORT)).await?; let cr_manifest_path = manifests_path.join("trusted_execution_cluster_cr.yaml"); let cr_content = std::fs::read_to_string(&cr_manifest_path)?; @@ -848,10 +818,9 @@ impl TestContext { ); if get_virt_provider()? == VirtProvider::Kubevirt { - let platform = get_k8s_platform(); - let svc = ATTESTATION_KEY_REGISTER_SERVICE; + let platform = get_k8s_platform(&self.client, &self.test_namespace); let port = ATTESTATION_KEY_REGISTER_PORT; - let address = platform.get_cluster_url(&self.client, ns, svc, Some(port)); + let address = platform.get_cluster_url(ATTESTATION_KEY_REGISTER_SERVICE, Some(port)); spec_map.insert( serde_yaml::Value::String("publicAttestationKeyRegisterAddr".to_string()), serde_yaml::Value::String(address.await?), @@ -902,16 +871,14 @@ impl TestContext { timeout(scaled_duration(300), done).await.context(ctx)??; } - let platform = get_k8s_platform(); + let platform = get_k8s_platform(&self.client, &self.test_namespace); let ak_port = ATTESTATION_KEY_REGISTER_PORT; for (svc, port) in [ (TRUSTEE_SERVICE, TRUSTEE_PORT), (ATTESTATION_KEY_REGISTER_SERVICE, ak_port), (REGISTER_SERVER_SERVICE, REGISTER_SERVER_PORT), ] { - platform - .expose(&self.client, ns, svc, &self.test_name, port) - .await?; + platform.expose(svc, &self.test_name, port).await?; } test_info!( diff --git a/test_utils/src/virt/mod.rs b/test_utils/src/virt/mod.rs index cf5d33cc..38309158 100644 --- a/test_utils/src/virt/mod.rs +++ b/test_utils/src/virt/mod.rs @@ -79,7 +79,7 @@ pub async fn generate_ignition(config: &VmConfig) -> Result { let client = config.client.clone(); let ns = &config.namespace; let port = Some(REGISTER_SERVER_PORT); - let register_server_url = get_cluster_url(client, ns, REGISTER_SERVER_SERVICE, port).await?; + let register_server_url = get_cluster_url(&client, ns, REGISTER_SERVER_SERVICE, port).await?; let root_pem_encoded = utf8_percent_encode(&config.ca_pem, NON_ALPHANUMERIC); let ignition = Ignition { version: "3.6.0-experimental".to_string(), From efaf8b74195c2faa92cc68094e1942d4ca97adfe Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Wed, 3 Jun 2026 18:49:41 +0200 Subject: [PATCH 08/14] tests: Expose OpenShift services with LB services Patch services to LoadBalancer ones to expose on OpenShift. Patch certificates once address is known. Depending on platform, ingress may be an IP address and thus entirely unpredictable. Therefore, patch TrustedExecutionCluster with Trustee address after service creation. This also reverts commit f2ad17b6fe449d528070b92f7985c621171c0bf4. Signed-off-by: Jakob Naucke --- Makefile | 2 - go.mod | 2 - go.sum | 33 -------- lib/src/kopium.rs | 2 - lib/src/lib.rs | 2 - test_utils/src/lib.rs | 191 ++++++++++++++++++++++++++++++------------ tools.go | 2 - 7 files changed, 139 insertions(+), 95 deletions(-) diff --git a/Makefile b/Makefile index ef0966ee..1cb3154b 100644 --- a/Makefile +++ b/Makefile @@ -55,8 +55,6 @@ RBAC_YAML_PATH = config/rbac API_PATH = api/v1alpha1 generate: $(CONTROLLER_GEN) $(call controller-gen,./...,*) - $(call controller-gen,github.com/openshift/api/route/v1,*) - $(call controller-gen,github.com/openshift/api/config/v1,*_ingresses.yaml) $(call controller-gen,github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1,*) RS_LIB_PATH = lib/src diff --git a/go.mod b/go.mod index 0d78775f..aa76a40d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ go 1.25.0 require ( github.com/cert-manager/cert-manager v1.20.2 github.com/mikefarah/yq/v4 v4.53.3 - github.com/openshift/api v0.0.0-20260213204242-d34f11c515b3 github.com/projectcalico/api v0.0.0-20251022175904-f2ab03771208 k8s.io/api v0.35.3 k8s.io/apimachinery v0.35.3 @@ -40,7 +39,6 @@ require ( github.com/gobuffalo/flect v1.0.3 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/goccy/go-yaml v1.19.2 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect diff --git a/go.sum b/go.sum index 38b9ced0..0a0fc138 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,6 @@ github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= @@ -104,8 +102,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -138,8 +134,6 @@ github.com/onsi/ginkgo/v2 v2.28.0 h1:Rrf+lVLmtlBIKv6KrIGJCjyY8N36vDVcutbGJkyqjJc github.com/onsi/ginkgo/v2 v2.28.0/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= -github.com/openshift/api v0.0.0-20260213204242-d34f11c515b3 h1:SZ8+jxtkMvpb4HDTjSAbaOyhFsw5PiWhjBog+XLY7jc= -github.com/openshift/api v0.0.0-20260213204242-d34f11c515b3/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= @@ -183,8 +177,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA= github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= github.com/zclconf/go-cty v1.18.1 h1:yEGE8M4iIZlyKQURZNb2SnEyZlZHUcBCnx6KF81KuwM= @@ -215,56 +207,31 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U= go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w= diff --git a/lib/src/kopium.rs b/lib/src/kopium.rs index 91ecd4d5..328edfd0 100644 --- a/lib/src/kopium.rs +++ b/lib/src/kopium.rs @@ -7,8 +7,6 @@ pub mod attestationkeys; pub mod certificaterequests; pub mod certificates; pub mod clusterissuers; -pub mod ingresses; pub mod issuers; pub mod machines; -pub mod routes; pub mod trustedexecutionclusters; diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 8861b6ce..03adb539 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -13,9 +13,7 @@ mod vendor_kopium; use k8s_openapi::jiff::Timestamp; pub use kopium::approvedimages::*; pub use kopium::attestationkeys::*; -pub use kopium::ingresses as openshift_ingresses; pub use kopium::machines::*; -pub use kopium::routes; pub use kopium::trustedexecutionclusters::*; pub use kopium::certificaterequests; diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 33da9307..1d60f5f1 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -6,23 +6,25 @@ use anyhow::{Context, Result, anyhow}; use fs_extra::dir; use k8s_openapi::api::apps::v1::{Deployment, DeploymentCondition, DeploymentStatus}; -use k8s_openapi::api::core::v1::{ConfigMap, Namespace, Secret, Service, ServicePort, ServiceSpec}; -use kube::api::{DeleteParams, ObjectMeta}; +use k8s_openapi::api::core::v1::{ + ConfigMap, LoadBalancerStatus, Namespace, Secret, Service, ServicePort, ServiceSpec, + ServiceStatus, +}; +use kube::api::{DeleteParams, ObjectMeta, Patch}; use kube::runtime::wait::await_condition; use kube::{Api, Client}; +use serde_json::json; use std::path::{Path, PathBuf}; use std::{collections::BTreeMap, env, sync::Once, time::Duration}; use tokio::process::Command; use tokio::time::timeout; use trusted_cluster_operator_lib::certificates::{ - Certificate, CertificateIssuerRef, CertificateSpec, + Certificate, CertificateIssuerRef, CertificateSpec, CertificateStatus, }; use trusted_cluster_operator_lib::issuers::{Issuer, IssuerCa, IssuerSpec}; use trusted_cluster_operator_lib::Machine; use trusted_cluster_operator_lib::TrustedExecutionCluster; -use trusted_cluster_operator_lib::openshift_ingresses::Ingress; -use trusted_cluster_operator_lib::routes::Route; use trusted_cluster_operator_lib::{endpoints::*, images::*}; pub mod timer; @@ -50,6 +52,9 @@ const ROOT_SECRET: &str = "root-secret"; const REG_SECRET: &str = "reg-srv-secret"; const TRUSTEE_SECRET: &str = "trustee-secret"; const ATT_REG_SECRET: &str = "att-reg-secret"; +const REG_CERT: &str = "reg-srv-cert"; +const TRUSTEE_CERT: &str = "trustee-cert"; +const ATT_REG_CERT: &str = "att-reg-cert"; pub fn compare_pcrs(actual: &[Pcr], expected: &[Pcr]) -> bool { if actual.len() != expected.len() { @@ -169,7 +174,13 @@ fn kubectl() -> Command { #[auto_impl::auto_impl(Box)] trait K8sPlatform: Send + Sync { fn add_scc(&self, kustomization: &mut serde_yaml::Value); - async fn expose(&self, service: &str, test_name: &str, port: i32) -> Result<()>; + async fn expose( + &self, + service: &str, + deployment: &str, + cert_name: &str, + test_name: &str, + ) -> Result<()>; async fn get_cluster_url(&self, service: &str, port: Option) -> Result; } @@ -206,7 +217,7 @@ fn get_k8s_platform(client: &Client, namespace: &str) -> Box { #[async_trait::async_trait] impl K8sPlatform for Kind { fn add_scc(&self, _: &mut serde_yaml::Value) {} - async fn expose(&self, service: &str, _: &str, _: i32) -> Result<()> { + async fn expose(&self, service: &str, _: &str, _: &str, _: &str) -> Result<()> { if !self.public { return Ok(()); } @@ -257,6 +268,27 @@ impl K8sPlatform for Kind { } } +enum OpenShiftHost { + Ip(String), + Hostname(String), + None, +} + +impl OpenShift { + async fn get_url(&self, service: &str) -> OpenShiftHost { + let services: Api = Api::namespaced(self.client.clone(), &self.namespace); + let Ok(svc) = services.get(service).await else { + return OpenShiftHost::None; + }; + let ingress = &svc.status.unwrap().load_balancer.unwrap().ingress.unwrap()[0]; + match (&ingress.hostname, &ingress.ip) { + (Some(hostname), _) => OpenShiftHost::Hostname(hostname.clone()), + (_, Some(ip)) => OpenShiftHost::Ip(ip.clone()), + (None, None) => OpenShiftHost::None, + } + } +} + #[async_trait::async_trait] impl K8sPlatform for OpenShift { fn add_scc(&self, kustomization: &mut serde_yaml::Value) { @@ -266,30 +298,81 @@ impl K8sPlatform for OpenShift { resource_seq.push(serde_yaml::Value::String("scc.yaml".to_string())) } - async fn expose(&self, service: &str, _: &str, port: i32) -> Result<()> { - ensure_command("oc")?; - let mut args = vec!["create", "route", "passthrough", service]; - let svc = format!("--service={service}"); - let port = format!("--port={port}"); - args.extend_from_slice(&["-n", &self.namespace, &svc, &port]); - let output = Command::new("oc").args(args).output().await?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(anyhow!("oc command failed: {stderr}")); - } + async fn expose( + &self, + service: &str, + deployment: &str, + cert_name: &str, + _: &str, + ) -> Result<()> { + let services: Api = Api::namespaced(self.client.clone(), &self.namespace); + let pp = Default::default(); + let json = json!({ + "spec": { + "type": "LoadBalancer" + } + }); + services.patch(service, &pp, &Patch::Merge(&json)).await?; + let has_ingress = |svc: Option<&Service>| { + let chk_lb = |bal: &LoadBalancerStatus| bal.ingress.is_some(); + let chk_st = |st: &ServiceStatus| st.load_balancer.as_ref().map(chk_lb); + let chk_svc = |svc: &Service| svc.status.as_ref().and_then(chk_st); + svc.and_then(chk_svc).unwrap_or(false) + }; + let ingress_ready = await_condition(services, service, has_ingress); + let ctx = format!("waiting for ingress on {service} to be ready"); + let duration = scaled_duration(60); + timeout(duration, ingress_ready).await.context(ctx)??; + + let certs: Api = Api::namespaced(self.client.clone(), &self.namespace); + let cert = certs.get(cert_name).await?; + let old_revision = cert.status.and_then(|st| st.revision).unwrap_or(0); + let cert_patch = match self.get_url(service).await { + OpenShiftHost::Ip(ip) => json!({ + "spec": { + "ipAddresses": [ip], + "dnsNames": [], + } + }), + OpenShiftHost::Hostname(name) => json!({ + "spec": { + "dnsNames": [name], + "ipAddresses": [] + } + }), + OpenShiftHost::None => { + return Err(anyhow!("expected service {service}")); + } + }; + let cert_merge = Patch::Merge(cert_patch); + certs.patch(cert_name, &pp, &cert_merge).await?; + + let cert_reissued = |cert: Option<&Certificate>| { + let chk = |st: &CertificateStatus| st.revision.map(|r| r > old_revision); + cert.and_then(|c| c.status.as_ref().and_then(chk)) + .unwrap_or(false) + }; + let cert_done = await_condition(certs, cert_name, cert_reissued); + let ctx = format!("waiting for cert {cert_name} to have a rev newer than {old_revision}"); + timeout(duration, cert_done).await.context(ctx)??; + + let deployments: Api = Api::namespaced(self.client.clone(), &self.namespace); + deployments.restart(deployment).await?; + Ok(()) } - async fn get_cluster_url(&self, service: &str, _: Option) -> Result { - let routes: Api = Api::namespaced(self.client.clone(), &self.namespace); - if let Ok(route) = routes.get(service).await { - return Ok(route.spec.host.expect("route existed, but had no host")); - } - // Fallback when route does not exist yet - let ingresses: Api = Api::all(self.client.clone()); - let ingress = ingresses.get("cluster").await?; - let domain = ingress.spec.domain.unwrap(); - Ok(format!("{service}-{}.{domain}", self.namespace)) + async fn get_cluster_url(&self, service: &str, port: Option) -> Result { + let append_port = |e| match port { + Some(p) => format!("{e}:{p}"), + None => e, + }; + Ok(match self.get_url(service).await { + OpenShiftHost::Ip(ip) => append_port(ip), + OpenShiftHost::Hostname(name) => append_port(name), + // Service did not exist yet, put empty name in cert and patch upon expose + OpenShiftHost::None => String::new(), + }) } } @@ -297,7 +380,7 @@ impl K8sPlatform for OpenShift { impl K8sPlatform for OtherK8s { fn add_scc(&self, _: &mut serde_yaml::Value) {} - async fn expose(&self, _: &str, test_name: &str, _: i32) -> Result<()> { + async fn expose(&self, _: &str, _: &str, _: &str, test_name: &str) -> Result<()> { let warn = "You appear to be on an environment that is not Kind or OpenShift. \ Ensure operator services are reachable"; test_warn!(test_name, "{warn}"); @@ -590,12 +673,12 @@ impl TestContext { issuers.create(&Default::default(), &issuer).await?; let svc = REGISTER_SERVER_SERVICE; - self.create_certificate(svc, "reg-srv-cert", REG_SECRET, issuer_name) + self.create_certificate(svc, REG_CERT, REG_SECRET, issuer_name) .await?; - self.create_certificate(TRUSTEE_SERVICE, "trustee-cert", TRUSTEE_SECRET, issuer_name) + self.create_certificate(TRUSTEE_SERVICE, TRUSTEE_CERT, TRUSTEE_SECRET, issuer_name) .await?; let svc = ATTESTATION_KEY_REGISTER_SERVICE; - self.create_certificate(svc, "att-reg-cert", ATT_REG_SECRET, issuer_name) + self.create_certificate(svc, ATT_REG_CERT, ATT_REG_SECRET, issuer_name) .await?; let secrets: Api = Api::namespaced(self.client.clone(), &self.test_namespace); @@ -791,19 +874,12 @@ impl TestContext { async fn apply_cr_manifests(&self, manifests_path: &Path) -> Result<()> { let ns = &self.test_namespace; - let trustee_addr = - get_cluster_url(&self.client, ns, TRUSTEE_SERVICE, Some(TRUSTEE_PORT)).await?; let cr_manifest_path = manifests_path.join("trusted_execution_cluster_cr.yaml"); let cr_content = std::fs::read_to_string(&cr_manifest_path)?; let mut cr_value: serde_yaml::Value = serde_yaml::from_str(&cr_content)?; let spec_map = cr_value.get_mut("spec").unwrap().as_mapping_mut().unwrap(); - spec_map.insert( - serde_yaml::Value::String("publicTrusteeAddr".to_string()), - serde_yaml::Value::String(trustee_addr.clone()), - ); - spec_map.insert( serde_yaml::Value::String("trusteeSecret".to_string()), serde_yaml::Value::String(TRUSTEE_SECRET.to_string()), @@ -830,11 +906,6 @@ impl TestContext { let updated_content = serde_yaml::to_string(&cr_value)?; std::fs::write(&cr_manifest_path, updated_content)?; - test_info!( - &self.test_name, - "Updated CR manifest with publicTrusteeAddr: {trustee_addr}", - ); - let cr_manifest_str = cr_manifest_path.to_str().unwrap(); kube_apply!(cr_manifest_str, &self.test_name, "Applying CR manifest"); @@ -872,14 +943,30 @@ impl TestContext { } let platform = get_k8s_platform(&self.client, &self.test_namespace); - let ak_port = ATTESTATION_KEY_REGISTER_PORT; - for (svc, port) in [ - (TRUSTEE_SERVICE, TRUSTEE_PORT), - (ATTESTATION_KEY_REGISTER_SERVICE, ak_port), - (REGISTER_SERVER_SERVICE, REGISTER_SERVER_PORT), - ] { - platform.expose(svc, &self.test_name, port).await?; - } + let svc = REGISTER_SERVER_SERVICE; + let depl = REGISTER_SERVER_DEPLOYMENT; + let test_name = &self.test_name; + platform.expose(svc, depl, REG_CERT, test_name).await?; + let svc = TRUSTEE_SERVICE; + let depl = TRUSTEE_DEPLOYMENT; + platform.expose(svc, depl, TRUSTEE_CERT, test_name).await?; + let svc = ATTESTATION_KEY_REGISTER_SERVICE; + let depl = ATTESTATION_KEY_REGISTER_DEPLOYMENT; + platform.expose(svc, depl, ATT_REG_CERT, test_name).await?; + + let tecs: Api = Api::namespaced(self.client.clone(), ns); + let trustee_addr = + get_cluster_url(&self.client, ns, TRUSTEE_SERVICE, Some(TRUSTEE_PORT)).await?; + let json = json!({ + "spec": { + "publicTrusteeAddr": trustee_addr + } + }); + let patch = Patch::Merge(&json); + tecs.patch("trusted-execution-cluster", &Default::default(), &patch) + .await?; + let info = format!("Updated TEC resource with publicTrusteeAddr: {trustee_addr}"); + test_info!(&self.test_name, "{info}"); test_info!( &self.test_name, diff --git a/tools.go b/tools.go index db5c5401..79e22d62 100644 --- a/tools.go +++ b/tools.go @@ -13,8 +13,6 @@ package tools import ( _ "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" _ "github.com/mikefarah/yq/v4" - _ "github.com/openshift/api/config/v1" - _ "github.com/openshift/api/route/v1" _ "github.com/projectcalico/api/pkg/lib/numorstring" _ "sigs.k8s.io/controller-tools/cmd/controller-gen" _ "sigs.k8s.io/kind" From 9b1fd8d0652cb2dae0ba3f9addbb5bd9ed13a5ef Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 15 Jun 2026 17:05:54 +0200 Subject: [PATCH 09/14] rbac: Fix finalizers for OpenShift following #246 Signed-off-by: Jakob Naucke --- api/v1alpha1/crds.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/v1alpha1/crds.go b/api/v1alpha1/crds.go index 42b8025b..066ffe29 100644 --- a/api/v1alpha1/crds.go +++ b/api/v1alpha1/crds.go @@ -30,8 +30,7 @@ var ( // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=trustedexecutionclusters;machines;approvedimages;attestationkeys,verbs=create;delete;get;list;patch;update;watch -// +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=trustedexecutionclusters/finalizers,verbs=update -// +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=machines/finalizers,verbs=update +// +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=trustedexecutionclusters/finalizers;machines/finalizers;attestationkeys/finalizers;approvedimages/finalizers,verbs=update // +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=trustedexecutionclusters/status;machines/status;approvedimages/status;attestationkeys/status,verbs=get;patch;update // TrustedExecutionClusterSpec defines the desired state of TrustedExecutionCluster From b2c559ac7c649413bc1ca83cf691d4fc9f7b2929 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 16 Jun 2026 17:16:04 +0200 Subject: [PATCH 10/14] make: ?= OPERATOR_IMAGE which can also be set for `make integration-tests`. Signed-off-by: Jakob Naucke --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1cb3154b..faeeea4a 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ KOPIUM ?= $(LOCALBIN)/kopium-$(KOPIUM_VERSION) REGISTRY ?= quay.io/trusted-execution-clusters TAG ?= latest PUSH_FLAGS ?= -OPERATOR_IMAGE=$(REGISTRY)/trusted-cluster-operator:$(TAG) +OPERATOR_IMAGE ?= $(REGISTRY)/trusted-cluster-operator:$(TAG) COMPUTE_PCRS_IMAGE=$(REGISTRY)/compute-pcrs:$(TAG) REG_SERVER_IMAGE=$(REGISTRY)/registration-server:$(TAG) ATTESTATION_KEY_REGISTER_IMAGE=$(REGISTRY)/attestation-key-register:$(TAG) From 08e0a84334044c483edf8e661483cd4dec715b3e Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Thu, 18 Jun 2026 12:18:26 +0200 Subject: [PATCH 11/14] tests/azure: Pass Ignition as user-data Custom data is sometimes not yet available at Ignition fetch stage, i.e. fetch skips our merge even though other parts of config are available. Use user-data (IMDS) instead. Signed-off-by: Jakob Naucke --- test_utils/src/virt/azure.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_utils/src/virt/azure.rs b/test_utils/src/virt/azure.rs index 7d67e812..f2b09e8a 100644 --- a/test_utils/src/virt/azure.rs +++ b/test_utils/src/virt/azure.rs @@ -110,7 +110,7 @@ impl VmBackend for AzureBackend { args.extend(["--storage-sku", "StandardSSD_LRS"]); args.extend(["--admin-username", "core"]); args.extend(["--ssh-key-values", &self.config.ssh_public_key]); - args.extend(["--custom-data", &custom_data]); + args.extend(["--user-data", &custom_data]); args.extend(["--security-type", "ConfidentialVM"]); args.extend(["--enable-secure-boot", "true", "--enable-vtpm", "true"]); args.extend(["--os-disk-security-encryption-type", "VMGuestStateOnly"]); From 2cf06f8d0115f37aa7d84b810d53467b81256ca4 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 19 Jun 2026 14:52:51 +0200 Subject: [PATCH 12/14] Requeue upon failed component installation which can occur because creation of resources with owner refereces to resources of incompletely propagated CRDs can fail Signed-off-by: Jakob Naucke --- operator/src/main.rs | 105 ++++++++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 46 deletions(-) diff --git a/operator/src/main.rs b/operator/src/main.rs index 63307d37..03ff138e 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -7,7 +7,7 @@ use std::env; use std::sync::Arc; use std::time::Duration; -use anyhow::Result; +use anyhow::{Context, Result}; use env_logger::Env; use futures_util::StreamExt; use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; @@ -15,7 +15,7 @@ use kube::runtime::controller::{Action, Controller}; use kube::runtime::reflector::{self, Store}; use kube::runtime::watcher; use kube::{Api, Client}; -use log::{error, info, warn}; +use log::{info, warn}; use operator::{generate_owner_reference, upsert_condition}; use trusted_cluster_operator_lib::{TrustedExecutionCluster, TrustedExecutionClusterStatus}; @@ -132,9 +132,11 @@ async fn reconcile( update_status!(clusters, name, status)?; } - install_trustee_configuration(kube_client.clone(), &cluster).await?; - install_register_server(kube_client.clone(), &cluster).await?; - install_attestation_key_register(kube_client.clone(), &cluster).await?; + if let Err(e) = install_components(&kube_client, &cluster).await { + // warn with `:?` to also get context + warn!("Installation of a component failed: {e:?}\nRequeueing..."); + return Ok(Action::requeue(Duration::from_secs(60))); + } reference_values::adopt_approved_images(kube_client, &cluster).await?; let installed_condition = installed_condition(INSTALLED_REASON, generation, existing_status); @@ -146,6 +148,13 @@ async fn reconcile( Ok(Action::await_change()) } +async fn install_components(client: &Client, cluster: &TrustedExecutionCluster) -> Result<()> { + install_trustee_configuration(client.clone(), cluster).await?; + install_register_server(client.clone(), cluster).await?; + install_attestation_key_register(client.clone(), cluster).await?; + Ok(()) +} + async fn install_trustee_configuration( client: Client, cluster: &TrustedExecutionCluster, @@ -153,32 +162,28 @@ async fn install_trustee_configuration( let owner_reference = generate_owner_reference(cluster)?; let trustee_secret = &cluster.spec.trustee_secret; - match trustee::generate_trustee_data(client.clone(), owner_reference.clone(), trustee_secret) + trustee::generate_trustee_data(client.clone(), owner_reference.clone(), trustee_secret) .await - { - Ok(_) => info!("Generate configmap for the KBS configuration",), - Err(e) => error!("Failed to create the KBS configuration configmap: {e}"), - } + .context("Failed to create the KBS configuration configmap")?; + info!("Generated configmap for the KBS configuration"); - match trustee::generate_attestation_policy(client.clone(), owner_reference.clone()).await { - Ok(_) => info!("Generate configmap for the attestation policy",), - Err(e) => error!("Failed to create the attestation policy configmap: {e}"), - } + trustee::generate_attestation_policy(client.clone(), owner_reference.clone()) + .await + .context("Failed to create the attestation policy configmap")?; + info!("Generated configmap for the attestation policy"); let kbs_port = cluster.spec.trustee_kbs_port; - match trustee::generate_kbs_service(client.clone(), owner_reference.clone(), kbs_port).await { - Ok(_) => info!("Generate the KBS service"), - Err(e) => error!("Failed to create the KBS service: {e}"), - } + trustee::generate_kbs_service(client.clone(), owner_reference.clone(), kbs_port) + .await + .context("Failed to create the KBS service")?; + info!("Generated the KBS service"); let default = format!("{TEC_REGISTRY}/key-broker-service:{TRUSTEE_VERSION}"); let trustee_image = env::var(RELATED_IMAGE_TRUSTEE).ok().unwrap_or(default); - match trustee::generate_kbs_deployment(client, owner_reference, &trustee_image, trustee_secret) + trustee::generate_kbs_deployment(client, owner_reference, &trustee_image, trustee_secret) .await - { - Ok(_) => info!("Generate the KBS deployment"), - Err(e) => error!("Failed to create the KBS deployment: {e}"), - } + .context("Failed to create the KBS deployment")?; + info!("Generated the KBS deployment"); Ok(()) } @@ -189,25 +194,21 @@ async fn install_register_server(client: Client, cluster: &TrustedExecutionClust let env = RELATED_IMAGE_REGISTRATION_SERVER; let default_image = format!("{TEC_REGISTRY}/registration-server:{COMPONENT_VERSION}"); let register_server_image = env::var(env).ok().unwrap_or(default_image); - match register_server::create_register_server_deployment( + register_server::create_register_server_deployment( client.clone(), owner_reference.clone(), ®ister_server_image, &cluster.spec.register_server_secret, ) .await - { - Ok(_) => info!("Register server deployment created/updated successfully"), - Err(e) => error!("Failed to create register server deployment: {e}"), - } + .context("Failed to create register server deployment")?; + info!("Register server deployment created/updated successfully"); let port = cluster.spec.register_server_port; - match register_server::create_register_server_service(client.clone(), owner_reference, port) + register_server::create_register_server_service(client.clone(), owner_reference, port) .await - { - Ok(_) => info!("Register server service created/updated successfully"), - Err(e) => error!("Failed to create register server service: {e}"), - } + .context("Failed to create register server service")?; + info!("Register server service created/updated successfully"); Ok(()) } @@ -221,29 +222,24 @@ async fn install_attestation_key_register( let env = RELATED_IMAGE_ATTESTATION_KEY_REGISTER; let default_image = format!("{TEC_REGISTRY}/attestation-key-register:{COMPONENT_VERSION}"); let attestation_key_register_image = env::var(env).ok().unwrap_or(default_image); - match attestation_key_register::create_attestation_key_register_deployment( + attestation_key_register::create_attestation_key_register_deployment( client.clone(), owner_reference.clone(), &attestation_key_register_image, &cluster.spec.attestation_key_register_secret, ) .await - { - Ok(_) => info!("Attestation key register deployment created/updated successfully"), - Err(e) => error!("Failed to create attestation key register deployment: {e}"), - } + .context("Failed to create attestation key register deployment")?; + info!("Attestation key register deployment created/updated successfully"); - let port = cluster.spec.attestation_key_register_port; - match attestation_key_register::create_attestation_key_register_service( + attestation_key_register::create_attestation_key_register_service( client.clone(), owner_reference, - port, + cluster.spec.attestation_key_register_port, ) .await - { - Ok(_) => info!("Attestation key register service created/updated successfully"), - Err(e) => error!("Failed to create attestation key register service: {e}"), - } + .context("Failed to create attestation key register service")?; + info!("Attestation key register service created/updated successfully"); Ok(()) } @@ -297,6 +293,8 @@ async fn main() -> Result<()> { #[cfg(test)] mod tests { use http::{Method, Request, StatusCode}; + use k8s_openapi::api::apps::v1::Deployment; + use k8s_openapi::api::core::v1::{ConfigMap, Service}; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}; use kube::api::ObjectList; use kube::client::Body; @@ -439,7 +437,22 @@ mod tests { let clos = async |req: Request, ctr| { if ctr < 8 && req.method() == Method::POST { - Ok(serde_json::to_string(&dummy_cluster()).unwrap()) + use serde_json::to_string; + let resp = match ctr { + // Trustee + 0 => to_string(&ConfigMap::default()), + 1 => to_string(&ConfigMap::default()), + 2 => to_string(&Service::default()), + 3 => to_string(&Deployment::default()), + // Registration server + 4 => to_string(&Deployment::default()), + 5 => to_string(&Service::default()), + // Attestation key register server + 6 => to_string(&Deployment::default()), + 7 => to_string(&Service::default()), + _ => unreachable!("unexpected counter {ctr}"), + }; + Ok(resp.unwrap()) } else if ctr == 8 && req.method() == Method::GET { let object_list = ObjectList:: { items: Vec::new(), From 30ef74a0c9f4e6171f27b1f2d4d779937b5c4d83 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 19 Jun 2026 15:58:53 +0200 Subject: [PATCH 13/14] tests: Wait for custom resources to remove PR #278 correctly recognized that Machines sometimes stayed behind when deleting the namespace immediately after TEC. This extends to ApprovedImages and AttestationKeys, but because all these resources are owned, wait for their removal before namespace removal instead of deleting them. This also reverts commit ed64522964a3453cafdcfa2c6525886a71996eae. Signed-off-by: Jakob Naucke --- test_utils/src/lib.rs | 48 ++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 1d60f5f1..de277124 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -23,9 +23,8 @@ use trusted_cluster_operator_lib::certificates::{ }; use trusted_cluster_operator_lib::issuers::{Issuer, IssuerCa, IssuerSpec}; -use trusted_cluster_operator_lib::Machine; -use trusted_cluster_operator_lib::TrustedExecutionCluster; -use trusted_cluster_operator_lib::{endpoints::*, images::*}; +use trusted_cluster_operator_lib::{ApprovedImage, AttestationKey, Machine}; +use trusted_cluster_operator_lib::{TrustedExecutionCluster, endpoints::*, images::*}; pub mod timer; pub use timer::Poller; @@ -471,7 +470,16 @@ impl TestContext { pub async fn cleanup(&self) -> Result<()> { self.delete_trusted_execution_cluster().await?; - self.delete_machines().await?; + let timeout = scaled_duration(60); + let msg = format!("Resources were left behind after {timeout:?}"); + let poller = Poller::new().with_timeout(timeout).with_error_message(msg); + let chk = || async move { + self.check_no_resources::().await?; + self.check_no_resources::().await?; + self.check_no_resources::().await?; + Ok::<_, anyhow::Error>(()) + }; + poller.poll_async(chk).await?; self.cleanup_namespace().await?; self.cleanup_manifests_dir()?; Ok(()) @@ -499,6 +507,19 @@ impl TestContext { Ok(()) } + async fn check_no_resources(&self) -> Result<()> + where + K: kube::Resource + Clone, + K: k8s_openapi::serde::de::DeserializeOwned + std::fmt::Debug + Send + 'static, + { + let api: Api = Api::namespaced(self.client.clone(), &self.test_namespace); + let list = api.list(&Default::default()).await?; + if let Some(item) = list.items.first() { + return Err(anyhow!("Resource still present: {item:?}")); + } + Ok(()) + } + async fn delete_trusted_execution_cluster(&self) -> Result<()> { let tec_api: Api = Api::namespaced(self.client.clone(), &self.test_namespace); @@ -528,25 +549,6 @@ impl TestContext { Ok(()) } - async fn delete_machines(&self) -> Result<()> { - let machine_api: Api = Api::namespaced(self.client.clone(), &self.test_namespace); - let machine_list = machine_api.list(&Default::default()).await?; - - for machine in &machine_list.items { - if let Some(name) = &machine.metadata.name { - test_info!( - &self.test_name, - "Waiting for Machine {} to be deleted", - name - ); - wait_for_resource_deleted(&machine_api, name, scaled_timeout(120)).await?; - test_info!(&self.test_name, "Machine {} has been deleted", name); - } - } - - Ok(()) - } - async fn cleanup_namespace(&self) -> Result<()> { let namespace_api: Api = Api::all(self.client.clone()); let dp = DeleteParams::default(); From 2296e03a3e079b3bcc466db7ebfd896f536d4377 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 23 Jun 2026 18:35:26 +0200 Subject: [PATCH 14/14] tests: Wait for services important especially for OpenShift exposure, but can matter on all platforms Signed-off-by: Jakob Naucke --- test_utils/src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index de277124..53d75d37 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -944,6 +944,14 @@ impl TestContext { timeout(scaled_duration(300), done).await.context(ctx)??; } + let svc = ATTESTATION_KEY_REGISTER_SERVICE; + let services: Api = Api::namespaced(self.client.clone(), ns); + for svc in [REGISTER_SERVER_SERVICE, TRUSTEE_SERVICE, svc] { + let done = await_condition(services.clone(), svc, |s: Option<&Service>| s.is_some()); + let ctx = format!("waiting for service {svc} to exist"); + timeout(scaled_duration(60), done).await.context(ctx)??; + } + let platform = get_k8s_platform(&self.client, &self.test_namespace); let svc = REGISTER_SERVER_SERVICE; let depl = REGISTER_SERVER_DEPLOYMENT;