diff --git a/Makefile b/Makefile index 72d647e8..faeeea4a 100644 --- a/Makefile +++ b/Makefile @@ -12,21 +12,19 @@ PLATFORM ?= kind KUBECTL=kubectl INTEGRATION_TEST_THREADS ?= 1 -# Azure CI only: which image to use as Kind host -KIND_HOST_URN = RedHat:RHEL:10-lvm-gen2:10.1.2026022409 LOCALBIN ?= $(shell pwd)/bin CONTROLLER_TOOLS_VERSION ?= $(shell go list -m -f '{{.Version}}' sigs.k8s.io/controller-tools) CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) YQ_VERSION ?= $(shell go list -m -f '{{.Version}}' github.com/mikefarah/yq/v4) YQ ?= $(LOCALBIN)/yq-$(YQ_VERSION) -KOPIUM_VERSION ?= $(shell grep kopium lib/Cargo.toml | sed -E 's/.*"(.*)"/\1/') +KOPIUM_VERSION ?= $(shell cargo metadata --format-version 1 | jq -r '.resolve.nodes[] | select(.deps[]?.name == "kopium") | .deps[] | select(.name == "kopium") | .pkg | split("@")[1]') KOPIUM ?= $(LOCALBIN)/kopium-$(KOPIUM_VERSION) REGISTRY ?= quay.io/trusted-execution-clusters TAG ?= latest PUSH_FLAGS ?= -OPERATOR_IMAGE=$(REGISTRY)/trusted-cluster-operator:$(TAG) +OPERATOR_IMAGE ?= $(REGISTRY)/trusted-cluster-operator:$(TAG) COMPUTE_PCRS_IMAGE=$(REGISTRY)/compute-pcrs:$(TAG) REG_SERVER_IMAGE=$(REGISTRY)/registration-server:$(TAG) ATTESTATION_KEY_REGISTER_IMAGE=$(REGISTRY)/attestation-key-register:$(TAG) @@ -57,8 +55,6 @@ RBAC_YAML_PATH = config/rbac API_PATH = api/v1alpha1 generate: $(CONTROLLER_GEN) $(call controller-gen,./...,*) - $(call controller-gen,github.com/openshift/api/route/v1,*) - $(call controller-gen,github.com/openshift/api/config/v1,*_ingresses.yaml) $(call controller-gen,github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1,*) RS_LIB_PATH = lib/src diff --git a/api/v1alpha1/crds.go b/api/v1alpha1/crds.go index 42b8025b..066ffe29 100644 --- a/api/v1alpha1/crds.go +++ b/api/v1alpha1/crds.go @@ -30,8 +30,7 @@ var ( // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=trustedexecutionclusters;machines;approvedimages;attestationkeys,verbs=create;delete;get;list;patch;update;watch -// +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=trustedexecutionclusters/finalizers,verbs=update -// +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=machines/finalizers,verbs=update +// +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=trustedexecutionclusters/finalizers;machines/finalizers;attestationkeys/finalizers;approvedimages/finalizers,verbs=update // +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=trustedexecutionclusters/status;machines/status;approvedimages/status;attestationkeys/status,verbs=get;patch;update // TrustedExecutionClusterSpec defines the desired state of TrustedExecutionCluster diff --git a/go.mod b/go.mod index 0d78775f..aa76a40d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ go 1.25.0 require ( github.com/cert-manager/cert-manager v1.20.2 github.com/mikefarah/yq/v4 v4.53.3 - github.com/openshift/api v0.0.0-20260213204242-d34f11c515b3 github.com/projectcalico/api v0.0.0-20251022175904-f2ab03771208 k8s.io/api v0.35.3 k8s.io/apimachinery v0.35.3 @@ -40,7 +39,6 @@ require ( github.com/gobuffalo/flect v1.0.3 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/goccy/go-yaml v1.19.2 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect diff --git a/go.sum b/go.sum index 38b9ced0..0a0fc138 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,6 @@ github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= @@ -104,8 +102,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -138,8 +134,6 @@ github.com/onsi/ginkgo/v2 v2.28.0 h1:Rrf+lVLmtlBIKv6KrIGJCjyY8N36vDVcutbGJkyqjJc github.com/onsi/ginkgo/v2 v2.28.0/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= -github.com/openshift/api v0.0.0-20260213204242-d34f11c515b3 h1:SZ8+jxtkMvpb4HDTjSAbaOyhFsw5PiWhjBog+XLY7jc= -github.com/openshift/api v0.0.0-20260213204242-d34f11c515b3/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= @@ -183,8 +177,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA= github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= github.com/zclconf/go-cty v1.18.1 h1:yEGE8M4iIZlyKQURZNb2SnEyZlZHUcBCnx6KF81KuwM= @@ -215,56 +207,31 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U= go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w= diff --git a/lib/src/kopium.rs b/lib/src/kopium.rs index 91ecd4d5..328edfd0 100644 --- a/lib/src/kopium.rs +++ b/lib/src/kopium.rs @@ -7,8 +7,6 @@ pub mod attestationkeys; pub mod certificaterequests; pub mod certificates; pub mod clusterissuers; -pub mod ingresses; pub mod issuers; pub mod machines; -pub mod routes; pub mod trustedexecutionclusters; diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 8861b6ce..03adb539 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -13,9 +13,7 @@ mod vendor_kopium; use k8s_openapi::jiff::Timestamp; pub use kopium::approvedimages::*; pub use kopium::attestationkeys::*; -pub use kopium::ingresses as openshift_ingresses; pub use kopium::machines::*; -pub use kopium::routes; pub use kopium::trustedexecutionclusters::*; pub use kopium::certificaterequests; diff --git a/operator/src/main.rs b/operator/src/main.rs index d6fc9944..03ff138e 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -7,7 +7,7 @@ use std::env; use std::sync::Arc; use std::time::Duration; -use anyhow::Result; +use anyhow::{Context, Result}; use env_logger::Env; use futures_util::StreamExt; use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; @@ -15,7 +15,7 @@ use kube::runtime::controller::{Action, Controller}; use kube::runtime::reflector::{self, Store}; use kube::runtime::watcher; use kube::{Api, Client}; -use log::{error, info, warn}; +use log::{info, warn}; use operator::{generate_owner_reference, upsert_condition}; use trusted_cluster_operator_lib::{TrustedExecutionCluster, TrustedExecutionClusterStatus}; @@ -35,7 +35,7 @@ use operator::*; /// Default tag for Trustee image const TRUSTEE_VERSION: &str = "v0.17.0"; /// Default version tag for operator-managed component images -const COMPONENT_VERSION: &str = "0.2.0"; +const COMPONENT_VERSION: &str = "v0.2.0"; /// Default registry const TEC_REGISTRY: &str = "quay.io/trusted-execution-clusters"; @@ -132,9 +132,11 @@ async fn reconcile( update_status!(clusters, name, status)?; } - install_trustee_configuration(kube_client.clone(), &cluster).await?; - install_register_server(kube_client.clone(), &cluster).await?; - install_attestation_key_register(kube_client.clone(), &cluster).await?; + if let Err(e) = install_components(&kube_client, &cluster).await { + // warn with `:?` to also get context + warn!("Installation of a component failed: {e:?}\nRequeueing..."); + return Ok(Action::requeue(Duration::from_secs(60))); + } reference_values::adopt_approved_images(kube_client, &cluster).await?; let installed_condition = installed_condition(INSTALLED_REASON, generation, existing_status); @@ -146,6 +148,13 @@ async fn reconcile( Ok(Action::await_change()) } +async fn install_components(client: &Client, cluster: &TrustedExecutionCluster) -> Result<()> { + install_trustee_configuration(client.clone(), cluster).await?; + install_register_server(client.clone(), cluster).await?; + install_attestation_key_register(client.clone(), cluster).await?; + Ok(()) +} + async fn install_trustee_configuration( client: Client, cluster: &TrustedExecutionCluster, @@ -153,32 +162,28 @@ async fn install_trustee_configuration( let owner_reference = generate_owner_reference(cluster)?; let trustee_secret = &cluster.spec.trustee_secret; - match trustee::generate_trustee_data(client.clone(), owner_reference.clone(), trustee_secret) + trustee::generate_trustee_data(client.clone(), owner_reference.clone(), trustee_secret) .await - { - Ok(_) => info!("Generate configmap for the KBS configuration",), - Err(e) => error!("Failed to create the KBS configuration configmap: {e}"), - } + .context("Failed to create the KBS configuration configmap")?; + info!("Generated configmap for the KBS configuration"); - match trustee::generate_attestation_policy(client.clone(), owner_reference.clone()).await { - Ok(_) => info!("Generate configmap for the attestation policy",), - Err(e) => error!("Failed to create the attestation policy configmap: {e}"), - } + trustee::generate_attestation_policy(client.clone(), owner_reference.clone()) + .await + .context("Failed to create the attestation policy configmap")?; + info!("Generated configmap for the attestation policy"); let kbs_port = cluster.spec.trustee_kbs_port; - match trustee::generate_kbs_service(client.clone(), owner_reference.clone(), kbs_port).await { - Ok(_) => info!("Generate the KBS service"), - Err(e) => error!("Failed to create the KBS service: {e}"), - } + trustee::generate_kbs_service(client.clone(), owner_reference.clone(), kbs_port) + .await + .context("Failed to create the KBS service")?; + info!("Generated the KBS service"); let default = format!("{TEC_REGISTRY}/key-broker-service:{TRUSTEE_VERSION}"); let trustee_image = env::var(RELATED_IMAGE_TRUSTEE).ok().unwrap_or(default); - match trustee::generate_kbs_deployment(client, owner_reference, &trustee_image, trustee_secret) + trustee::generate_kbs_deployment(client, owner_reference, &trustee_image, trustee_secret) .await - { - Ok(_) => info!("Generate the KBS deployment"), - Err(e) => error!("Failed to create the KBS deployment: {e}"), - } + .context("Failed to create the KBS deployment")?; + info!("Generated the KBS deployment"); Ok(()) } @@ -189,25 +194,21 @@ async fn install_register_server(client: Client, cluster: &TrustedExecutionClust let env = RELATED_IMAGE_REGISTRATION_SERVER; let default_image = format!("{TEC_REGISTRY}/registration-server:{COMPONENT_VERSION}"); let register_server_image = env::var(env).ok().unwrap_or(default_image); - match register_server::create_register_server_deployment( + register_server::create_register_server_deployment( client.clone(), owner_reference.clone(), ®ister_server_image, &cluster.spec.register_server_secret, ) .await - { - Ok(_) => info!("Register server deployment created/updated successfully"), - Err(e) => error!("Failed to create register server deployment: {e}"), - } + .context("Failed to create register server deployment")?; + info!("Register server deployment created/updated successfully"); let port = cluster.spec.register_server_port; - match register_server::create_register_server_service(client.clone(), owner_reference, port) + register_server::create_register_server_service(client.clone(), owner_reference, port) .await - { - Ok(_) => info!("Register server service created/updated successfully"), - Err(e) => error!("Failed to create register server service: {e}"), - } + .context("Failed to create register server service")?; + info!("Register server service created/updated successfully"); Ok(()) } @@ -221,29 +222,24 @@ async fn install_attestation_key_register( let env = RELATED_IMAGE_ATTESTATION_KEY_REGISTER; let default_image = format!("{TEC_REGISTRY}/attestation-key-register:{COMPONENT_VERSION}"); let attestation_key_register_image = env::var(env).ok().unwrap_or(default_image); - match attestation_key_register::create_attestation_key_register_deployment( + attestation_key_register::create_attestation_key_register_deployment( client.clone(), owner_reference.clone(), &attestation_key_register_image, &cluster.spec.attestation_key_register_secret, ) .await - { - Ok(_) => info!("Attestation key register deployment created/updated successfully"), - Err(e) => error!("Failed to create attestation key register deployment: {e}"), - } + .context("Failed to create attestation key register deployment")?; + info!("Attestation key register deployment created/updated successfully"); - let port = cluster.spec.attestation_key_register_port; - match attestation_key_register::create_attestation_key_register_service( + attestation_key_register::create_attestation_key_register_service( client.clone(), owner_reference, - port, + cluster.spec.attestation_key_register_port, ) .await - { - Ok(_) => info!("Attestation key register service created/updated successfully"), - Err(e) => error!("Failed to create attestation key register service: {e}"), - } + .context("Failed to create attestation key register service")?; + info!("Attestation key register service created/updated successfully"); Ok(()) } @@ -297,6 +293,8 @@ async fn main() -> Result<()> { #[cfg(test)] mod tests { use http::{Method, Request, StatusCode}; + use k8s_openapi::api::apps::v1::Deployment; + use k8s_openapi::api::core::v1::{ConfigMap, Service}; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}; use kube::api::ObjectList; use kube::client::Body; @@ -439,7 +437,22 @@ mod tests { let clos = async |req: Request, ctr| { if ctr < 8 && req.method() == Method::POST { - Ok(serde_json::to_string(&dummy_cluster()).unwrap()) + use serde_json::to_string; + let resp = match ctr { + // Trustee + 0 => to_string(&ConfigMap::default()), + 1 => to_string(&ConfigMap::default()), + 2 => to_string(&Service::default()), + 3 => to_string(&Deployment::default()), + // Registration server + 4 => to_string(&Deployment::default()), + 5 => to_string(&Service::default()), + // Attestation key register server + 6 => to_string(&Deployment::default()), + 7 => to_string(&Service::default()), + _ => unreachable!("unexpected counter {ctr}"), + }; + Ok(resp.unwrap()) } else if ctr == 8 && req.method() == Method::GET { let object_list = ObjectList:: { items: Vec::new(), diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 85be2a94..53d75d37 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -3,25 +3,28 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Result, anyhow}; +use anyhow::{Context, Result, anyhow}; use fs_extra::dir; -use k8s_openapi::api::apps::v1::Deployment; -use k8s_openapi::api::core::v1::{ConfigMap, Namespace, Secret, Service, ServicePort, ServiceSpec}; -use kube::api::{DeleteParams, ObjectMeta}; +use k8s_openapi::api::apps::v1::{Deployment, DeploymentCondition, DeploymentStatus}; +use k8s_openapi::api::core::v1::{ + ConfigMap, LoadBalancerStatus, Namespace, Secret, Service, ServicePort, ServiceSpec, + ServiceStatus, +}; +use kube::api::{DeleteParams, ObjectMeta, Patch}; +use kube::runtime::wait::await_condition; use kube::{Api, Client}; +use serde_json::json; use std::path::{Path, PathBuf}; use std::{collections::BTreeMap, env, sync::Once, time::Duration}; use tokio::process::Command; +use tokio::time::timeout; use trusted_cluster_operator_lib::certificates::{ - Certificate, CertificateIssuerRef, CertificateSpec, + Certificate, CertificateIssuerRef, CertificateSpec, CertificateStatus, }; use trusted_cluster_operator_lib::issuers::{Issuer, IssuerCa, IssuerSpec}; -use trusted_cluster_operator_lib::Machine; -use trusted_cluster_operator_lib::TrustedExecutionCluster; -use trusted_cluster_operator_lib::openshift_ingresses::Ingress; -use trusted_cluster_operator_lib::routes::Route; -use trusted_cluster_operator_lib::{endpoints::*, images::*}; +use trusted_cluster_operator_lib::{ApprovedImage, AttestationKey, Machine}; +use trusted_cluster_operator_lib::{TrustedExecutionCluster, endpoints::*, images::*}; pub mod timer; pub use timer::Poller; @@ -48,6 +51,9 @@ const ROOT_SECRET: &str = "root-secret"; const REG_SECRET: &str = "reg-srv-secret"; const TRUSTEE_SECRET: &str = "trustee-secret"; const ATT_REG_SECRET: &str = "att-reg-secret"; +const REG_CERT: &str = "reg-srv-cert"; +const TRUSTEE_CERT: &str = "trustee-cert"; +const ATT_REG_CERT: &str = "att-reg-cert"; pub fn compare_pcrs(actual: &[Pcr], expected: &[Pcr]) -> bool { if actual.len() != expected.len() { @@ -116,8 +122,7 @@ macro_rules! kube_apply { args.extend_from_slice(&["--server-side", "--force-conflicts"]) } )? - let mut cmd = get_k8s_platform().kubectl(); - let apply_output = cmd.args(args).output().await?; + let apply_output = kubectl().args(args).output().await?; if !apply_output.status.success() { let stderr = String::from_utf8_lossy(&apply_output.stderr); return Err(anyhow!("{} failed: {}", $log, stderr)); @@ -157,39 +162,53 @@ pub fn ensure_command(name: &str) -> Result<()> { result.map_err(|_| anyhow!("Command {name} not found. Please install {name} first.")) } +fn kubectl() -> Command { + match env::var(PLATFORM_ENV).as_deref().unwrap_or("kind") { + "openshift" => Command::new("oc"), + _ => Command::new("kubectl"), + } +} + #[async_trait::async_trait] #[auto_impl::auto_impl(Box)] trait K8sPlatform: Send + Sync { fn add_scc(&self, kustomization: &mut serde_yaml::Value); async fn expose( &self, - client: &Client, - namespace: &str, service: &str, + deployment: &str, + cert_name: &str, test_name: &str, - port: i32, ) -> Result<()>; - async fn get_cluster_url( - &self, - client: &Client, - namespace: &str, - service: &str, - port: Option, - ) -> Result; - fn kubectl(&self) -> Command; + async fn get_cluster_url(&self, service: &str, port: Option) -> Result; } struct Kind { public: bool, + client: Client, + namespace: String, +} +struct OpenShift { + client: Client, + namespace: String, } -struct OpenShift {} struct OtherK8s {} -fn get_k8s_platform() -> Box { +fn get_k8s_platform(client: &Client, namespace: &str) -> Box { + let client = client.clone(); + let namespace = namespace.to_string(); match env::var(PLATFORM_ENV).as_deref().unwrap_or("kind") { - "kind" => Box::new(Kind { public: false }), - "kind_public" => Box::new(Kind { public: true }), - "openshift" => Box::new(OpenShift {}), + "kind" => Box::new(Kind { + public: false, + client, + namespace, + }), + "kind_public" => Box::new(Kind { + public: true, + client, + namespace, + }), + "openshift" => Box::new(OpenShift { client, namespace }), _ => Box::new(OtherK8s {}), } } @@ -197,14 +216,7 @@ fn get_k8s_platform() -> Box { #[async_trait::async_trait] impl K8sPlatform for Kind { fn add_scc(&self, _: &mut serde_yaml::Value) {} - async fn expose( - &self, - client: &Client, - namespace: &str, - service: &str, - _: &str, - _: i32, - ) -> Result<()> { + async fn expose(&self, service: &str, _: &str, _: &str, _: &str) -> Result<()> { if !self.public { return Ok(()); } @@ -228,7 +240,7 @@ impl K8sPlatform for Kind { port, ..Default::default() }; - let services: Api = Api::namespaced(client.clone(), namespace); + let services: Api = Api::namespaced(self.client.clone(), &self.namespace); let service = Service { metadata: ObjectMeta { name: Some(format!("{service}-forward")), @@ -246,22 +258,33 @@ impl K8sPlatform for Kind { Ok(()) } - async fn get_cluster_url( - &self, - _: &Client, - namespace: &str, - service: &str, - port: Option, - ) -> Result { - let url = format!("{service}.{namespace}.svc.cluster.local"); + async fn get_cluster_url(&self, service: &str, port: Option) -> Result { + let url = format!("{service}.{}.svc.cluster.local", self.namespace); Ok(match port { Some(port) => format!("{url}:{port}"), None => url, }) } +} + +enum OpenShiftHost { + Ip(String), + Hostname(String), + None, +} - fn kubectl(&self) -> Command { - Command::new("kubectl") +impl OpenShift { + async fn get_url(&self, service: &str) -> OpenShiftHost { + let services: Api = Api::namespaced(self.client.clone(), &self.namespace); + let Ok(svc) = services.get(service).await else { + return OpenShiftHost::None; + }; + let ingress = &svc.status.unwrap().load_balancer.unwrap().ingress.unwrap()[0]; + match (&ingress.hostname, &ingress.ip) { + (Some(hostname), _) => OpenShiftHost::Hostname(hostname.clone()), + (_, Some(ip)) => OpenShiftHost::Ip(ip.clone()), + (None, None) => OpenShiftHost::None, + } } } @@ -276,45 +299,79 @@ impl K8sPlatform for OpenShift { async fn expose( &self, - _: &Client, - namespace: &str, service: &str, + deployment: &str, + cert_name: &str, _: &str, - port: i32, ) -> Result<()> { - ensure_command("oc")?; - let mut args = vec!["create", "route", "passthrough", service, "-n", namespace]; - let svc = format!("--service={service}"); - let port = format!("--port={port}"); - args.extend_from_slice(&[&svc, &port]); - let output = Command::new("oc").args(args).output().await?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(anyhow!("oc command failed: {stderr}")); - } - Ok(()) - } + let services: Api = Api::namespaced(self.client.clone(), &self.namespace); + let pp = Default::default(); + let json = json!({ + "spec": { + "type": "LoadBalancer" + } + }); + services.patch(service, &pp, &Patch::Merge(&json)).await?; + let has_ingress = |svc: Option<&Service>| { + let chk_lb = |bal: &LoadBalancerStatus| bal.ingress.is_some(); + let chk_st = |st: &ServiceStatus| st.load_balancer.as_ref().map(chk_lb); + let chk_svc = |svc: &Service| svc.status.as_ref().and_then(chk_st); + svc.and_then(chk_svc).unwrap_or(false) + }; + let ingress_ready = await_condition(services, service, has_ingress); + let ctx = format!("waiting for ingress on {service} to be ready"); + let duration = scaled_duration(60); + timeout(duration, ingress_ready).await.context(ctx)??; + + let certs: Api = Api::namespaced(self.client.clone(), &self.namespace); + let cert = certs.get(cert_name).await?; + let old_revision = cert.status.and_then(|st| st.revision).unwrap_or(0); + let cert_patch = match self.get_url(service).await { + OpenShiftHost::Ip(ip) => json!({ + "spec": { + "ipAddresses": [ip], + "dnsNames": [], + } + }), + OpenShiftHost::Hostname(name) => json!({ + "spec": { + "dnsNames": [name], + "ipAddresses": [] + } + }), + OpenShiftHost::None => { + return Err(anyhow!("expected service {service}")); + } + }; + let cert_merge = Patch::Merge(cert_patch); + certs.patch(cert_name, &pp, &cert_merge).await?; - async fn get_cluster_url( - &self, - client: &Client, - namespace: &str, - service: &str, - _: Option, - ) -> Result { - let routes: Api = Api::namespaced(client.clone(), namespace); - if let Ok(route) = routes.get(service).await { - return Ok(route.spec.host.expect("route existed, but had no host")); - } - // Fallback when route does not exist yet - let ingresses: Api = Api::all(client.clone()); - let ingress = ingresses.get("cluster").await?; - let domain = ingress.spec.domain.unwrap(); - Ok(format!("{service}-{namespace}.{domain}")) + let cert_reissued = |cert: Option<&Certificate>| { + let chk = |st: &CertificateStatus| st.revision.map(|r| r > old_revision); + cert.and_then(|c| c.status.as_ref().and_then(chk)) + .unwrap_or(false) + }; + let cert_done = await_condition(certs, cert_name, cert_reissued); + let ctx = format!("waiting for cert {cert_name} to have a rev newer than {old_revision}"); + timeout(duration, cert_done).await.context(ctx)??; + + let deployments: Api = Api::namespaced(self.client.clone(), &self.namespace); + deployments.restart(deployment).await?; + + Ok(()) } - fn kubectl(&self) -> Command { - Command::new("oc") + async fn get_cluster_url(&self, service: &str, port: Option) -> Result { + let append_port = |e| match port { + Some(p) => format!("{e}:{p}"), + None => e, + }; + Ok(match self.get_url(service).await { + OpenShiftHost::Ip(ip) => append_port(ip), + OpenShiftHost::Hostname(name) => append_port(name), + // Service did not exist yet, put empty name in cert and patch upon expose + OpenShiftHost::None => String::new(), + }) } } @@ -322,30 +379,20 @@ impl K8sPlatform for OpenShift { impl K8sPlatform for OtherK8s { fn add_scc(&self, _: &mut serde_yaml::Value) {} - async fn expose(&self, _: &Client, _: &str, _: &str, test_name: &str, _: i32) -> Result<()> { + async fn expose(&self, _: &str, _: &str, _: &str, test_name: &str) -> Result<()> { let warn = "You appear to be on an environment that is not Kind or OpenShift. \ Ensure operator services are reachable"; test_warn!(test_name, "{warn}"); Ok(()) } - async fn get_cluster_url( - &self, - _: &Client, - _: &str, - _: &str, - _: Option, - ) -> Result { + async fn get_cluster_url(&self, _: &str, _: Option) -> Result { Err(anyhow!(SET_CLUSTER_ERR)) } - - fn kubectl(&self) -> Command { - Command::new("kubectl") - } } pub async fn get_cluster_url( - client: Client, + client: &Client, namespace: &str, service: &str, port: Option, @@ -357,8 +404,8 @@ pub async fn get_cluster_url( None => full_url, }); } - get_k8s_platform() - .get_cluster_url(&client, namespace, service, port) + get_k8s_platform(client, namespace) + .get_cluster_url(service, port) .await } @@ -423,7 +470,16 @@ impl TestContext { pub async fn cleanup(&self) -> Result<()> { self.delete_trusted_execution_cluster().await?; - self.delete_machines().await?; + let timeout = scaled_duration(60); + let msg = format!("Resources were left behind after {timeout:?}"); + let poller = Poller::new().with_timeout(timeout).with_error_message(msg); + let chk = || async move { + self.check_no_resources::().await?; + self.check_no_resources::().await?; + self.check_no_resources::().await?; + Ok::<_, anyhow::Error>(()) + }; + poller.poll_async(chk).await?; self.cleanup_namespace().await?; self.cleanup_manifests_dir()?; Ok(()) @@ -451,6 +507,19 @@ impl TestContext { Ok(()) } + async fn check_no_resources(&self) -> Result<()> + where + K: kube::Resource + Clone, + K: k8s_openapi::serde::de::DeserializeOwned + std::fmt::Debug + Send + 'static, + { + let api: Api = Api::namespaced(self.client.clone(), &self.test_namespace); + let list = api.list(&Default::default()).await?; + if let Some(item) = list.items.first() { + return Err(anyhow!("Resource still present: {item:?}")); + } + Ok(()) + } + async fn delete_trusted_execution_cluster(&self) -> Result<()> { let tec_api: Api = Api::namespaced(self.client.clone(), &self.test_namespace); @@ -468,7 +537,7 @@ impl TestContext { tec_api.delete(name, &dp).await?; // Wait for the resource to be deleted - wait_for_resource_deleted(&tec_api, name, scaled_timeout(120), 5).await?; + wait_for_resource_deleted(&tec_api, name, scaled_timeout(120)).await?; test_info!( &self.test_name, "TrustedExecutionCluster {} has been deleted", @@ -480,25 +549,6 @@ impl TestContext { Ok(()) } - async fn delete_machines(&self) -> Result<()> { - let machine_api: Api = Api::namespaced(self.client.clone(), &self.test_namespace); - let machine_list = machine_api.list(&Default::default()).await?; - - for machine in &machine_list.items { - if let Some(name) = &machine.metadata.name { - test_info!( - &self.test_name, - "Waiting for Machine {} to be deleted", - name - ); - wait_for_resource_deleted(&machine_api, name, 120, 5).await?; - test_info!(&self.test_name, "Machine {} has been deleted", name); - } - } - - Ok(()) - } - async fn cleanup_namespace(&self) -> Result<()> { let namespace_api: Api = Api::all(self.client.clone()); let dp = DeleteParams::default(); @@ -506,13 +556,8 @@ impl TestContext { match namespace_api.get(&self.test_namespace).await { Ok(_) => { namespace_api.delete(&self.test_namespace, &dp).await?; - wait_for_resource_deleted( - &namespace_api, - &self.test_namespace, - scaled_timeout(300), - 5, - ) - .await?; + let timeout = scaled_timeout(300); + wait_for_resource_deleted(&namespace_api, &self.test_namespace, timeout).await?; test_info!(&self.test_name, "Deleted namespace {}", self.test_namespace); } Err(kube::Error::Api(ae)) if ae.code == 404 => { @@ -547,49 +592,6 @@ impl TestContext { Ok(()) } - async fn wait_for_deployment_ready( - &self, - deployments_api: &Api, - deployment_name: &str, - timeout_secs: u64, - ) -> Result<()> { - test_info!( - &self.test_name, - "Waiting for deployment {} to be ready", - deployment_name - ); - let poller = Poller::new() - .with_timeout(Duration::from_secs(timeout_secs)) - .with_interval(Duration::from_secs(5)) - .with_error_message(format!( - "{deployment_name} deployment does not have 1 available replica after {timeout_secs} seconds" - )); - - let test_name_owned = self.test_name.clone(); - poller - .poll_async(move || { - let api = deployments_api.clone(); - let name = deployment_name.to_string(); - let tn = test_name_owned.clone(); - async move { - let deployment = api.get(&name).await?; - - if let Some(status) = &deployment.status - && let Some(available_replicas) = status.available_replicas - && available_replicas == 1 - { - test_info!(&tn, "{} deployment has 1 available replica", name); - return Ok(()); - } - - Err(anyhow!( - "{name} deployment does not have 1 available replica yet" - )) - } - }) - .await - } - async fn create_certificate( &self, service_name: &str, @@ -598,7 +600,7 @@ impl TestContext { issuer_name: &str, ) -> Result<()> { let ns = &self.test_namespace; - let domain = get_cluster_url(self.client.clone(), ns, service_name, None).await?; + let domain = get_cluster_url(&self.client, ns, service_name, None).await?; let certs: Api = Api::namespaced(self.client.clone(), ns); let cert = Certificate { metadata: ObjectMeta { @@ -673,18 +675,18 @@ impl TestContext { issuers.create(&Default::default(), &issuer).await?; let svc = REGISTER_SERVER_SERVICE; - self.create_certificate(svc, "reg-srv-cert", REG_SECRET, issuer_name) + self.create_certificate(svc, REG_CERT, REG_SECRET, issuer_name) .await?; - self.create_certificate(TRUSTEE_SERVICE, "trustee-cert", TRUSTEE_SECRET, issuer_name) + self.create_certificate(TRUSTEE_SERVICE, TRUSTEE_CERT, TRUSTEE_SECRET, issuer_name) .await?; let svc = ATTESTATION_KEY_REGISTER_SERVICE; - self.create_certificate(svc, "att-reg-cert", ATT_REG_SECRET, issuer_name) + self.create_certificate(svc, ATT_REG_CERT, ATT_REG_SECRET, issuer_name) .await?; let secrets: Api = Api::namespaced(self.client.clone(), &self.test_namespace); - wait_for_resource_created(&secrets, REG_SECRET, scaled_timeout(60), 1).await?; - wait_for_resource_created(&secrets, TRUSTEE_SECRET, scaled_timeout(60), 1).await?; - wait_for_resource_created(&secrets, ATT_REG_SECRET, scaled_timeout(60), 1).await?; + for secret in [REG_SECRET, TRUSTEE_SECRET, ATT_REG_SECRET] { + wait_for_resource_created(&secrets, secret, scaled_timeout(60)).await?; + } Ok(()) } @@ -773,7 +775,7 @@ impl TestContext { self.set_certificates().await?; let tec = "trustedexecutionclusters.trusted-execution-clusters.io"; let args = ["get", "crd", tec]; - let crd_check_output = Command::new("kubectl").args(args).output().await?; + let crd_check_output = kubectl().args(args).output().await?; if crd_check_output.status.success() { test_info!( @@ -828,7 +830,7 @@ impl TestContext { std::fs::write(&le_rb_dst, le_rb_content)?; test_info!(&self.test_name, "Preparing RBAC kustomization"); - let platform = get_k8s_platform(); + let platform = get_k8s_platform(&self.client, &self.test_namespace); let kustomization_src = workspace_root.join("config/rbac/kustomization.yaml.in"); let kustomization_content = std::fs::read_to_string(&kustomization_src)?; let mut kustom_value: serde_yaml::Value = serde_yaml::from_str(&kustomization_content)?; @@ -874,19 +876,12 @@ impl TestContext { async fn apply_cr_manifests(&self, manifests_path: &Path) -> Result<()> { let ns = &self.test_namespace; - let trustee_addr = - get_cluster_url(self.client.clone(), ns, TRUSTEE_SERVICE, Some(TRUSTEE_PORT)).await?; let cr_manifest_path = manifests_path.join("trusted_execution_cluster_cr.yaml"); let cr_content = std::fs::read_to_string(&cr_manifest_path)?; let mut cr_value: serde_yaml::Value = serde_yaml::from_str(&cr_content)?; let spec_map = cr_value.get_mut("spec").unwrap().as_mapping_mut().unwrap(); - spec_map.insert( - serde_yaml::Value::String("publicTrusteeAddr".to_string()), - serde_yaml::Value::String(trustee_addr.clone()), - ); - spec_map.insert( serde_yaml::Value::String("trusteeSecret".to_string()), serde_yaml::Value::String(TRUSTEE_SECRET.to_string()), @@ -901,10 +896,9 @@ impl TestContext { ); if get_virt_provider()? == VirtProvider::Kubevirt { - let platform = get_k8s_platform(); - let svc = ATTESTATION_KEY_REGISTER_SERVICE; + let platform = get_k8s_platform(&self.client, &self.test_namespace); let port = ATTESTATION_KEY_REGISTER_PORT; - let address = platform.get_cluster_url(&self.client, ns, svc, Some(port)); + let address = platform.get_cluster_url(ATTESTATION_KEY_REGISTER_SERVICE, Some(port)); spec_map.insert( serde_yaml::Value::String("publicAttestationKeyRegisterAddr".to_string()), serde_yaml::Value::String(address.await?), @@ -914,11 +908,6 @@ impl TestContext { let updated_content = serde_yaml::to_string(&cr_value)?; std::fs::write(&cr_manifest_path, updated_content)?; - test_info!( - &self.test_name, - "Updated CR manifest with publicTrusteeAddr: {trustee_addr}", - ); - let cr_manifest_str = cr_manifest_path.to_str().unwrap(); kube_apply!(cr_manifest_str, &self.test_name, "Applying CR manifest"); @@ -933,68 +922,68 @@ impl TestContext { "Applying ApprovedImage manifest" ); - let deployments_api: Api = Api::namespaced(self.client.clone(), ns); + let depl_ready = |depl: Option<&Deployment>| { + let chk_cond = |c: &DeploymentCondition| c.type_ == "Available" && c.status == "True"; + let chk_status = + |st: &DeploymentStatus| st.conditions.as_ref().map(|cs| cs.iter().any(chk_cond)); + let chk = |depl: &Deployment| depl.status.as_ref().and_then(chk_status); + depl.and_then(chk).unwrap_or(false) + }; - self.wait_for_deployment_ready( - &deployments_api, + let depls: Api = Api::namespaced(self.client.clone(), ns); + for depl in [ "trusted-cluster-operator", - scaled_timeout(120), - ) - .await?; - self.wait_for_deployment_ready( - &deployments_api, REGISTER_SERVER_DEPLOYMENT, - scaled_timeout(300), - ) - .await?; - self.wait_for_deployment_ready(&deployments_api, TRUSTEE_DEPLOYMENT, scaled_timeout(180)) - .await?; - self.wait_for_deployment_ready( - &deployments_api, + TRUSTEE_DEPLOYMENT, ATTESTATION_KEY_REGISTER_DEPLOYMENT, - scaled_timeout(120), - ) - .await?; - - let platform = get_k8s_platform(); - let ak_port = ATTESTATION_KEY_REGISTER_PORT; - for (svc, port) in [ - (TRUSTEE_SERVICE, TRUSTEE_PORT), - (ATTESTATION_KEY_REGISTER_SERVICE, ak_port), - (REGISTER_SERVER_SERVICE, REGISTER_SERVER_PORT), ] { - platform - .expose(&self.client, ns, svc, &self.test_name, port) - .await?; + let info = format!("Waiting for deployment {depl} to be ready"); + test_info!(&self.test_name, "{info}"); + let done = await_condition(depls.clone(), depl, depl_ready); + let ctx = format!("waiting for deployment {depl} to be ready"); + timeout(scaled_duration(300), done).await.context(ctx)??; } + let svc = ATTESTATION_KEY_REGISTER_SERVICE; + let services: Api = Api::namespaced(self.client.clone(), ns); + for svc in [REGISTER_SERVER_SERVICE, TRUSTEE_SERVICE, svc] { + let done = await_condition(services.clone(), svc, |s: Option<&Service>| s.is_some()); + let ctx = format!("waiting for service {svc} to exist"); + timeout(scaled_duration(60), done).await.context(ctx)??; + } + + let platform = get_k8s_platform(&self.client, &self.test_namespace); + let svc = REGISTER_SERVER_SERVICE; + let depl = REGISTER_SERVER_DEPLOYMENT; + let test_name = &self.test_name; + platform.expose(svc, depl, REG_CERT, test_name).await?; + let svc = TRUSTEE_SERVICE; + let depl = TRUSTEE_DEPLOYMENT; + platform.expose(svc, depl, TRUSTEE_CERT, test_name).await?; + let svc = ATTESTATION_KEY_REGISTER_SERVICE; + let depl = ATTESTATION_KEY_REGISTER_DEPLOYMENT; + platform.expose(svc, depl, ATT_REG_CERT, test_name).await?; + + let tecs: Api = Api::namespaced(self.client.clone(), ns); + let trustee_addr = + get_cluster_url(&self.client, ns, TRUSTEE_SERVICE, Some(TRUSTEE_PORT)).await?; + let json = json!({ + "spec": { + "publicTrusteeAddr": trustee_addr + } + }); + let patch = Patch::Merge(&json); + tecs.patch("trusted-execution-cluster", &Default::default(), &patch) + .await?; + let info = format!("Updated TEC resource with publicTrusteeAddr: {trustee_addr}"); + test_info!(&self.test_name, "{info}"); + test_info!( &self.test_name, "Waiting for image-pcrs ConfigMap to be created" ); let configmap_api: Api = Api::namespaced(self.client.clone(), ns); - - let err = format!("image-pcrs ConfigMap in the namespace {ns} not found"); - let poller = Poller::new() - .with_timeout(scaled_duration(60)) - .with_interval(Duration::from_secs(5)) - .with_error_message(err); - - let test_name_owned = self.test_name.clone(); - let check_fn = move || { - let api = configmap_api.clone(); - let tn = test_name_owned.clone(); - async move { - let result = api.get("image-pcrs").await; - if result.is_ok() { - test_info!(&tn, "image-pcrs ConfigMap created"); - } - result - } - }; - poller.poll_async(check_fn).await?; - - Ok(()) + wait_for_resource_created(&configmap_api, "image-pcrs", scaled_timeout(60)).await } } @@ -1043,60 +1032,34 @@ pub async fn wait_for_resource_created( api: &Api, resource_name: &str, timeout_secs: u64, - interval_secs: u64, -) -> anyhow::Result<()> -where - K: kube::Resource + Clone + std::fmt::Debug, - K: k8s_openapi::serde::de::DeserializeOwned, -{ - wait_for_resource_state(api, resource_name, timeout_secs, interval_secs, true).await -} - -pub async fn wait_for_resource_deleted( - api: &Api, - resource_name: &str, - timeout_secs: u64, - interval_secs: u64, ) -> Result<()> where - K: kube::Resource + Clone + std::fmt::Debug, + K: kube::Resource + Clone + std::fmt::Debug + Send + 'static, K: k8s_openapi::serde::de::DeserializeOwned, { - wait_for_resource_state(api, resource_name, timeout_secs, interval_secs, false).await + let created = |r: Option<&K>| r.is_some(); + let done = await_condition(api.clone(), resource_name, created); + let type_ = std::any::type_name::(); + let ctx = format!("waiting {timeout_secs} for {type_} '{resource_name}' creation"); + let duration = Duration::from_secs(timeout_secs); + timeout(duration, done).await.context(ctx)??; + Ok(()) } -async fn wait_for_resource_state( +pub async fn wait_for_resource_deleted( api: &Api, resource_name: &str, timeout_secs: u64, - interval_secs: u64, - state: bool, ) -> Result<()> where - K: kube::Resource + Clone + std::fmt::Debug, + K: kube::Resource + Clone + std::fmt::Debug + Send + 'static, K: k8s_openapi::serde::de::DeserializeOwned, { - let poller = Poller::new() - .with_timeout(Duration::from_secs(timeout_secs)) - .with_interval(Duration::from_secs(interval_secs)) - .with_error_message(format!( - "{resource_name} did not reach state {} after {timeout_secs} seconds", - if state { "created" } else { "deleted" } - )); - - let check = || { - let api = api.clone(); - let name = resource_name.to_string(); - async move { - let result = api.get(&name).await; - if let Err(kube::Error::Api(ae)) = &result - && ae.code != 404 - { - panic!("Unexpected error while fetching {name}: {ae:?}"); - } - let err = anyhow!("{name} not in desired state: {result:?}"); - (result.is_err() ^ state).then_some(()).ok_or(err) - } - }; - poller.poll_async(check).await + let deleted = |r: Option<&K>| r.is_none(); + let done = await_condition(api.clone(), resource_name, deleted); + let type_ = std::any::type_name::(); + let ctx = format!("waiting {timeout_secs} for {type_} '{resource_name}' deletion"); + let duration = Duration::from_secs(timeout_secs); + timeout(duration, done).await.context(ctx)??; + Ok(()) } diff --git a/test_utils/src/virt/azure.rs b/test_utils/src/virt/azure.rs index 7d67e812..f2b09e8a 100644 --- a/test_utils/src/virt/azure.rs +++ b/test_utils/src/virt/azure.rs @@ -110,7 +110,7 @@ impl VmBackend for AzureBackend { args.extend(["--storage-sku", "StandardSSD_LRS"]); args.extend(["--admin-username", "core"]); args.extend(["--ssh-key-values", &self.config.ssh_public_key]); - args.extend(["--custom-data", &custom_data]); + args.extend(["--user-data", &custom_data]); args.extend(["--security-type", "ConfidentialVM"]); args.extend(["--enable-secure-boot", "true", "--enable-vtpm", "true"]); args.extend(["--os-disk-security-encryption-type", "VMGuestStateOnly"]); diff --git a/test_utils/src/virt/kubevirt.rs b/test_utils/src/virt/kubevirt.rs index b4ad9fac..a19d0054 100644 --- a/test_utils/src/virt/kubevirt.rs +++ b/test_utils/src/virt/kubevirt.rs @@ -5,12 +5,13 @@ use anyhow::{Context, Result, anyhow}; use k8s_openapi::{api::core::v1::Secret, apimachinery::pkg::util::intstr::IntOrString}; -use kube::{Api, api::ObjectMeta}; +use kube::{Api, api::ObjectMeta, runtime::wait::await_condition}; use std::{collections::BTreeMap, time::Duration}; +use tokio::time::timeout; use trusted_cluster_operator_lib::virtualmachines::*; use super::{VmBackend, VmConfig, generate_ignition, ssh_exec}; -use crate::{Poller, ensure_command}; +use crate::ensure_command; pub struct KubevirtBackend(pub VmConfig); @@ -141,29 +142,17 @@ impl VmBackend for KubevirtBackend { async fn wait_for_running(&self, timeout_secs: u64) -> Result<()> { let api: Api = Api::namespaced(self.0.client.clone(), &self.0.namespace); - - let poller = Poller::new() - .with_timeout(Duration::from_secs(timeout_secs)) - .with_interval(Duration::from_secs(5)) - .with_error_message(format!( - "VirtualMachine {} did not reach Running phase after {timeout_secs} seconds", - self.0.vm_name - )); - - let check_fn = || { - let api = api.clone(); - async move { - let vm = api.get(&self.0.vm_name).await?; - let status = vm.status.and_then(|p| p.printable_status); - if status.map(|s| s == "Running").unwrap_or(false) { - return Ok(()); - } - let vm_name = &self.0.vm_name; - let err = anyhow!("VirtualMachine {vm_name} is not in Running phase yet"); - Err(err) - } + let machine_running = |m: Option<&VirtualMachine>| { + let status = m.as_ref().and_then(|m| m.status.as_ref()); + let print_status = status.and_then(|s| s.printable_status.as_ref()); + print_status.map(|s| s == "Running").unwrap_or(false) }; - poller.poll_async(check_fn).await + let vm_name = &self.0.vm_name; + let done = await_condition(api, vm_name, machine_running); + let ctx = format!("waiting {timeout_secs} for VirtualMachine {vm_name} to be running"); + let duration = Duration::from_secs(timeout_secs); + timeout(duration, done).await.context(ctx)??; + Ok(()) } async fn ssh_exec(&self, command: &str) -> Result { diff --git a/test_utils/src/virt/mod.rs b/test_utils/src/virt/mod.rs index cf5d33cc..38309158 100644 --- a/test_utils/src/virt/mod.rs +++ b/test_utils/src/virt/mod.rs @@ -79,7 +79,7 @@ pub async fn generate_ignition(config: &VmConfig) -> Result { let client = config.client.clone(); let ns = &config.namespace; let port = Some(REGISTER_SERVER_PORT); - let register_server_url = get_cluster_url(client, ns, REGISTER_SERVER_SERVICE, port).await?; + let register_server_url = get_cluster_url(&client, ns, REGISTER_SERVER_SERVICE, port).await?; let root_pem_encoded = utf8_percent_encode(&config.ca_pem, NON_ALPHANUMERIC); let ignition = Ignition { version: "3.6.0-experimental".to_string(), diff --git a/tests/attestation.rs b/tests/attestation.rs index cb1a2157..c9afcdec 100644 --- a/tests/attestation.rs +++ b/tests/attestation.rs @@ -204,7 +204,7 @@ async fn test_vm_reboot_delete_machine() -> anyhow::Result<()> { let list = machines.list(&Default::default()).await?; let name = list.items[0].metadata.name.as_ref().unwrap(); machines.delete(name, &Default::default()).await?; - wait_for_resource_deleted(&machines, name, scaled_timeout(120), 5).await?; + wait_for_resource_deleted(&machines, name, scaled_timeout(120)).await?; test_ctx.info("Performing reboot, expecting missing resource"); let _reboot_result = att_ctx.backend.ssh_exec("sudo systemctl reboot").await; diff --git a/tests/trusted_execution_cluster.rs b/tests/trusted_execution_cluster.rs index 742ac840..bceb9e16 100644 --- a/tests/trusted_execution_cluster.rs +++ b/tests/trusted_execution_cluster.rs @@ -3,14 +3,18 @@ // // SPDX-License-Identifier: MIT -use anyhow::anyhow; +use anyhow::Context; use compute_pcrs_lib::{Part, Pcr}; use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::{ConfigMap, Secret}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, OwnerReference}; use kube::api::ObjectMeta; +use kube::runtime::wait::await_condition; use kube::{Api, api::DeleteParams}; use std::time::Duration; +use tokio::time::timeout; use trusted_cluster_operator_lib::conditions::NOT_COMMITTED_REASON_PENDING; +use trusted_cluster_operator_lib::endpoints::{REGISTER_SERVER_DEPLOYMENT, TRUSTEE_DEPLOYMENT}; use trusted_cluster_operator_lib::reference_values::ImagePcrs; use trusted_cluster_operator_lib::{ ApprovedImage, AttestationKey, Machine, TrustedExecutionCluster, generate_owner_reference, @@ -23,6 +27,12 @@ const APPROVED_IMAGE_NAME: &str = "coreos"; const TRUSTEE_CONFIG_MAP: &str = "trustee-data"; const RV_JSON_KEY: &str = "reference-values.json"; +fn ak_approved(ak: Option<&AttestationKey>) -> bool { + let is_approved = |c: &Condition| c.type_ == "Approved" && c.status == "True"; + let cs = ak.and_then(|ak| ak.status.as_ref().and_then(|s| s.conditions.as_ref())); + cs.map(|cs| cs.iter().any(is_approved)).unwrap_or(false) +} + named_test!( async fn test_trusted_execution_cluster_uninstall() -> anyhow::Result<()> { let test_ctx = setup!().await?; @@ -82,40 +92,9 @@ named_test!( )); // Wait for the AttestationKey to be approved (operator should match Machine IP and approve it) - let poller = Poller::new() - .with_timeout(scaled_duration(30)) - .with_interval(Duration::from_millis(500)) - .with_error_message("AttestationKey was not approved".to_string()); - - poller - .poll_async(|| { - let ak_api = attestation_keys.clone(); - let ak_name_clone = ak_name.clone(); - async move { - let ak = ak_api.get(&ak_name_clone).await?; - - // Check for Approved condition - let has_approved_condition = ak - .status - .as_ref() - .and_then(|s| s.conditions.as_ref()) - .map(|conditions| { - conditions - .iter() - .any(|c| c.type_ == "Approved" && c.status == "True") - }) - .unwrap_or(false); - - if !has_approved_condition { - return Err(anyhow!( - "AttestationKey does not have Approved condition yet" - )); - } - - Ok(()) - } - }) - .await?; + let done = await_condition(attestation_keys.clone(), &ak_name, ak_approved); + let ctx = format!("waiting for AttestationKey {ak_name} to be approved"); + timeout(scaled_duration(30), done).await.context(ctx)??; test_ctx.info("AttestationKey successfully approved"); @@ -125,26 +104,20 @@ named_test!( api.delete(TEC_NAME, &dp).await?; // Wait until it disappears - wait_for_resource_deleted(&api, TEC_NAME, scaled_timeout(120), 5).await?; + wait_for_resource_deleted(&api, TEC_NAME, scaled_timeout(120)).await?; let deployments_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted( - &deployments_api, - "trustee-deployment", - scaled_timeout(120), - 1, - ) - .await?; - wait_for_resource_deleted(&deployments_api, "register-server", scaled_timeout(120), 1) - .await?; + let timeout = scaled_timeout(120); + wait_for_resource_deleted(&deployments_api, TRUSTEE_DEPLOYMENT, timeout).await?; + wait_for_resource_deleted(&deployments_api, REGISTER_SERVER_DEPLOYMENT, timeout).await?; let images_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted(&images_api, APPROVED_IMAGE_NAME, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&images_api, APPROVED_IMAGE_NAME, scaled_timeout(120)).await?; - wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120), 1).await?; - wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120)).await?; let secrets_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120)).await?; test_ctx.cleanup().await?; @@ -159,30 +132,15 @@ async fn test_image_pcrs_configmap_updates() -> anyhow::Result<()> { let namespace = test_ctx.namespace(); let configmap_api: Api = Api::namespaced(client.clone(), namespace); - - let poller = Poller::new() - .with_timeout(scaled_duration(180)) - .with_interval(Duration::from_secs(5)) - .with_error_message("image-pcrs ConfigMap not populated with data".to_string()); - - poller - .poll_async(|| { - let api = configmap_api.clone(); - async move { - let cm = api.get("image-pcrs").await?; - - if let Some(data) = &cm.data - && let Some(image_pcrs_json) = data.get("image-pcrs.json") - && let Ok(image_pcrs) = serde_json::from_str::(image_pcrs_json) - && !image_pcrs.0.is_empty() - { - return Ok(()); - } - - Err(anyhow!("image-pcrs ConfigMap not yet populated with image-pcrs.json data")) - } - }) - .await?; + let populated = |cm: Option<&ConfigMap>| { + let data = cm.and_then(|cm| cm.data.as_ref()); + let json = data.and_then(|data| data.get("image-pcrs.json")); + let pcrs = json.and_then(|json| serde_json::from_str::(json).ok()); + pcrs.map(|pcrs| !pcrs.0.is_empty()).unwrap_or(false) + }; + let done = await_condition(configmap_api.clone(), "image-pcrs", populated); + let ctx = "waiting for ConfigMap image-pcrs to be populated"; + timeout(scaled_duration(180), done).await.context(ctx)??; let image_pcrs_cm = configmap_api.get("image-pcrs").await?; assert_eq!(image_pcrs_cm.metadata.name.as_deref(), Some("image-pcrs")); @@ -268,23 +226,14 @@ async fn test_image_disallow() -> anyhow::Result<()> { images.delete(APPROVED_IMAGE_NAME, &DeleteParams::default()).await?; let configmap_api: Api = Api::namespaced(client.clone(), namespace); - let poller = Poller::new() - .with_timeout(scaled_duration(180)) - .with_interval(Duration::from_secs(5)) - .with_error_message("Reference value not removed".to_string()); - poller.poll_async(|| { - let api = configmap_api.clone(); - async move { - let cm = api.get(TRUSTEE_CONFIG_MAP).await?; - if let Some(data) = &cm.data - && let Some(reference_values_json) = data.get(RV_JSON_KEY) - && !reference_values_json.contains(EXPECTED_PCR4) - { - return Ok(()); - } - Err(anyhow!("Reference value not yet removed")) - } - }).await?; + let chk_removed = |cm: Option<&ConfigMap>| { + let data = cm.and_then(|cm| cm.data.as_ref()); + let json = data.and_then(|data| data.get(RV_JSON_KEY)); + json.map(|json| !json.contains(EXPECTED_PCR4)).unwrap_or(false) + }; + let rv_removed = await_condition(configmap_api, TRUSTEE_CONFIG_MAP, chk_removed); + let ctx = format!("waiting for ConfigMap {TRUSTEE_CONFIG_MAP} to not contain PCR value"); + timeout(scaled_duration(180), rv_removed).await.context(ctx)??; test_ctx.cleanup().await?; Ok(()) @@ -348,81 +297,27 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { "Created test Machine: {machine_name} with uuid: {machine_uuid}", )); - // Poll for the AttestationKey to be approved, have owner reference, and have a Secret created + // Timeout for the AttestationKey to be approved, have owner reference, and have a Secret created + let approved = await_condition(attestation_keys.clone(), &ak_name, ak_approved); + let ctx = format!("waiting for AttestationKey {ak_name} to be approved"); + timeout(scaled_duration(30), approved).await.context(ctx)??; + let chk_machine_owner = |ak: Option<&AttestationKey>| { + let chk_owner = |owner: &OwnerReference| owner.kind == "Machine" && owner.name == machine_name; + let refs = ak.and_then(|ak| ak.metadata.owner_references.as_ref()); + refs.map(|refs| refs.iter().any(chk_owner)).unwrap_or(false) + }; + let has_machine_owner = await_condition(attestation_keys.clone(), &ak_name, chk_machine_owner); + let ctx = format!("waiting for AttestationKey {ak_name} to be owned by Machine {machine_name}"); + timeout(scaled_duration(30), has_machine_owner).await.context(ctx)??; let secrets_api: Api = Api::namespaced(client.clone(), namespace); - let poller = Poller::new() - .with_timeout(scaled_duration(30)) - .with_interval(Duration::from_millis(500)) - .with_error_message("AttestationKey was not approved with owner reference and secret".to_string()); - - poller - .poll_async(|| { - let ak_api = attestation_keys.clone(); - let secrets = secrets_api.clone(); - let ak_name_clone = ak_name.clone(); - let machine_name_clone = machine_name.clone(); - async move { - let ak = ak_api.get(&ak_name_clone).await?; - - // Check for Approved condition - let has_approved_condition = ak - .status - .as_ref() - .and_then(|s| s.conditions.as_ref()) - .map(|conditions| { - conditions - .iter() - .any(|c| c.type_ == "Approved" && c.status == "True") - }) - .unwrap_or(false); - - if !has_approved_condition { - return Err(anyhow!( - "AttestationKey does not have Approved condition yet" - )); - } - - // Check for owner reference to the Machine - let has_machine_owner_ref = ak - .metadata - .owner_references - .as_ref() - .map(|owner_refs| { - owner_refs.iter().any(|owner_ref| { - owner_ref.kind == "Machine" && owner_ref.name == machine_name_clone - }) - }) - .unwrap_or(false); - - if !has_machine_owner_ref { - return Err(anyhow!( - "AttestationKey does not have owner reference to Machine yet" - )); - } - - // Check that a Secret with the same name exists and has the AttestationKey as owner - let secret = secrets.get(&ak_name_clone).await?; - let has_ak_owner_ref = secret - .metadata - .owner_references - .as_ref() - .map(|owner_refs| { - owner_refs.iter().any(|owner_ref| { - owner_ref.kind == "AttestationKey" && owner_ref.name == ak_name_clone - }) - }) - .unwrap_or(false); - - if !has_ak_owner_ref { - return Err(anyhow!( - "Secret does not have owner reference to AttestationKey yet" - )); - } - - Ok(()) - } - }) - .await?; + let chk_ak_owner = |secret: Option<&Secret>| { + let chk_owner = |owner: &OwnerReference| owner.kind == "AttestationKey" && owner.name == ak_name; + let refs = secret.and_then(|s| s.metadata.owner_references.as_ref()); + refs.map(|refs| refs.iter().any(chk_owner)).unwrap_or(false) + }; + let has_ak_owner = await_condition(secrets_api.clone(), &ak_name, chk_ak_owner); + let ctx = format!("waiting for Secret {ak_name} to be owned by AttestationKey {ak_name}"); + timeout(scaled_duration(30), has_ak_owner).await.context(ctx)??; test_ctx.info(format!( "AttestationKey successfully approved with owner reference to Machine: {machine_name} and Secret created" @@ -433,11 +328,11 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { machines.delete(&machine_name, &dp).await?; test_ctx.info(format!("Deleted Machine: {machine_name}")); - wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120)).await?; test_ctx.info("Machine successfully deleted"); - wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120)).await?; test_ctx.info("AttestationKey successfully deleted"); - wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120), 1).await?; + wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120)).await?; test_ctx.info("Secret successfully deleted"); test_ctx.cleanup().await?; @@ -465,22 +360,14 @@ async fn test_nonexistent_approved_image() -> anyhow::Result<()> { status: None, }).await?; - let poller = Poller::new() - .with_timeout(scaled_duration(30)) - .with_interval(Duration::from_millis(500)) - .with_error_message("ApprovedImage not created".to_string()); - poller.poll_async(|| { - let api = images.clone(); - async move { - let img = api.get("coreos1").await?; - if img.status.as_ref().and_then(|s| s.conditions.as_ref()).map(|conditions| { - conditions.iter().any(|c| c.reason == NOT_COMMITTED_REASON_PENDING) - }).unwrap_or(false) { - return Ok(()); - } - Err(anyhow::anyhow!("ApprovedImage not yet committed")) - } - }).await?; + let is_pending = |img: Option<&ApprovedImage>| { + let pending = |c: &Condition| c.reason == NOT_COMMITTED_REASON_PENDING; + let cs = img.and_then(|img| img.status.as_ref()).and_then(|s| s.conditions.as_ref()); + cs.map(|cs| cs.iter().any(pending)).unwrap_or(false) + }; + let done = await_condition(images, "coreos1", is_pending); + let ctx = "waiting for ApprovedImage coreos1 to be PodPending"; + timeout(scaled_duration(30), done).await.context(ctx)??; test_ctx.cleanup().await?; Ok(()) @@ -502,28 +389,8 @@ async fn test_approved_image_readoption() -> anyhow::Result<()> { test_ctx.info(format!("Deleting TrustedExecuctionCluster {TEC_NAME}")); clusters.delete(TEC_NAME, &Default::default()).await?; - let removal_poller = Poller::new() - .with_timeout(Duration::from_secs(60)) - .with_interval(Duration::from_secs(5)) - .with_error_message(format!( - "ConfigMap {TRUSTEE_CONFIG_MAP} or ApprovedImage {APPROVED_IMAGE_NAME} not removed" - )); - removal_poller - .poll_async(|| { - let configmaps = configmaps.clone(); - let images = images.clone(); - async move { - if configmaps.get(TRUSTEE_CONFIG_MAP).await.is_ok() { - return Err(anyhow!("ConfigMap {TRUSTEE_CONFIG_MAP} not yet removed")); - } - if images.get(APPROVED_IMAGE_NAME).await.is_ok() { - let err = anyhow!("ApprovedImage {APPROVED_IMAGE_NAME} not yet removed"); - return Err(err); - } - Ok(()) - } - }) - .await?; + wait_for_resource_deleted(&configmaps, TRUSTEE_CONFIG_MAP, scaled_timeout(60)).await?; + wait_for_resource_deleted(&images, APPROVED_IMAGE_NAME, scaled_timeout(60)).await?; test_ctx.info(format!("Configmap {TRUSTEE_CONFIG_MAP} was removed")); let image = ApprovedImage { @@ -548,25 +415,14 @@ async fn test_approved_image_readoption() -> anyhow::Result<()> { // Ensure adoption works even when cluster creation was delayed tokio::time::sleep(Duration::from_secs(5)).await; clusters.create(&Default::default(), &cluster).await?; - let regeneration_poller = Poller::new() - .with_timeout(Duration::from_secs(180)) - .with_interval(Duration::from_secs(5)) - .with_error_message("Reference value not regenerated".to_string()); - regeneration_poller - .poll_async(|| { - let configmaps = configmaps.clone(); - async move { - let configmap = configmaps.get(TRUSTEE_CONFIG_MAP).await?; - if let Some(data) = &configmap.data - && let Some(json) = data.get(RV_JSON_KEY) - && json.contains(EXPECTED_PCR4) - { - return Ok(()); - } - Err(anyhow!("Reference value not yet regenerated")) - } - }) - .await?; + let chk_added = |cm: Option<&ConfigMap>| { + let data = cm.and_then(|cm| cm.data.as_ref()); + let json = data.and_then(|data| data.get(RV_JSON_KEY)); + json.map(|json| json.contains(EXPECTED_PCR4)).unwrap_or(false) + }; + let rv_added = await_condition(configmaps, TRUSTEE_CONFIG_MAP, chk_added); + let ctx = format!("waiting for ConfigMap {TRUSTEE_CONFIG_MAP} to contain PCR value"); + timeout(scaled_duration(180), rv_added).await.context(ctx)??; test_ctx.info("Reference values regenerated"); test_ctx.cleanup().await?; diff --git a/tools.go b/tools.go index db5c5401..79e22d62 100644 --- a/tools.go +++ b/tools.go @@ -13,8 +13,6 @@ package tools import ( _ "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" _ "github.com/mikefarah/yq/v4" - _ "github.com/openshift/api/config/v1" - _ "github.com/openshift/api/route/v1" _ "github.com/projectcalico/api/pkg/lib/numorstring" _ "sigs.k8s.io/controller-tools/cmd/controller-gen" _ "sigs.k8s.io/kind"