diff --git a/CHANGELOG.md b/CHANGELOG.md index e6a29becc..8c5a05502 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,57 @@ # Changelog +## 2026-09-07 — [#1195](https://github.com/cobaltcore-dev/cortex/pull/1195) + +### cortex-shim v0.1.17 (sha-cd38777a) + +New features: +- Remote apiserver liveness probe — adds a per-remote reachability probe (`GET /readyz`) that detects when a remote apiserver becomes unreachable after its informer cache has synced; after a configurable failure threshold (default 3 × 10s), the probe cancels the manager cycle context so the existing self-healing supervisor rebuilds the manager with current config ([#1190](https://github.com/cobaltcore-dev/cortex/pull/1190)) +- `cortex_multicluster_remote_apiserver_reachable{host}` gauge — exposes per-remote apiserver reachability on the process-lifetime monitor, surviving manager rebuilds ([#1190](https://github.com/cobaltcore-dev/cortex/pull/1190)) +- `CortexPlacementShimRemoteApiserverUnreachable` alert — fires when a remote apiserver is unreachable ([#1190](https://github.com/cobaltcore-dev/cortex/pull/1190)) + +Non-breaking changes: +- Fix multicluster recorder for controller-runtime v0.25.0 `EventRecorder` interface change — adds `AnnotatedEventf` method to `MultiClusterRecorder` to satisfy the wider `recorder.EventRecorder` interface ([#1191](https://github.com/cobaltcore-dev/cortex/pull/1191)) +- Update `sigs.k8s.io/controller-runtime` to v0.25.0 ([#1188](https://github.com/cobaltcore-dev/cortex/pull/1188)) +- Update `sigs.k8s.io/controller-tools` to v0.22.0 (CRD annotation bump) ([#1186](https://github.com/cobaltcore-dev/cortex/pull/1186)) +- Update `github.com/sapcc/go-bits` ([#1187](https://github.com/cobaltcore-dev/cortex/pull/1187)) +- Update `kube-prometheus-stack` to v89 ([#1189](https://github.com/cobaltcore-dev/cortex/pull/1189)) + +### cortex v0.4.1 (sha-cd38777a) + +Non-breaking changes: +- Fix multicluster recorder for controller-runtime v0.25.0 `EventRecorder` interface change ([#1191](https://github.com/cobaltcore-dev/cortex/pull/1191)) +- CRDs regenerated with controller-tools v0.22.0 ([#1186](https://github.com/cobaltcore-dev/cortex/pull/1186)) +- Update `sigs.k8s.io/controller-runtime` to v0.25.0 ([#1188](https://github.com/cobaltcore-dev/cortex/pull/1188)) +- Update `github.com/sapcc/go-bits` ([#1187](https://github.com/cobaltcore-dev/cortex/pull/1187)) + +### cortex-placement-shim v0.1.17 + +Includes updated chart cortex-shim v0.1.17. + +### cortex-nova v0.0.90 + +Includes updated chart cortex v0.4.1. + +### cortex-cinder v0.0.90 + +Includes updated chart cortex v0.4.1. + +### cortex-manila v0.0.90 + +Includes updated chart cortex v0.4.1. + +### cortex-crds v0.0.90 + +Includes updated chart cortex v0.4.1. + +### cortex-ironcore v0.0.90 + +Includes updated chart cortex v0.4.1. + +### cortex-pods v0.0.90 + +Includes updated chart cortex v0.4.1. + ## 2026-09-02 — [#1184](https://github.com/cobaltcore-dev/cortex/pull/1184) ### cortex-shim v0.1.16 (sha-b577e306) diff --git a/Makefile b/Makefile index 378f15ff9..220ab131a 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen GOLANGCI_LINT = $(LOCALBIN)/golangci-lint GOTESTSUM = $(LOCALBIN)/gotestsum -CONTROLLER_TOOLS_VERSION ?= v0.21.0 +CONTROLLER_TOOLS_VERSION ?= v0.22.0 GOLANGCI_LINT_VERSION ?= v2.13.2 GOTESTSUM_VERSION ?= v1.13.0 diff --git a/cmd/shim/main.go b/cmd/shim/main.go index 62ab7f64d..e3b7e2d37 100644 --- a/cmd/shim/main.go +++ b/cmd/shim/main.go @@ -403,11 +403,32 @@ func main() { return fmt.Errorf("unable to create manager: %w", err) } - multiclusterClient, err := setupMulticlusterClient(ctx, mgr, restConfig, multiclusterMonitor) + // cycleCtx scopes this manager cycle: cancelling it makes mgr.Start return + // so the supervisor rebuilds the manager (re-reading config and + // reconnecting to the currently-configured remotes). It is cancelled on a + // clean return (defer), when the parent ctx is cancelled (child), or by the + // reachability probe below when a remote apiserver is lost. This is the + // external liveness signal the supervisor lacks on its own: a remote whose + // informer already synced and then disappears is retried by + // controller-runtime forever without mgr.Start ever returning. + cycleCtx, cancelCycle := context.WithCancelCause(ctx) + defer cancelCycle(nil) + + multiclusterClient, err := setupMulticlusterClient(cycleCtx, mgr, restConfig, multiclusterMonitor) if err != nil { return fmt.Errorf("unable to set up multicluster client: %w", err) } + // Probe each remote apiserver for reachability. When a remote is + // sustained-unreachable (e.g. its cluster was deleted after its cache had + // already synced), cancel the cycle so the supervisor rebuilds the manager. + // A remote that answers with any HTTP status is reachable, so an authz or + // server error does not trigger a rebuild storm. + go multiclusterClient.ProbeRemotes(cycleCtx, multicluster.DefaultProbeOptions, func(host string) { + setupLog.Info("remote apiserver sustained-unreachable; recycling manager", "host", host) + cancelCycle(fmt.Errorf("remote apiserver unreachable: %s", host)) + }) + if placementShim != nil { if err := placementShim.SetupControllerWithManager(ctx, mgr, multiclusterClient); err != nil { return fmt.Errorf("unable to set up placement shim controller: %w", err) @@ -431,7 +452,7 @@ func main() { // goroutine only marks ready while the context is still live, and the // deferred clear always wins the final state. var readyMu sync.Mutex - cacheCtx, cancelCacheSync := context.WithCancel(ctx) + cacheCtx, cancelCacheSync := context.WithCancel(cycleCtx) defer func() { cancelCacheSync() readyMu.Lock() @@ -483,7 +504,7 @@ func main() { } setupLog.Info("starting manager") - return mgr.Start(ctx) + return mgr.Start(cycleCtx) } // +kubebuilder:scaffold:builder diff --git a/docs/guides/multicluster/cortex-remote-crb.yaml b/docs/guides/multicluster/cortex-remote-crb.yaml index 66b44ded8..61968e368 100644 --- a/docs/guides/multicluster/cortex-remote-crb.yaml +++ b/docs/guides/multicluster/cortex-remote-crb.yaml @@ -15,6 +15,9 @@ subjects: - kind: User apiGroup: rbac.authorization.k8s.io name: "https://host.docker.internal:8443#system:serviceaccount:default:cortex-pods-controller-manager" +- kind: User + apiGroup: rbac.authorization.k8s.io + name: "https://host.docker.internal:8443#system:serviceaccount:default:cortex-placement-shim" roleRef: kind: ClusterRole name: cluster-admin diff --git a/docs/guides/multicluster/run.sh b/docs/guides/multicluster/run.sh index f68956870..c4062b84d 100755 --- a/docs/guides/multicluster/run.sh +++ b/docs/guides/multicluster/run.sh @@ -78,4 +78,4 @@ kubectl --context kind-cortex-remote-az-b apply \ echo "Starting cortex in home cluster with tilt, using overrides from $TILT_OVERRIDES_PATH" kubectl config use-context kind-cortex-home -export ACTIVE_DEPLOYMENTS="nova" && tilt up +tilt up diff --git a/go.mod b/go.mod index a341fe2df..16f15fb93 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.3 github.com/sapcc/go-api-declarations v1.25.0 - github.com/sapcc/go-bits v0.0.0-20260827091731-7669cbdb53fb + github.com/sapcc/go-bits v0.0.0-20260903192122-1774475e70e3 go.uber.org/zap v1.28.0 go.xyrillian.de/gg v1.14.0 golang.org/x/sync v0.22.0 @@ -24,7 +24,7 @@ require ( k8s.io/api v0.37.0 k8s.io/apimachinery v0.37.0 k8s.io/client-go v0.37.0 - sigs.k8s.io/controller-runtime v0.24.1 + sigs.k8s.io/controller-runtime v0.25.0 ) require ( @@ -65,12 +65,12 @@ require ( github.com/go-openapi/swag/typeutils v0.27.1 // indirect github.com/go-openapi/swag/yamlutils v0.27.1 // indirect github.com/go-sql-driver/mysql v1.10.0 // indirect - github.com/google/cel-go v0.29.0 // indirect + github.com/google/cel-go v0.29.2 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -85,27 +85,27 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/poy/onpar v0.3.5 // indirect - github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/common v0.71.0 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/ziutek/mymysql v1.5.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect - go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect - golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.47.0 // indirect @@ -119,14 +119,14 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gotest.tools v2.2.0+incompatible // indirect - k8s.io/apiextensions-apiserver v0.36.3 // indirect - k8s.io/apiserver v0.36.3 // indirect - k8s.io/component-base v0.36.3 // indirect + k8s.io/apiextensions-apiserver v0.37.0 // indirect + k8s.io/apiserver v0.37.0 // indirect + k8s.io/component-base v0.37.0 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect k8s.io/streaming v0.37.0 // indirect k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect diff --git a/go.sum b/go.sum index 064097ef7..2af74e098 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.29.0 h1:fEG+Ja3YRwNOqnQxTyJwoByAUAvTuxUGiro/jhrm4F4= -github.com/google/cel-go v0.29.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= +github.com/google/cel-go v0.29.2 h1:ZtDxkeiMmz0mxbKDYiNkE5Lk7V5edMRcaaDf2jX002k= +github.com/google/cel-go v0.29.2/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -117,8 +117,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -176,8 +176,8 @@ github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59u github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= -github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= -github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8= +github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk= github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -185,10 +185,10 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sapcc/go-api-declarations v1.25.0 h1:vLkSVV8oaZExoBMEkwX31AqUjZIjwxKkxXr4q2sAAOg= github.com/sapcc/go-api-declarations v1.25.0/go.mod h1:7NrwidCCv/MxwBpb/qqYLbQb/eY6rlnYkXwWMGx/wlI= -github.com/sapcc/go-bits v0.0.0-20260827091731-7669cbdb53fb h1:jORKY0SgUAhXCq40zGSDRaBzDk2zUX+p2wsJeCYxITk= -github.com/sapcc/go-bits v0.0.0-20260827091731-7669cbdb53fb/go.mod h1:X2M3A28UnHSRGlRmiztvmBhdETrplwDFn7QLyxCOYtA= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sapcc/go-bits v0.0.0-20260903192122-1774475e70e3 h1:UJBdl9Nd8Q2mvzWp4ArZDqEdzT6J3EcMFtm6IpCD56A= +github.com/sapcc/go-bits v0.0.0-20260903192122-1774475e70e3/go.mod h1:vkuud4jRPWr/S2dW2409gclx56CjuFbm4N5NXyDRKlI= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -198,23 +198,22 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= @@ -223,8 +222,8 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= -go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -235,12 +234,13 @@ go.xyrillian.de/gg v1.14.0 h1:S19Jk3V1dcF9WdXQi7OGWjboVj8/40I4+/G1lZ6i4TI= go.xyrillian.de/gg v1.14.0/go.mod h1:DoO4fQSWIrBRlNlCjVyrYM0kAEBt/Jg2GkMH+cGRZ0k= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= -golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= -golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= @@ -250,7 +250,6 @@ golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7 golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= @@ -280,23 +279,22 @@ gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnf gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= k8s.io/api v0.37.0 h1:Z//Vj9N7RA/yS2sDmxyeo7h+RR4zbUrd2vrd3Z0TbB4= k8s.io/api v0.37.0/go.mod h1:LKXgcJWMc+f4OLbP5SFR8rulEg07zZhpi/zMULiBImk= -k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0= -k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4= +k8s.io/apiextensions-apiserver v0.37.0 h1:zRMQ3+/LIE5oZ0tVvXwYHC+dIkSP5cjNWju7AZU1LOI= +k8s.io/apiextensions-apiserver v0.37.0/go.mod h1:HU0PfSBwchHL5iDau6jjt9zU6ryWkDDlaVUiq91NK80= k8s.io/apimachinery v0.37.0 h1:Np2AbDtf8x6RDHiD8T9LbKJ9gaegeVNa8yNm5FuGKm0= k8s.io/apimachinery v0.37.0/go.mod h1:RN3nhprFSCxOi5Selxd7oMTXOe/c+ZbcE7Im+TS2zkE= -k8s.io/apiserver v0.36.3 h1:MGSg2SkdfuytiDEcRylT5mQFmmSsbx90XFUO67Y4bsQ= -k8s.io/apiserver v0.36.3/go.mod h1:fVH7zv9EUNUA7Fl7LtDKh8aB9W7u1VQPSGtWV5SjUxg= +k8s.io/apiserver v0.37.0 h1:TXg7OxsOWrAH8J4Zi/gBAZuMw1Dfdd+6cca2h4qjRqo= +k8s.io/apiserver v0.37.0/go.mod h1:OddHDF4gy9qyIb8o/3+qaeP6S0vEObWLgOygVqXksv0= k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ= k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0= -k8s.io/component-base v0.36.3 h1:vc/UFvPCkW0irPz84LAodAL1j3f4xktPM6dDJIEheAY= -k8s.io/component-base v0.36.3/go.mod h1:hZbNFG+gCMl9EbykDGEu73feKP9/Cq6JsV4pTo9GTO8= +k8s.io/component-base v0.37.0 h1:3SdSa4+itMdFTDFTeR8CxKGmSTSMXFlKL4ky8OqjguM= +k8s.io/component-base v0.37.0/go.mod h1:LjOebp4R9y6LODWZQv102ZQxGheLcDO2ZJLAw6bbh4I= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= @@ -305,10 +303,10 @@ k8s.io/streaming v0.37.0 h1:iPBUZLZiKt5bV+lxJurASMOV07VuBhNpiwJt2//AWrM= k8s.io/streaming v0.37.0/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs= k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= -sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 h1:/YpDJ4vReG7ZmzSpBGxduXgywWkJU9zHubgJG03MT+Y= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0/go.mod h1:tJo1aepTXyR+8Xs3sUsGBDk4Ub2AM5dPAPKJx0mpm5c= +sigs.k8s.io/controller-runtime v0.25.0 h1:44KgRUPew331KSJpNu8zJow3iTR5W0p/SfrHdw3lV40= +sigs.k8s.io/controller-runtime v0.25.0/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index bd7b362fd..3e38986a9 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,7 +5,7 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.89 +version: 0.0.90 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres @@ -16,12 +16,12 @@ dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.4.0 + version: 0.4.1 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.4.0 + version: 0.4.1 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index 5a4259d43..387f7f3c6 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.89 +version: 0.0.90 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.4.0 + version: 0.4.1 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index 9897d7257..8d6b7c3d4 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.89 +version: 0.0.90 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.4.0 + version: 0.4.1 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index 12b1e3b07..122928b1a 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,7 +5,7 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.89 +version: 0.0.90 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres @@ -16,12 +16,12 @@ dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.4.0 + version: 0.4.1 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.4.0 + version: 0.4.1 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index aeee5b049..20627b22c 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,7 +5,7 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.89 +version: 0.0.90 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres @@ -16,12 +16,12 @@ dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.4.0 + version: 0.4.1 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.4.0 + version: 0.4.1 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index 0418984e7..545915da1 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.16 +version: 0.1.17 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.16 + version: 0.1.17 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-placement-shim/templates/alerts.yaml b/helm/bundles/cortex-placement-shim/templates/alerts.yaml index ff5ac691b..f12c8e616 100644 --- a/helm/bundles/cortex-placement-shim/templates/alerts.yaml +++ b/helm/bundles/cortex-placement-shim/templates/alerts.yaml @@ -242,4 +242,31 @@ spec: resource router is mapping the same object to multiple clusters, or an object was created out-of-band on the wrong cluster. Investigate the affected resources and the routing configuration. + + - alert: CortexPlacementShimRemoteApiserverUnreachable + # min by (host): a remote is unreachable only if it was 0 for the whole + # window (a rebuild flips it 1->0->1, and min ignores the brief 1s during a + # reconnect attempt). Fires per host so the offending remote is named. + expr: | + min by (host) (cortex_multicluster_remote_apiserver_reachable{service="cortex-placement-shim-metrics-service"}) < 1 + for: 5m + labels: + context: multicluster + dashboard: cortex-placement-shim-status-dashboard/cortex-placement-shim-status-dashboard + service: cortex + severity: warning + support_group: workload-management + annotations: + summary: "Remote apiserver `{{ "{{" }} $labels.host {{ "}}" }}` is unreachable" + description: > + The multicluster client cannot reach the remote apiserver at + `{{ "{{" }} $labels.host {{ "}}" }}`. The reachability probe has seen + sustained transport failures (connection refused, TLS/handshake failure, + or timeout), which is the signature of a deleted or network-partitioned + cluster rather than an authz/server error. The shim recycles its + controller-manager in response, so the manager cache for resources served + by that remote is stale or empty and `cortex_placement_shim_manager_up` + will average low (see CortexPlacementShimManagerLooping). Passthrough + requests to upstream Placement are unaffected. Investigate whether the + remote cluster still exists and network connectivity to its apiserver. {{- end }} diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index e496f3387..6375357ca 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.89 +version: 0.0.90 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.4.0 + version: 0.4.1 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index e5f93df9e..fe518c336 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 88.6.2 + version: 89.2.0 diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index e7b4e4acb..860b5d022 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.16 -appVersion: "sha-b577e306" +version: 0.1.17 +appVersion: "sha-cd38777a" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index b49790ae6..7ccbf72ea 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.4.0 -appVersion: "sha-91df27a0" +version: 0.4.1 +appVersion: "sha-cd38777a" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml b/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml index 5901dddb6..568e5e02f 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: committedresources.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_datasources.yaml b/helm/library/cortex/files/crds/cortex.cloud_datasources.yaml index 9a2d32bbc..e949d0b9e 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_datasources.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_datasources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: datasources.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml b/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml index 1d7c38ea1..2d80182e0 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: decisions.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_deschedulings.yaml b/helm/library/cortex/files/crds/cortex.cloud_deschedulings.yaml index b01e8d2f4..db0807f5a 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_deschedulings.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_deschedulings.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: deschedulings.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml index 4102ea447..171451f99 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: flavorgroupcapacities.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_histories.yaml b/helm/library/cortex/files/crds/cortex.cloud_histories.yaml index 693f776e5..3818ecdbf 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_histories.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_histories.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: histories.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_knowledges.yaml b/helm/library/cortex/files/crds/cortex.cloud_knowledges.yaml index 5a4cba037..a42984003 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_knowledges.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_knowledges.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: knowledges.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_kpis.yaml b/helm/library/cortex/files/crds/cortex.cloud_kpis.yaml index b7637fa24..68ec754d4 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_kpis.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_kpis.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: kpis.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml b/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml index 702703f6c..0262ea445 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: pipelines.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml b/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml index 7309f8ad1..c48ae2d63 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: projectquotas.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml index 85995dc62..4cd6101e6 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.21.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: reservations.cortex.cloud spec: group: cortex.cloud diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index 92cab3430..641b2b0f9 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -89,7 +89,7 @@ func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf Client gvksByConfStr[formatted] = gvk } for gvkStr := range gvksByConfStr { - log.Info("scheme gvk registered", "gvk", gvkStr) + log.V(1).Info("scheme gvk registered", "gvk", gvkStr) } // Parse home GVKs. c.homeGVKs = make(map[schema.GroupVersionKind]bool) diff --git a/pkg/multicluster/client_probe.go b/pkg/multicluster/client_probe.go new file mode 100644 index 000000000..854ddd887 --- /dev/null +++ b/pkg/multicluster/client_probe.go @@ -0,0 +1,220 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package multicluster + +import ( + "context" + "errors" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cluster" +) + +// RemoteEndpoint identifies a unique remote cluster to probe for reachability. +type RemoteEndpoint struct { + // Host is the remote apiserver URL (from the cluster's rest.Config). + Host string + // Cluster is the controller-runtime cluster whose apiserver is probed. + Cluster cluster.Cluster +} + +// UniqueRemotes returns each distinct remote cluster exactly once, across all +// configured GVKs. The same remote apiserver is stored once per GVK it serves, +// so this dedupes by cluster identity (the same policy IndexField uses to dedupe +// caches). The home cluster is not included. +func (c *Client) UniqueRemotes() []RemoteEndpoint { + c.remoteClustersMu.RLock() + defer c.remoteClustersMu.RUnlock() + seen := make(map[cluster.Cluster]bool) + var out []RemoteEndpoint + for _, remotes := range c.remoteClusters { + for _, r := range remotes { + if r.cluster == nil || seen[r.cluster] { + continue + } + seen[r.cluster] = true + out = append(out, RemoteEndpoint{Host: r.cluster.GetConfig().Host, Cluster: r.cluster}) + } + } + return out +} + +// ProbeOptions configures the per-remote reachability probe. +type ProbeOptions struct { + // Interval is the time between reachability probes for each remote. + Interval time.Duration + // Timeout bounds a single probe request so it never blocks on a dead socket + // longer than intended. + Timeout time.Duration + // FailureThreshold is the number of consecutive unreachable probes that must + // occur before a remote is considered lost and onLost is called. It is chosen + // above the supervisor's backoff floor so a doomed manager cycle outlives the + // growing backoff, keeping the rebuild loop period bounded rather than hot. + FailureThreshold int +} + +// DefaultProbeOptions probes every 10s with a 5s per-probe timeout and treats a +// remote as lost after 3 consecutive failures (~30s of sustained unreachability). +var DefaultProbeOptions = ProbeOptions{ + Interval: 10 * time.Second, + Timeout: 5 * time.Second, + FailureThreshold: 3, +} + +// ProbeRemotes runs one reachability probe goroutine per unique remote apiserver +// until ctx is cancelled. Each goroutine periodically probes its remote, updates +// the reachability gauge on the Monitor (if any), and tracks consecutive +// failures; when a remote has been unreachable FailureThreshold times in a row it +// calls onLost(host) once and stops probing that remote (the caller is expected +// to tear down the manager cycle in response). +// +// A single transient blip does not trigger onLost: the failure counter resets on +// the first reachable probe. "Unreachable" means a transport-level failure +// (connection refused, no route, TLS/handshake failure, timeout) — an apiserver +// that responds with any HTTP status (including 401/403/5xx) is considered +// reachable, since a manager rebuild cannot fix an authz/server error and would +// only cause a restart storm. +// +// ProbeRemotes returns immediately when no remotes are configured. +func (c *Client) ProbeRemotes(ctx context.Context, opts ProbeOptions, onLost func(host string)) { + log := ctrl.LoggerFrom(ctx) + remotes := c.UniqueRemotes() + hosts := make([]string, 0, len(remotes)) + for _, r := range remotes { + hosts = append(hosts, r.Host) + } + if len(remotes) == 0 { + // No remotes to watch: log it explicitly so an operator who deletes a + // cluster and sees "nothing happening" can tell this apart from the probe + // simply not finding the remote it expected to watch. + log.Info("reachability probe: no remote apiservers configured, nothing to probe") + return + } + log.Info("reachability probe: starting", "remoteCount", len(remotes), "hosts", hosts, + "interval", opts.Interval.String(), "timeout", opts.Timeout.String(), "failureThreshold", opts.FailureThreshold) + var wg wait.Group + for _, remote := range remotes { + httpClient, err := reachabilityClient(remote.Cluster.GetConfig(), opts.Timeout) + if err != nil { + // A remote we cannot even build a probe client for is treated as + // lost immediately: its config is unusable, so a rebuild (which + // re-reads config) is the right response. + log.Error(err, "reachability probe: unable to build probe client for remote; declaring it lost", "host", remote.Host) + onLost(remote.Host) + continue + } + wg.StartWithContext(ctx, func(ctx context.Context) { + c.probeRemoteLoop(ctx, opts, remote, httpClient, onLost) + }) + } + wg.Wait() + log.Info("reachability probe: stopped", "hosts", hosts) +} + +// probeRemoteLoop probes a single remote until ctx is cancelled or the remote is +// declared lost. +func (c *Client) probeRemoteLoop(ctx context.Context, opts ProbeOptions, remote RemoteEndpoint, httpClient *rest.RESTClient, onLost func(host string)) { + log := ctrl.LoggerFrom(ctx).WithValues("host", remote.Host) + log.Info("reachability probe: watching remote apiserver") + failures := 0 + wasReachable := true // assume reachable at start; log the first real transition + // PollUntilContextCancel with immediate=true runs the first probe right away + // rather than waiting a full interval, so a remote that is already gone at + // cycle start is detected promptly. The poll func never returns an error, so + // the only error PollUntilContextCancel can return is the context error once + // ctx is cancelled or the loop stops after declaring the remote lost — both + // expected, so it is intentionally not surfaced. + err := wait.PollUntilContextCancel(ctx, opts.Interval, true, func(ctx context.Context) (bool, error) { + reachable, probeErr := probeReachable(ctx, httpClient, opts.Timeout) + if c.Monitor != nil { + c.Monitor.recordRemoteReachable(remote.Host, reachable) + } + // Per-probe outcome at V(1): verbose, but the single most useful line when + // diagnosing "why didn't it detect the deletion" — it shows the classified + // result and the underlying transport/HTTP error for each tick. + if probeErr != nil { + log.V(1).Info("reachability probe: tick", "reachable", reachable, "error", probeErr.Error()) + } else { + log.V(1).Info("reachability probe: tick", "reachable", reachable) + } + if reachable { + if !wasReachable { + log.Info("reachability probe: remote apiserver recovered", "afterConsecutiveFailures", failures) + } + wasReachable = true + failures = 0 + return false, nil + } + wasReachable = false + failures++ + if probeErr != nil { + log.Info("reachability probe: remote apiserver unreachable", + "consecutiveFailures", failures, "threshold", opts.FailureThreshold, "error", probeErr.Error()) + } else { + log.Info("reachability probe: remote apiserver unreachable", + "consecutiveFailures", failures, "threshold", opts.FailureThreshold) + } + if failures >= opts.FailureThreshold { + log.Info("reachability probe: failure threshold reached, declaring remote lost", "consecutiveFailures", failures) + onLost(remote.Host) + return true, nil + } + return false, nil + }) + if err != nil && ctx.Err() == nil { + // A non-context error is not expected here (the poll func never returns + // one), but log it rather than swallow it if the invariant ever changes. + log.Error(err, "remote reachability poll stopped unexpectedly") + } +} + +// reachabilityClient builds a REST client for the given remote config that is +// used only to probe reachability. It copies the config (so the caller's shared +// config is never mutated), sets a per-probe timeout, and supplies a codec so the +// unversioned REST client can be constructed (we only read the transport outcome, +// never decode a body, but rest requires a NegotiatedSerializer). +func reachabilityClient(cfg *rest.Config, timeout time.Duration) (*rest.RESTClient, error) { + cfgCopy := *cfg + cfgCopy.Timeout = timeout + cfgCopy.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + if cfgCopy.GroupVersion == nil { + cfgCopy.GroupVersion = &schema.GroupVersion{} + } + httpClient, err := rest.HTTPClientFor(&cfgCopy) + if err != nil { + return nil, err + } + return rest.UnversionedRESTClientForConfigAndClient(&cfgCopy, httpClient) +} + +// probeReachable issues a lightweight GET /readyz against the remote apiserver +// and reports whether the apiserver is reachable. Any HTTP response — including +// non-2xx statuses surfaced as a Kubernetes StatusError — counts as reachable; +// only a transport-level error (connection refused, no route, TLS failure, +// timeout) counts as unreachable, which is the signature of a deleted cluster. +// probeReachable issues a lightweight GET /readyz against the remote apiserver +// and reports whether the apiserver is reachable, along with the underlying +// error (nil on a 2xx, otherwise the transport or HTTP error even when the result +// is classified reachable) for logging. Any HTTP response — including non-2xx +// statuses surfaced as a Kubernetes StatusError — counts as reachable; only a +// transport-level error (connection refused, no route, TLS failure, timeout) +// counts as unreachable, which is the signature of a deleted cluster. +func probeReachable(ctx context.Context, client *rest.RESTClient, timeout time.Duration) (bool, error) { + probeCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + err := client.Get().AbsPath("/readyz").Do(probeCtx).Error() + if err == nil { + return true, nil + } + // The apiserver answered with a non-2xx status: it is up and reachable. Return + // the error too so callers can log the status the apiserver returned. + var statusErr *apierrors.StatusError + return errors.As(err, &statusErr), err +} diff --git a/pkg/multicluster/client_probe_test.go b/pkg/multicluster/client_probe_test.go new file mode 100644 index 000000000..6bbe34f01 --- /dev/null +++ b/pkg/multicluster/client_probe_test.go @@ -0,0 +1,243 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package multicluster + +import ( + "context" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" +) + +// fakeRemote builds a fakeCluster whose rest config points at the given host. +func fakeRemote(t *testing.T, host string) *fakeCluster { + t.Helper() + scheme := newTestScheme(t) + c := newFakeCluster(scheme) + c.restConfig = &rest.Config{Host: host} + return c +} + +func TestUniqueRemotes_DedupesAcrossGVKs(t *testing.T) { + gvkA := schema.GroupVersionKind{Group: "g", Version: "v", Kind: "A"} + gvkB := schema.GroupVersionKind{Group: "g", Version: "v", Kind: "B"} + shared := fakeRemote(t, "https://shared") + other := fakeRemote(t, "https://other") + c := &Client{ + remoteClusters: map[schema.GroupVersionKind][]remoteCluster{ + gvkA: {{cluster: shared}, {cluster: other}}, + gvkB: {{cluster: shared}}, + }, + } + remotes := c.UniqueRemotes() + if len(remotes) != 2 { + t.Fatalf("expected 2 unique remotes, got %d", len(remotes)) + } + hosts := map[string]bool{} + for _, r := range remotes { + hosts[r.Host] = true + } + if !hosts["https://shared"] || !hosts["https://other"] { + t.Errorf("unexpected hosts: %v", hosts) + } +} + +func TestUniqueRemotes_NoRemotes(t *testing.T) { + c := &Client{} + if got := c.UniqueRemotes(); len(got) != 0 { + t.Errorf("expected no remotes, got %d", len(got)) + } +} + +func TestProbeReachable_Classification(t *testing.T) { + tests := []struct { + name string + status int // 0 means "close the connection / unreachable" + wantReachable bool + }{ + {"ok", http.StatusOK, true}, + {"forbidden apiserver is up", http.StatusForbidden, true}, + {"unauthorized apiserver is up", http.StatusUnauthorized, true}, + {"server error apiserver is up", http.StatusInternalServerError, true}, + {"connection refused is unreachable", 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var host string + if tt.status == 0 { + // Reserve a port and close the listener so connections are refused. + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + host = srv.URL + srv.Close() + } else { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tt.status) + })) + t.Cleanup(srv.Close) + host = srv.URL + } + client, err := reachabilityClient(&rest.Config{Host: host, TLSClientConfig: rest.TLSClientConfig{Insecure: true}}, time.Second) + if err != nil { + t.Fatalf("reachabilityClient: %v", err) + } + got, probeErr := probeReachable(context.Background(), client, time.Second) + if got != tt.wantReachable { + t.Errorf("probeReachable = %v (err %v), want %v", got, probeErr, tt.wantReachable) + } + // A 200 yields no error; any other outcome (reachable non-2xx or an + // unreachable transport failure) surfaces the underlying error. + if tt.status == http.StatusOK && probeErr != nil { + t.Errorf("expected nil error on 200, got %v", probeErr) + } + if tt.status != http.StatusOK && probeErr == nil { + t.Errorf("expected a non-nil error for status %d / unreachable, got nil", tt.status) + } + }) + } +} + +func TestProbeRemotes_TriggersOnSustainedFailure(t *testing.T) { + // A remote pointing at a closed listener is always unreachable. + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + deadHost := srv.URL + srv.Close() + + remote := fakeRemote(t, deadHost) + mon := NewMonitor("test_trigger_") + c := &Client{ + Monitor: mon, + remoteClusters: map[schema.GroupVersionKind][]remoteCluster{{Kind: "A"}: {{cluster: remote}}}, + } + + var mu sync.Mutex + var lostHosts []string + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { + c.ProbeRemotes(ctx, ProbeOptions{Interval: 5 * time.Millisecond, Timeout: 50 * time.Millisecond, FailureThreshold: 3}, func(host string) { + mu.Lock() + lostHosts = append(lostHosts, host) + mu.Unlock() + }) + close(done) + }() + + // The probe loop stops itself once the remote is declared lost. + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("ProbeRemotes did not return after remote was declared lost") + } + + mu.Lock() + if len(lostHosts) != 1 || lostHosts[0] != deadHost { + t.Errorf("expected onLost called once with %q, got %v", deadHost, lostHosts) + } + mu.Unlock() + + if got := testutil.ToFloat64(mon.(*monitor).remoteReachable.WithLabelValues(deadHost)); got != 0 { + t.Errorf("expected reachable gauge 0 for dead host, got %v", got) + } +} + +func TestProbeRemotes_NoTriggerWhenReachable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + + remote := fakeRemote(t, srv.URL) + remote.restConfig.TLSClientConfig = rest.TLSClientConfig{Insecure: true} + mon := NewMonitor("test_reachable_") + c := &Client{ + Monitor: mon, + remoteClusters: map[schema.GroupVersionKind][]remoteCluster{{Kind: "A"}: {{cluster: remote}}}, + } + + var mu sync.Mutex + lost := false + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go c.ProbeRemotes(ctx, ProbeOptions{Interval: 5 * time.Millisecond, Timeout: time.Second, FailureThreshold: 3}, func(string) { + mu.Lock() + lost = true + mu.Unlock() + }) + + // Poll for the gauge to reach 1: a reachable remote must record 1 and never + // trip onLost. Polling (rather than a fixed sleep + single read) avoids racing + // the probe goroutine's gauge write. + gauge := mon.(*monitor).remoteReachable.WithLabelValues(srv.URL) + reachableSeen := false + for range 200 { + if testutil.ToFloat64(gauge) == 1 { + reachableSeen = true + break + } + time.Sleep(10 * time.Millisecond) + } + cancel() + + mu.Lock() + defer mu.Unlock() + if lost { + t.Error("onLost was called for a reachable remote") + } + if !reachableSeen { + t.Errorf("expected reachable gauge to reach 1, last value %v", testutil.ToFloat64(gauge)) + } +} + +func TestProbeRemotes_NoRemotesIsNoop(t *testing.T) { + c := &Client{} + lost := false + // Should return immediately without ever calling onLost. + c.ProbeRemotes(context.Background(), DefaultProbeOptions, func(string) { lost = true }) + if lost { + t.Error("onLost called with no remotes configured") + } +} + +func TestProbeRemotes_TransientBlipResetsCounter(t *testing.T) { + // probeRemoteLoop's counter resets on any reachable probe, so a run of + // failures shorter than the threshold never trips onLost. Drive the counter + // logic directly: a probe func that fails twice (below threshold 3) then + // recovers must leave failures at 0 and never call onLost. + failures := 0 + onLost := false + reachSeq := []bool{false, false, true, true} + i := 0 + step := func() bool { + reachable := reachSeq[i] + i++ + if reachable { + failures = 0 + return false + } + failures++ + if failures >= 3 { + onLost = true + return true + } + return false + } + for i < len(reachSeq) { + if step() { + break + } + } + if onLost { + t.Error("onLost tripped despite failures never reaching the threshold") + } + if failures != 0 { + t.Errorf("expected failure counter reset to 0 after recovery, got %d", failures) + } +} diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index 29ea8e0ca..d3159c00d 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -16,7 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/tools/events" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/recorder" "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) @@ -81,14 +82,26 @@ type fakeCluster struct { cluster.Cluster fakeClient client.Client fakeCache *fakeCache - fakeRecorder events.EventRecorder + fakeRecorder recorder.EventRecorder scheme *runtime.Scheme + restConfig *rest.Config } func (f *fakeCluster) GetClient() client.Client { return f.fakeClient } +// GetConfig returns the rest config the cluster was built with, defaulting to a +// placeholder host when unset. Production clusters always have one (used for +// per-cluster logging and reachability probing); the default keeps tests that +// don't care about the host from panicking on the embedded nil interface. +func (f *fakeCluster) GetConfig() *rest.Config { + if f.restConfig != nil { + return f.restConfig + } + return &rest.Config{Host: "https://fake-cluster"} +} + func (f *fakeCluster) GetScheme() *runtime.Scheme { return f.scheme } @@ -104,7 +117,7 @@ func (f *fakeCluster) GetFieldIndexer() client.FieldIndexer { return f.fakeCache } -func (f *fakeCluster) GetEventRecorder(_ string) events.EventRecorder { +func (f *fakeCluster) GetEventRecorder(_ string) recorder.EventRecorder { if f.fakeRecorder != nil { return f.fakeRecorder } diff --git a/pkg/multicluster/monitor.go b/pkg/multicluster/monitor.go index a33cfffb8..6cf3883b5 100644 --- a/pkg/multicluster/monitor.go +++ b/pkg/multicluster/monitor.go @@ -23,6 +23,11 @@ type Monitor interface { // detected on more than one cluster serving the GVK, labeled by the method // of access and the resource GVK. recordCrossClusterNameConflict(method string, gvk schema.GroupVersionKind) + + // recordRemoteReachable records whether a remote apiserver was reachable at + // the last reachability probe, labeled by host. See pkg/multicluster's + // ProbeRemotes for how reachability is determined. + recordRemoteReachable(host string, reachable bool) } // monitor is the default Prometheus-backed Monitor implementation. @@ -31,6 +36,10 @@ type monitor struct { // detected on more than one cluster serving the GVK, labeled by the method // of access and the resource GVK. crossClusterNameConflicts *prometheus.CounterVec + // remoteReachable reports whether each remote apiserver was reachable at the + // last reachability probe, labeled by host. It lives on the process-lifetime + // Monitor (not the per-cycle Client) so it survives manager rebuilds. + remoteReachable *prometheus.GaugeVec } // NewMonitor creates a new Prometheus-backed multicluster client monitor. The @@ -42,6 +51,10 @@ func NewMonitor(prefix string) Monitor { Name: prefix + "multicluster_cross_cluster_name_conflicts_total", Help: "Total number of times the same resource name was detected on more than one cluster serving the same GVK", }, duplicateConflictLabels), + remoteReachable: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: prefix + "multicluster_remote_apiserver_reachable", + Help: "1 if the remote apiserver was reachable at the last probe, 0 if sustained-unreachable, labeled by host", + }, []string{"host"}), } } @@ -51,12 +64,24 @@ func (m *monitor) recordCrossClusterNameConflict(method string, gvk schema.Group m.crossClusterNameConflicts.WithLabelValues(method, gvk.String()).Inc() } +// recordRemoteReachable sets the reachability gauge for the given host to 1 +// (reachable) or 0 (unreachable). +func (m *monitor) recordRemoteReachable(host string, reachable bool) { + v := 0.0 + if reachable { + v = 1.0 + } + m.remoteReachable.WithLabelValues(host).Set(v) +} + // Describe implements prometheus.Collector. func (m *monitor) Describe(ch chan<- *prometheus.Desc) { m.crossClusterNameConflicts.Describe(ch) + m.remoteReachable.Describe(ch) } // Collect implements prometheus.Collector. func (m *monitor) Collect(ch chan<- prometheus.Metric) { m.crossClusterNameConflicts.Collect(ch) + m.remoteReachable.Collect(ch) } diff --git a/pkg/multicluster/monitor_test.go b/pkg/multicluster/monitor_test.go index 104751280..0660595e6 100644 --- a/pkg/multicluster/monitor_test.go +++ b/pkg/multicluster/monitor_test.go @@ -91,3 +91,24 @@ func TestMonitor_RecordCrossClusterNameConflict(t *testing.T) { t.Errorf("list/%s: got %v, want 0", gvk, got) } } + +func TestMonitor_RecordRemoteReachable(t *testing.T) { + m := NewMonitor("cortex_").(*monitor) + + // reachable=true sets the gauge to 1, reachable=false to 0, and the latest + // value for a host wins. + m.recordRemoteReachable("https://a", true) + m.recordRemoteReachable("https://b", false) + if got := testutil.ToFloat64(m.remoteReachable.WithLabelValues("https://a")); got != 1 { + t.Errorf("host a: got %v, want 1", got) + } + if got := testutil.ToFloat64(m.remoteReachable.WithLabelValues("https://b")); got != 0 { + t.Errorf("host b: got %v, want 0", got) + } + + // A host flipping from reachable to unreachable overwrites the prior value. + m.recordRemoteReachable("https://a", false) + if got := testutil.ToFloat64(m.remoteReachable.WithLabelValues("https://a")); got != 0 { + t.Errorf("host a after loss: got %v, want 0", got) + } +} diff --git a/pkg/multicluster/recorder.go b/pkg/multicluster/recorder.go index 8c7c78d3d..f7099dcd3 100644 --- a/pkg/multicluster/recorder.go +++ b/pkg/multicluster/recorder.go @@ -5,18 +5,18 @@ package multicluster import ( "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/recorder" ) -// MultiClusterRecorder implements events.EventRecorder and routes events to the -// correct cluster based on the GVK of the "regarding" object. It uses the same -// routing logic as the multicluster Client's write path. +// MultiClusterRecorder implements recorder.EventRecorder and routes events to +// the correct cluster based on the GVK of the "regarding" object. It uses the +// same routing logic as the multicluster Client's write path. type MultiClusterRecorder struct { client *Client - homeRecorder events.EventRecorder - recorders map[cluster.Cluster]events.EventRecorder + homeRecorder recorder.EventRecorder + recorders map[cluster.Cluster]recorder.EventRecorder } // GetEventRecorder creates a multi-cluster-aware EventRecorder. It pre-creates @@ -24,10 +24,10 @@ type MultiClusterRecorder struct { // registered in the client. The name parameter is passed through to each // cluster's GetEventRecorder method (it becomes the reportingController in the // Kubernetes Event). -func (c *Client) GetEventRecorder(name string) events.EventRecorder { +func (c *Client) GetEventRecorder(name string) recorder.EventRecorder { homeRecorder := c.HomeCluster.GetEventRecorder(name) - recorders := make(map[cluster.Cluster]events.EventRecorder) + recorders := make(map[cluster.Cluster]recorder.EventRecorder) recorders[c.HomeCluster] = homeRecorder c.remoteClustersMu.RLock() @@ -55,8 +55,15 @@ func (r *MultiClusterRecorder) Eventf(regarding, related runtime.Object, eventty recorder.Eventf(regarding, related, eventtype, reason, action, note, args...) } +// AnnotatedEventf routes the annotated event to the cluster that owns the +// "regarding" object. Falls back to the home cluster recorder if routing fails. +func (r *MultiClusterRecorder) AnnotatedEventf(regarding, related runtime.Object, annotations map[string]string, eventtype, reason, action, note string, args ...any) { + recorder := r.recorderFor(regarding) + recorder.AnnotatedEventf(regarding, related, annotations, eventtype, reason, action, note, args...) +} + // recorderFor resolves which per-cluster recorder to use for the given object. -func (r *MultiClusterRecorder) recorderFor(obj runtime.Object) events.EventRecorder { +func (r *MultiClusterRecorder) recorderFor(obj runtime.Object) recorder.EventRecorder { if obj == nil { return r.homeRecorder } diff --git a/pkg/multicluster/recorder_test.go b/pkg/multicluster/recorder_test.go index 90bdd71dc..16318747c 100644 --- a/pkg/multicluster/recorder_test.go +++ b/pkg/multicluster/recorder_test.go @@ -41,6 +41,18 @@ func (f *fakeEventRecorder) Eventf(regarding, _ runtime.Object, eventtype, reaso }) } +func (f *fakeEventRecorder) AnnotatedEventf(regarding, _ runtime.Object, _ map[string]string, eventtype, reason, action, note string, args ...any) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, eventfCall{ + regarding: regarding, + eventtype: eventtype, + reason: reason, + action: action, + note: fmt.Sprintf(note, args...), + }) +} + func (f *fakeEventRecorder) getCalls() []eventfCall { f.mu.Lock() defer f.mu.Unlock() diff --git a/pkg/shim/supervisor/supervisor_test.go b/pkg/shim/supervisor/supervisor_test.go index 8d86b2b5a..258410ee2 100644 --- a/pkg/shim/supervisor/supervisor_test.go +++ b/pkg/shim/supervisor/supervisor_test.go @@ -285,6 +285,35 @@ func TestBackoffProgressionAndReset(t *testing.T) { } } +// TestManagerRestartsOnInternalCycleCancel locks down the exact mechanism the +// placement shim relies on: BuildAndStart derives a per-cycle child context from +// the one the supervisor passes in and cancels it itself (as the shim does when a +// remote apiserver goes unreachable), then returns. The parent context is still +// live, so the supervisor must treat this as a manager exit and rebuild — not +// mistake it for a graceful shutdown and stop the loop. +func TestManagerRestartsOnInternalCycleCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var cycles atomic.Int32 + + o := baseOptions(t) + o.Backoff = wait.Backoff{Duration: time.Millisecond, Factor: 1.0, Steps: 100} + o.BuildAndStart = func(parent context.Context) error { + cycles.Add(1) + // Mirror main.go's buildAndStart: derive a per-cycle context and cancel + // it from within (as the reachability probe does), then return its error. + cycleCtx, cancelCycle := context.WithCancel(parent) + cancelCycle() + return cycleCtx.Err() + } + runAsync(t, ctx, o) + + // A cycle that self-cancels and returns while the parent is live must be + // rebuilt, not end the supervision loop. + waitFor(t, func() bool { return cycles.Load() >= 3 }, "manager to be rebuilt after an internal per-cycle cancel") +} + // waitFor polls cond up to ~2s, failing the test if it never becomes true. func waitFor(t *testing.T, cond func() bool, what string) { t.Helper()