Release cortex v0.4.1, cortex-shim v0.1.17

CHANGELOG.md

@@ -1,5 +1,57 @@
 # Changelog
 
+## 2026-09-07 — [#1195](https://github.com/cobaltcore-dev/cortex/pull/1195)
+
+### cortex-shim v0.1.17 (sha-cd38777a)
+
+New features:
+- Remote apiserver liveness probe — adds a per-remote reachability probe (`GET /readyz`) that detects when a remote apiserver becomes unreachable after its informer cache has synced; after a configurable failure threshold (default 3 × 10s), the probe cancels the manager cycle context so the existing self-healing supervisor rebuilds the manager with current config ([#1190](https://github.com/cobaltcore-dev/cortex/pull/1190))
+- `cortex_multicluster_remote_apiserver_reachable{host}` gauge — exposes per-remote apiserver reachability on the process-lifetime monitor, surviving manager rebuilds ([#1190](https://github.com/cobaltcore-dev/cortex/pull/1190))
+- `CortexPlacementShimRemoteApiserverUnreachable` alert — fires when a remote apiserver is unreachable ([#1190](https://github.com/cobaltcore-dev/cortex/pull/1190))
+
+Non-breaking changes:
+- Fix multicluster recorder for controller-runtime v0.25.0 `EventRecorder` interface change — adds `AnnotatedEventf` method to `MultiClusterRecorder` to satisfy the wider `recorder.EventRecorder` interface ([#1191](https://github.com/cobaltcore-dev/cortex/pull/1191))
+- Update `sigs.k8s.io/controller-runtime` to v0.25.0 ([#1188](https://github.com/cobaltcore-dev/cortex/pull/1188))
+- Update `sigs.k8s.io/controller-tools` to v0.22.0 (CRD annotation bump) ([#1186](https://github.com/cobaltcore-dev/cortex/pull/1186))
+- Update `github.com/sapcc/go-bits` ([#1187](https://github.com/cobaltcore-dev/cortex/pull/1187))
+- Update `kube-prometheus-stack` to v89 ([#1189](https://github.com/cobaltcore-dev/cortex/pull/1189))
+
+### cortex v0.4.1 (sha-cd38777a)
+
+Non-breaking changes:
+- Fix multicluster recorder for controller-runtime v0.25.0 `EventRecorder` interface change ([#1191](https://github.com/cobaltcore-dev/cortex/pull/1191))
+- CRDs regenerated with controller-tools v0.22.0 ([#1186](https://github.com/cobaltcore-dev/cortex/pull/1186))
+- Update `sigs.k8s.io/controller-runtime` to v0.25.0 ([#1188](https://github.com/cobaltcore-dev/cortex/pull/1188))
+- Update `github.com/sapcc/go-bits` ([#1187](https://github.com/cobaltcore-dev/cortex/pull/1187))
+
+### cortex-placement-shim v0.1.17
+
+Includes updated chart cortex-shim v0.1.17.
+
+### cortex-nova v0.0.90
+
+Includes updated chart cortex v0.4.1.
+
+### cortex-cinder v0.0.90
+
+Includes updated chart cortex v0.4.1.
+
+### cortex-manila v0.0.90
+
+Includes updated chart cortex v0.4.1.
+
+### cortex-crds v0.0.90
+
+Includes updated chart cortex v0.4.1.
+
+### cortex-ironcore v0.0.90
+
+Includes updated chart cortex v0.4.1.
+
+### cortex-pods v0.0.90
+
+Includes updated chart cortex v0.4.1.
+
 ## 2026-09-02 — [#1184](https://github.com/cobaltcore-dev/cortex/pull/1184)
 
 ### cortex-shim v0.1.16 (sha-b577e306)

Makefile

@@ -58,7 +58,7 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
 GOTESTSUM = $(LOCALBIN)/gotestsum
 
-CONTROLLER_TOOLS_VERSION ?= v0.21.0
+CONTROLLER_TOOLS_VERSION ?= v0.22.0
 GOLANGCI_LINT_VERSION ?= v2.13.2
 GOTESTSUM_VERSION ?= v1.13.0
 

cmd/shim/main.go

@@ -403,11 +403,32 @@ func main() {
 			return fmt.Errorf("unable to create manager: %w", err)
 		}
 
-		multiclusterClient, err := setupMulticlusterClient(ctx, mgr, restConfig, multiclusterMonitor)
+		// cycleCtx scopes this manager cycle: cancelling it makes mgr.Start return
+		// so the supervisor rebuilds the manager (re-reading config and
+		// reconnecting to the currently-configured remotes). It is cancelled on a
+		// clean return (defer), when the parent ctx is cancelled (child), or by the
+		// reachability probe below when a remote apiserver is lost. This is the
+		// external liveness signal the supervisor lacks on its own: a remote whose
+		// informer already synced and then disappears is retried by
+		// controller-runtime forever without mgr.Start ever returning.
+		cycleCtx, cancelCycle := context.WithCancelCause(ctx)
+		defer cancelCycle(nil)
+
+		multiclusterClient, err := setupMulticlusterClient(cycleCtx, mgr, restConfig, multiclusterMonitor)
 		if err != nil {
 			return fmt.Errorf("unable to set up multicluster client: %w", err)
 		}
 
+		// Probe each remote apiserver for reachability. When a remote is
+		// sustained-unreachable (e.g. its cluster was deleted after its cache had
+		// already synced), cancel the cycle so the supervisor rebuilds the manager.
+		// A remote that answers with any HTTP status is reachable, so an authz or
+		// server error does not trigger a rebuild storm.
+		go multiclusterClient.ProbeRemotes(cycleCtx, multicluster.DefaultProbeOptions, func(host string) {
+			setupLog.Info("remote apiserver sustained-unreachable; recycling manager", "host", host)
+			cancelCycle(fmt.Errorf("remote apiserver unreachable: %s", host))
+		})
+
 		if placementShim != nil {
 			if err := placementShim.SetupControllerWithManager(ctx, mgr, multiclusterClient); err != nil {
 				return fmt.Errorf("unable to set up placement shim controller: %w", err)
@@ -431,7 +452,7 @@ func main() {
 		// goroutine only marks ready while the context is still live, and the
 		// deferred clear always wins the final state.
 		var readyMu sync.Mutex
-		cacheCtx, cancelCacheSync := context.WithCancel(ctx)
+		cacheCtx, cancelCacheSync := context.WithCancel(cycleCtx)
 		defer func() {
 			cancelCacheSync()
 			readyMu.Lock()
@@ -483,7 +504,7 @@ func main() {
 		}
 
 		setupLog.Info("starting manager")
-		return mgr.Start(ctx)
+		return mgr.Start(cycleCtx)
 	}
 
 	// +kubebuilder:scaffold:builder

docs/guides/multicluster/cortex-remote-crb.yaml

@@ -15,6 +15,9 @@ subjects:
 - kind: User
   apiGroup: rbac.authorization.k8s.io
   name: "https://host.docker.internal:8443#system:serviceaccount:default:cortex-pods-controller-manager"
+- kind: User
+  apiGroup: rbac.authorization.k8s.io
+  name: "https://host.docker.internal:8443#system:serviceaccount:default:cortex-placement-shim"
 roleRef:
   kind: ClusterRole
   name: cluster-admin

docs/guides/multicluster/run.sh

@@ -78,4 +78,4 @@ kubectl --context kind-cortex-remote-az-b apply \
 
 echo "Starting cortex in home cluster with tilt, using overrides from $TILT_OVERRIDES_PATH"
 kubectl config use-context kind-cortex-home
-export ACTIVE_DEPLOYMENTS="nova" && tilt up
+tilt up

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/prometheus/client_golang v1.24.1
 	github.com/prometheus/client_model v0.6.3
 	github.com/sapcc/go-api-declarations v1.25.0
-	github.com/sapcc/go-bits v0.0.0-20260827091731-7669cbdb53fb
+	github.com/sapcc/go-bits v0.0.0-20260903192122-1774475e70e3
 	go.uber.org/zap v1.28.0
 	go.xyrillian.de/gg v1.14.0
 	golang.org/x/sync v0.22.0
@@ -24,7 +24,7 @@ require (
 	k8s.io/api v0.37.0
 	k8s.io/apimachinery v0.37.0
 	k8s.io/client-go v0.37.0
-	sigs.k8s.io/controller-runtime v0.24.1
+	sigs.k8s.io/controller-runtime v0.25.0
 )
 
 require (
@@ -65,12 +65,12 @@ require (
 	github.com/go-openapi/swag/typeutils v0.27.1 // indirect
 	github.com/go-openapi/swag/yamlutils v0.27.1 // indirect
 	github.com/go-sql-driver/mysql v1.10.0 // indirect
-	github.com/google/cel-go v0.29.0 // indirect
+	github.com/google/cel-go v0.29.2 // indirect
 	github.com/google/gnostic-models v0.7.1 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -85,27 +85,27 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/poy/onpar v0.3.5 // indirect
-	github.com/prometheus/common v0.70.1 // indirect
+	github.com/prometheus/common v0.71.0 // indirect
 	github.com/prometheus/procfs v0.21.1 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/sirupsen/logrus v1.9.4 // indirect
 	github.com/spf13/cobra v1.10.2 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/ziutek/mymysql v1.5.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
 	go.opentelemetry.io/otel v1.44.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect
 	go.opentelemetry.io/otel/metric v1.44.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.44.0 // indirect
 	go.opentelemetry.io/otel/trace v1.44.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	go.yaml.in/yaml/v3 v3.0.5 // indirect
 	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
-	golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
+	golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
 	golang.org/x/net v0.58.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect
@@ -119,14 +119,14 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gotest.tools v2.2.0+incompatible // indirect
-	k8s.io/apiextensions-apiserver v0.36.3 // indirect
-	k8s.io/apiserver v0.36.3 // indirect
-	k8s.io/component-base v0.36.3 // indirect
+	k8s.io/apiextensions-apiserver v0.37.0 // indirect
+	k8s.io/apiserver v0.37.0 // indirect
+	k8s.io/component-base v0.37.0 // indirect
 	k8s.io/klog/v2 v2.140.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect
 	k8s.io/streaming v0.37.0 // indirect
 	k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect
-	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 // indirect
 	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
 	sigs.k8s.io/randfill v1.0.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect