Placement shim: detect a lost remote apiserver and rebuild the manager

cmd/shim/main.go

@@ -403,11 +403,32 @@ func main() {
 			return fmt.Errorf("unable to create manager: %w", err)
 		}
 
-		multiclusterClient, err := setupMulticlusterClient(ctx, mgr, restConfig, multiclusterMonitor)
+		// cycleCtx scopes this manager cycle: cancelling it makes mgr.Start return
+		// so the supervisor rebuilds the manager (re-reading config and
+		// reconnecting to the currently-configured remotes). It is cancelled on a
+		// clean return (defer), when the parent ctx is cancelled (child), or by the
+		// reachability probe below when a remote apiserver is lost. This is the
+		// external liveness signal the supervisor lacks on its own: a remote whose
+		// informer already synced and then disappears is retried by
+		// controller-runtime forever without mgr.Start ever returning.
+		cycleCtx, cancelCycle := context.WithCancelCause(ctx)
+		defer cancelCycle(nil)
+
+		multiclusterClient, err := setupMulticlusterClient(cycleCtx, mgr, restConfig, multiclusterMonitor)
 		if err != nil {
 			return fmt.Errorf("unable to set up multicluster client: %w", err)
 		}
 
+		// Probe each remote apiserver for reachability. When a remote is
+		// sustained-unreachable (e.g. its cluster was deleted after its cache had
+		// already synced), cancel the cycle so the supervisor rebuilds the manager.
+		// A remote that answers with any HTTP status is reachable, so an authz or
+		// server error does not trigger a rebuild storm.
+		go multiclusterClient.ProbeRemotes(cycleCtx, multicluster.DefaultProbeOptions, func(host string) {
+			setupLog.Info("remote apiserver sustained-unreachable; recycling manager", "host", host)
+			cancelCycle(fmt.Errorf("remote apiserver unreachable: %s", host))
+		})
+
 		if placementShim != nil {
 			if err := placementShim.SetupControllerWithManager(ctx, mgr, multiclusterClient); err != nil {
 				return fmt.Errorf("unable to set up placement shim controller: %w", err)
@@ -431,7 +452,7 @@ func main() {
 		// goroutine only marks ready while the context is still live, and the
 		// deferred clear always wins the final state.
 		var readyMu sync.Mutex
-		cacheCtx, cancelCacheSync := context.WithCancel(ctx)
+		cacheCtx, cancelCacheSync := context.WithCancel(cycleCtx)
 		defer func() {
 			cancelCacheSync()
 			readyMu.Lock()
@@ -483,7 +504,7 @@ func main() {
 		}
 
 		setupLog.Info("starting manager")
-		return mgr.Start(ctx)
+		return mgr.Start(cycleCtx)
 	}
 
 	// +kubebuilder:scaffold:builder

docs/guides/multicluster/cortex-remote-crb.yaml

@@ -15,6 +15,9 @@ subjects:
 - kind: User
   apiGroup: rbac.authorization.k8s.io
   name: "https://host.docker.internal:8443#system:serviceaccount:default:cortex-pods-controller-manager"
+- kind: User
+  apiGroup: rbac.authorization.k8s.io
+  name: "https://host.docker.internal:8443#system:serviceaccount:default:cortex-placement-shim"
 roleRef:
   kind: ClusterRole
   name: cluster-admin

docs/guides/multicluster/run.sh

@@ -78,4 +78,4 @@ kubectl --context kind-cortex-remote-az-b apply \
 
 echo "Starting cortex in home cluster with tilt, using overrides from $TILT_OVERRIDES_PATH"
 kubectl config use-context kind-cortex-home
-export ACTIVE_DEPLOYMENTS="nova" && tilt up
+tilt up

helm/bundles/cortex-placement-shim/templates/alerts.yaml

@@ -242,4 +242,31 @@ spec:
           resource router is mapping the same object to multiple clusters, or an
           object was created out-of-band on the wrong cluster. Investigate the
           affected resources and the routing configuration.
+
+    - alert: CortexPlacementShimRemoteApiserverUnreachable
+      # min by (host): a remote is unreachable only if it was 0 for the whole
+      # window (a rebuild flips it 1->0->1, and min ignores the brief 1s during a
+      # reconnect attempt). Fires per host so the offending remote is named.
+      expr: |
+        min by (host) (cortex_multicluster_remote_apiserver_reachable{service="cortex-placement-shim-metrics-service"}) < 1
+      for: 5m
+      labels:
+        context: multicluster
+        dashboard: cortex-placement-shim-status-dashboard/cortex-placement-shim-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+      annotations:
+        summary: "Remote apiserver `{{ "{{" }} $labels.host {{ "}}" }}` is unreachable"
+        description: >
+          The multicluster client cannot reach the remote apiserver at
+          `{{ "{{" }} $labels.host {{ "}}" }}`. The reachability probe has seen
+          sustained transport failures (connection refused, TLS/handshake failure,
+          or timeout), which is the signature of a deleted or network-partitioned
+          cluster rather than an authz/server error. The shim recycles its
+          controller-manager in response, so the manager cache for resources served
+          by that remote is stale or empty and `cortex_placement_shim_manager_up`
+          will average low (see CortexPlacementShimManagerLooping). Passthrough
+          requests to upstream Placement are unaffected. Investigate whether the
+          remote cluster still exists and network connectivity to its apiserver.
 {{- end }}

pkg/multicluster/client.go

@@ -89,7 +89,7 @@ func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf Client
 		gvksByConfStr[formatted] = gvk
 	}
 	for gvkStr := range gvksByConfStr {
-		log.Info("scheme gvk registered", "gvk", gvkStr)
+		log.V(1).Info("scheme gvk registered", "gvk", gvkStr)
 	}
 	// Parse home GVKs.
 	c.homeGVKs = make(map[schema.GroupVersionKind]bool)

pkg/multicluster/client_probe.go

@@ -0,0 +1,220 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package multicluster
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cluster"
+)
+
+// RemoteEndpoint identifies a unique remote cluster to probe for reachability.
+type RemoteEndpoint struct {
+	// Host is the remote apiserver URL (from the cluster's rest.Config).
+	Host string
+	// Cluster is the controller-runtime cluster whose apiserver is probed.
+	Cluster cluster.Cluster
+}
+
+// UniqueRemotes returns each distinct remote cluster exactly once, across all
+// configured GVKs. The same remote apiserver is stored once per GVK it serves,
+// so this dedupes by cluster identity (the same policy IndexField uses to dedupe
+// caches). The home cluster is not included.
+func (c *Client) UniqueRemotes() []RemoteEndpoint {
+	c.remoteClustersMu.RLock()
+	defer c.remoteClustersMu.RUnlock()
+	seen := make(map[cluster.Cluster]bool)
+	var out []RemoteEndpoint
+	for _, remotes := range c.remoteClusters {
+		for _, r := range remotes {
+			if r.cluster == nil || seen[r.cluster] {
+				continue
+			}
+			seen[r.cluster] = true
+			out = append(out, RemoteEndpoint{Host: r.cluster.GetConfig().Host, Cluster: r.cluster})
+		}
+	}
+	return out
+}
+
+// ProbeOptions configures the per-remote reachability probe.
+type ProbeOptions struct {
+	// Interval is the time between reachability probes for each remote.
+	Interval time.Duration
+	// Timeout bounds a single probe request so it never blocks on a dead socket
+	// longer than intended.
+	Timeout time.Duration
+	// FailureThreshold is the number of consecutive unreachable probes that must
+	// occur before a remote is considered lost and onLost is called. It is chosen
+	// above the supervisor's backoff floor so a doomed manager cycle outlives the
+	// growing backoff, keeping the rebuild loop period bounded rather than hot.
+	FailureThreshold int
+}
+
+// DefaultProbeOptions probes every 10s with a 5s per-probe timeout and treats a
+// remote as lost after 3 consecutive failures (~30s of sustained unreachability).
+var DefaultProbeOptions = ProbeOptions{
+	Interval:         10 * time.Second,
+	Timeout:          5 * time.Second,
+	FailureThreshold: 3,
+}
+
+// ProbeRemotes runs one reachability probe goroutine per unique remote apiserver
+// until ctx is cancelled. Each goroutine periodically probes its remote, updates
+// the reachability gauge on the Monitor (if any), and tracks consecutive
+// failures; when a remote has been unreachable FailureThreshold times in a row it
+// calls onLost(host) once and stops probing that remote (the caller is expected
+// to tear down the manager cycle in response).
+//
+// A single transient blip does not trigger onLost: the failure counter resets on
+// the first reachable probe. "Unreachable" means a transport-level failure
+// (connection refused, no route, TLS/handshake failure, timeout) — an apiserver
+// that responds with any HTTP status (including 401/403/5xx) is considered
+// reachable, since a manager rebuild cannot fix an authz/server error and would
+// only cause a restart storm.
+//
+// ProbeRemotes returns immediately when no remotes are configured.
+func (c *Client) ProbeRemotes(ctx context.Context, opts ProbeOptions, onLost func(host string)) {
+	log := ctrl.LoggerFrom(ctx)
+	remotes := c.UniqueRemotes()
+	hosts := make([]string, 0, len(remotes))
+	for _, r := range remotes {
+		hosts = append(hosts, r.Host)
+	}
+	if len(remotes) == 0 {
+		// No remotes to watch: log it explicitly so an operator who deletes a
+		// cluster and sees "nothing happening" can tell this apart from the probe
+		// simply not finding the remote it expected to watch.
+		log.Info("reachability probe: no remote apiservers configured, nothing to probe")
+		return
+	}
+	log.Info("reachability probe: starting", "remoteCount", len(remotes), "hosts", hosts,
+		"interval", opts.Interval.String(), "timeout", opts.Timeout.String(), "failureThreshold", opts.FailureThreshold)
+	var wg wait.Group
+	for _, remote := range remotes {
+		httpClient, err := reachabilityClient(remote.Cluster.GetConfig(), opts.Timeout)
+		if err != nil {
+			// A remote we cannot even build a probe client for is treated as
+			// lost immediately: its config is unusable, so a rebuild (which
+			// re-reads config) is the right response.
+			log.Error(err, "reachability probe: unable to build probe client for remote; declaring it lost", "host", remote.Host)
+			onLost(remote.Host)
+			continue
+		}
+		wg.StartWithContext(ctx, func(ctx context.Context) {
+			c.probeRemoteLoop(ctx, opts, remote, httpClient, onLost)
+		})
+	}
+	wg.Wait()
+	log.Info("reachability probe: stopped", "hosts", hosts)
+}
+
+// probeRemoteLoop probes a single remote until ctx is cancelled or the remote is
+// declared lost.
+func (c *Client) probeRemoteLoop(ctx context.Context, opts ProbeOptions, remote RemoteEndpoint, httpClient *rest.RESTClient, onLost func(host string)) {
+	log := ctrl.LoggerFrom(ctx).WithValues("host", remote.Host)
+	log.Info("reachability probe: watching remote apiserver")
+	failures := 0
+	wasReachable := true // assume reachable at start; log the first real transition
+	// PollUntilContextCancel with immediate=true runs the first probe right away
+	// rather than waiting a full interval, so a remote that is already gone at
+	// cycle start is detected promptly. The poll func never returns an error, so
+	// the only error PollUntilContextCancel can return is the context error once
+	// ctx is cancelled or the loop stops after declaring the remote lost — both
+	// expected, so it is intentionally not surfaced.
+	err := wait.PollUntilContextCancel(ctx, opts.Interval, true, func(ctx context.Context) (bool, error) {
+		reachable, probeErr := probeReachable(ctx, httpClient, opts.Timeout)
+		if c.Monitor != nil {
+			c.Monitor.recordRemoteReachable(remote.Host, reachable)
+		}
+		// Per-probe outcome at V(1): verbose, but the single most useful line when
+		// diagnosing "why didn't it detect the deletion" — it shows the classified
+		// result and the underlying transport/HTTP error for each tick.
+		if probeErr != nil {
+			log.V(1).Info("reachability probe: tick", "reachable", reachable, "error", probeErr.Error())
+		} else {
+			log.V(1).Info("reachability probe: tick", "reachable", reachable)
+		}
+		if reachable {
+			if !wasReachable {
+				log.Info("reachability probe: remote apiserver recovered", "afterConsecutiveFailures", failures)
+			}
+			wasReachable = true
+			failures = 0
+			return false, nil
+		}
+		wasReachable = false
+		failures++
+		if probeErr != nil {
+			log.Info("reachability probe: remote apiserver unreachable",
+				"consecutiveFailures", failures, "threshold", opts.FailureThreshold, "error", probeErr.Error())
+		} else {
+			log.Info("reachability probe: remote apiserver unreachable",
+				"consecutiveFailures", failures, "threshold", opts.FailureThreshold)
+		}
+		if failures >= opts.FailureThreshold {
+			log.Info("reachability probe: failure threshold reached, declaring remote lost", "consecutiveFailures", failures)
+			onLost(remote.Host)
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil && ctx.Err() == nil {
+		// A non-context error is not expected here (the poll func never returns
+		// one), but log it rather than swallow it if the invariant ever changes.
+		log.Error(err, "remote reachability poll stopped unexpectedly")
+	}
+}
+
+// reachabilityClient builds a REST client for the given remote config that is
+// used only to probe reachability. It copies the config (so the caller's shared
+// config is never mutated), sets a per-probe timeout, and supplies a codec so the
+// unversioned REST client can be constructed (we only read the transport outcome,
+// never decode a body, but rest requires a NegotiatedSerializer).
+func reachabilityClient(cfg *rest.Config, timeout time.Duration) (*rest.RESTClient, error) {
+	cfgCopy := *cfg
+	cfgCopy.Timeout = timeout
+	cfgCopy.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
+	if cfgCopy.GroupVersion == nil {
+		cfgCopy.GroupVersion = &schema.GroupVersion{}
+	}
+	httpClient, err := rest.HTTPClientFor(&cfgCopy)
+	if err != nil {
+		return nil, err
+	}
+	return rest.UnversionedRESTClientForConfigAndClient(&cfgCopy, httpClient)
+}
+
+// probeReachable issues a lightweight GET /readyz against the remote apiserver
+// and reports whether the apiserver is reachable. Any HTTP response — including
+// non-2xx statuses surfaced as a Kubernetes StatusError — counts as reachable;
+// only a transport-level error (connection refused, no route, TLS failure,
+// timeout) counts as unreachable, which is the signature of a deleted cluster.
+// probeReachable issues a lightweight GET /readyz against the remote apiserver
+// and reports whether the apiserver is reachable, along with the underlying
+// error (nil on a 2xx, otherwise the transport or HTTP error even when the result
+// is classified reachable) for logging. Any HTTP response — including non-2xx
+// statuses surfaced as a Kubernetes StatusError — counts as reachable; only a
+// transport-level error (connection refused, no route, TLS failure, timeout)
+// counts as unreachable, which is the signature of a deleted cluster.
+func probeReachable(ctx context.Context, client *rest.RESTClient, timeout time.Duration) (bool, error) {
+	probeCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	err := client.Get().AbsPath("/readyz").Do(probeCtx).Error()
+	if err == nil {
+		return true, nil
+	}
+	// The apiserver answered with a non-2xx status: it is up and reachable. Return
+	// the error too so callers can log the status the apiserver returned.
+	var statusErr *apierrors.StatusError
+	return errors.As(err, &statusErr), err
+}