feat(toolkit): add REST healthcheck infrastructure

gears/system/api-gateway/src/config.rs

@@ -8,8 +8,12 @@ fn default_body_limit_bytes() -> usize {
     16 * 1024 * 1024
 }
 
+fn default_healthcheck_timeout_ms() -> u64 {
+    500
+}
+
 /// API gateway configuration - reused from `api_gateway` gear
-#[derive(Debug, Clone, Deserialize, Serialize, Default)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(deny_unknown_fields)]
 #[allow(clippy::struct_excessive_bools)]
 pub struct ApiGatewayConfig {
@@ -57,6 +61,31 @@ pub struct ApiGatewayConfig {
     /// HTTP metrics configuration.
     #[serde(default)]
     pub metrics: MetricsConfig,
+
+    /// Per-check `/health`/`/readyz` timeout (ms); raise for slow dependencies.
+    #[serde(default = "default_healthcheck_timeout_ms")]
+    pub healthcheck_timeout_ms: u64,
+}
+
+impl Default for ApiGatewayConfig {
+    // Manual (not derived) so defaults match the `#[serde(default = ...)]` fns above;
+    // a derived Default would zero each field (e.g. timeout 0 = instant probe failure).
+    fn default() -> Self {
+        Self {
+            bind_addr: String::default(),
+            enable_docs: false,
+            cors_enabled: false,
+            cors: None,
+            openapi: OpenApiConfig::default(),
+            defaults: Defaults::default(),
+            auth_disabled: false,
+            require_auth_by_default: default_require_auth_by_default(),
+            prefix_path: String::default(),
+            route_policies: RoutePoliciesConfig::default(),
+            metrics: MetricsConfig::default(),
+            healthcheck_timeout_ms: default_healthcheck_timeout_ms(),
+        }
+    }
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize)]

gears/system/api-gateway/src/gear.rs

@@ -3,7 +3,7 @@
 //! Contains the `ApiGateway` gear struct and its trait implementations.
 
 use async_trait::async_trait;
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
 
 use arc_swap::ArcSwap;
 use dashmap::DashMap;
@@ -12,12 +12,12 @@ use anyhow::Result;
 use axum::error_handling::HandleErrorLayer;
 use axum::http::Method;
 use axum::middleware::from_fn_with_state;
-use axum::{Router, extract::DefaultBodyLimit, middleware::from_fn, routing::get};
+use axum::{Extension, Router, extract::DefaultBodyLimit, middleware::from_fn, routing::get};
 use parking_lot::Mutex;
 use std::net::SocketAddr;
 use std::time::Duration;
 use tokio_util::sync::CancellationToken;
-use toolkit::api::{OpenApiRegistry, OpenApiRegistryImpl};
+use toolkit::api::{OpenApiRegistry, OpenApiRegistryImpl, OperationBuilder};
 use toolkit::lifecycle::ReadySignal;
 use tower::{BoxError, ServiceBuilder};
 use tower_http::{
@@ -77,6 +77,13 @@ pub struct ApiGateway {
     pub(crate) final_router: Mutex<Option<axum::Router>>,
     // AuthN Resolver client (resolved during init, None when auth_disabled)
     pub(crate) authn_client: Mutex<Option<Arc<dyn AuthNResolverClient>>>,
+    // Readiness registry, set once from `rest_prepare`; `OnceLock` = lock-free reads.
+    pub(crate) healthcheck_registry: OnceLock<Arc<toolkit::RestHealthcheckRegistry>>,
+    // Built-once health router. `health_routes` registers OperationSpecs into the shared
+    // OpenAPI registry, so it must run exactly once per instance; caching the built router
+    // makes repeat `build_router`/`rebuild_and_cache_router` calls idempotent (no duplicate
+    // handler_id/route registration and its ERROR log).
+    pub(crate) health_router: OnceLock<axum::Router>,
 
     // Duplicate detection (per (method, path) and per handler id)
     pub(crate) registered_routes: DashMap<(Method, String), ()>,
@@ -92,24 +99,103 @@ impl Default for ApiGateway {
             router_cache: RouterCache::new(default_router),
             final_router: Mutex::new(None),
             authn_client: Mutex::new(None),
+            healthcheck_registry: OnceLock::new(),
+            health_router: OnceLock::new(),
             registered_routes: DashMap::new(),
             registered_handlers: DashMap::new(),
         }
     }
 }
 
+// Built-in health-probe paths, shared by route registration and auth policy so they can't drift.
+const HEALTH_DETAIL_PATH: &str = "/health";
+const HEALTHZ_PATH: &str = "/healthz";
+const READYZ_PATH: &str = "/readyz";
+
 impl ApiGateway {
-    fn apply_prefix_nesting(mut router: Router, prefix: &str) -> Router {
-        if prefix.is_empty() {
-            return router;
+    /// Nest the middleware-wrapped `router` under `prefix`, then merge the
+    /// pre-built `health_router` at root (always unprefixed).
+    ///
+    /// Both routers must have their own middleware already applied by the caller:
+    /// `router`'s auth matching is keyed on unprefixed `OperationBuilder` paths, so
+    /// its layers must run before `nest()` strips the prefix; health must stay at
+    /// root regardless of `prefix`, hence merged after nesting rather than nested.
+    fn apply_prefix_nesting(router: Router, prefix: &str, health_router: Router) -> Router {
+        let nested = if prefix.is_empty() {
+            router
+        } else {
+            Router::new().nest(prefix, router)
+        };
+
+        health_router.merge(nested)
+    }
+
+    /// Health router with its own middleware stack, ready to merge via [`apply_prefix_nesting`].
+    ///
+    /// Built once and cached: [`health_routes`](Self::health_routes) registers `OperationSpec`s
+    /// into the shared `OpenAPI` registry, so re-running it would trip duplicate detection. The
+    /// health deps (`hc_registry`, `authn_client`, `healthcheck_timeout`) are all fixed after
+    /// init/`rest_prepare`, so the first build is authoritative; later calls reuse the clone.
+    fn build_health_router(
+        &self,
+        hc_registry: Arc<toolkit::RestHealthcheckRegistry>,
+        authn_client: Option<Arc<dyn AuthNResolverClient>>,
+        healthcheck_timeout: Duration,
+    ) -> Result<Router> {
+        if let Some(cached) = self.health_router.get() {
+            return Ok(cached.clone());
         }
+        let built = self.apply_middleware_stack(
+            self.health_routes(hc_registry, healthcheck_timeout),
+            authn_client,
+        )?;
+        // First build wins; a concurrent racer's `set` is a no-op (router builds are
+        // sequential during lifecycle setup, so this is a belt-and-braces guard).
+        drop(self.health_router.set(built.clone()));
+        Ok(built)
+    }
 
-        let top = Router::new()
-            .route("/health", get(web::health_check))
-            .route("/healthz", get(|| async { "ok" }));
+    /// Health-probe routes, defined via [`OperationBuilder`] so each route's auth intent is
+    /// co-located with its definition (and flows into the route policy via `OperationSpec`,
+    /// not a hand-maintained set). `/health` is authenticated (per-component detail); `/healthz`
+    /// and `/readyz` are public (status code only). Deps injected via `Extension`.
+    fn health_routes(
+        &self,
+        hc_registry: Arc<toolkit::RestHealthcheckRegistry>,
+        healthcheck_timeout: Duration,
+    ) -> Router {
+        use http::StatusCode;
+
+        let router = OperationBuilder::get(HEALTH_DETAIL_PATH)
+            .tag("Health Check")
+            .summary("Detailed readiness report")
+            .authenticated()
+            .no_license_required()
+            .handler(web::health_detail)
+            .json_response(StatusCode::OK, "Aggregate healthy or degraded")
+            .json_response(StatusCode::SERVICE_UNAVAILABLE, "Aggregate unhealthy")
+            .register(Router::new(), self);
+
+        let router = OperationBuilder::get(HEALTHZ_PATH)
+            .tag("Health Check")
+            .summary("Liveness probe")
+            .public()
+            .handler(|| async { "ok" })
+            .text_response(StatusCode::OK, "Service is live", "text/plain")
+            .register(router, self);
 
-        router = Router::new().nest(prefix, router);
-        top.merge(router)
+        let router = OperationBuilder::get(READYZ_PATH)
+            .tag("Health Check")
+            .summary("Readiness probe")
+            .public()
+            .handler(web::readyz_check)
+            .no_content_response(StatusCode::OK, "Ready (healthy or degraded)")
+            .no_content_response(StatusCode::SERVICE_UNAVAILABLE, "Not ready (unhealthy)")
+            .register(router, self);
+
+        router
+            .layer(Extension(hc_registry))
+            .layer(Extension(web::HealthcheckTimeout(healthcheck_timeout)))
     }
 
     /// Create a new `ApiGateway` instance with the given configuration
@@ -122,6 +208,8 @@ impl ApiGateway {
             router_cache: RouterCache::new(default_router),
             final_router: Mutex::new(None),
             authn_client: Mutex::new(None),
+            healthcheck_registry: OnceLock::new(),
+            health_router: OnceLock::new(),
             registered_routes: DashMap::new(),
             registered_handlers: DashMap::new(),
         }
@@ -157,10 +245,9 @@ impl ApiGateway {
         let mut authenticated_routes = std::collections::HashSet::new();
         let mut public_routes = std::collections::HashSet::new();
 
-        // Always mark built-in health check routes as public
-        public_routes.insert((Method::GET, "/health".to_owned()));
-        public_routes.insert((Method::GET, "/healthz".to_owned()));
-
+        // Health probes (`/health` authenticated, `/healthz` + `/readyz` public) declare
+        // their auth via `OperationBuilder` in `health_routes`, so they flow into the sets
+        // through the `operation_specs` loop below like every other route.
         public_routes.insert((Method::GET, "/docs".to_owned()));
         public_routes.insert((Method::GET, "/openapi.json".to_owned()));
 
@@ -256,7 +343,11 @@ impl ApiGateway {
         // 14) Propagate MatchedPath to response extensions (route_layer — innermost).
         // This copies MatchedPath from the request (populated by Axum route matching)
         // into the response so outer layer() middleware (metrics) can read it.
-        router = router.route_layer(from_fn(middleware::http_metrics::propagate_matched_path));
+        // `route_layer` panics on a routeless router — reachable when no REST provider
+        // has registered anything yet.
+        if router.has_routes() {
+            router = router.route_layer(from_fn(middleware::http_metrics::propagate_matched_path));
+        }
 
         let config = self.get_cached_config();
 
@@ -484,19 +575,28 @@ impl ApiGateway {
         }
 
         tracing::debug!("Building new router (standalone/fallback mode)");
-        // In standalone mode (no REST pipeline), register both health endpoints here.
-        // In normal operation, rest_prepare() registers these instead.
-        let mut router = Router::new()
-            .route("/health", get(web::health_check))
-            .route("/healthz", get(|| async { "ok" }));
+        // Standalone/fallback path; normally rest_prepare()/rest_finalize() wire health.
+        let hc_registry = self.healthcheck_registry.get().cloned().unwrap_or_else(|| {
+            tracing::warn!(
+                "healthcheck_registry not set (rest_prepare was not called); \
+                 building standalone router with an empty registry"
+            );
+            Arc::new(toolkit::RestHealthcheckRegistry::new())
+        });
 
-        // Apply all middleware layers including auth, above the router
+        // No "main" routes here — the empty router is tolerated (`has_routes` guard)
+        // and health is merged in unprefixed with its own middleware.
         let authn_client = self.authn_client.lock().clone();
-        router = self.apply_middleware_stack(router, authn_client)?;
+        let router = self.apply_middleware_stack(Router::new(), authn_client.clone())?;
 
         let config = self.get_cached_config();
         let prefix = Self::normalize_prefix_path(&config.prefix_path)?;
-        router = Self::apply_prefix_nesting(router, &prefix);
+        let health_router = self.build_health_router(
+            hc_registry,
+            authn_client,
+            Duration::from_millis(config.healthcheck_timeout_ms),
+        )?;
+        let router = Self::apply_prefix_nesting(router, &prefix, health_router);
 
         // Cache the built router for future use
         self.router_cache.store(router.clone());
@@ -708,42 +808,50 @@ impl toolkit::contracts::ApiGatewayCapability for ApiGateway {
         &self,
         _ctx: &toolkit::context::GearCtx,
         router: axum::Router,
+        hc_registry: Arc<toolkit::RestHealthcheckRegistry>,
     ) -> anyhow::Result<axum::Router> {
-        // Add health check endpoints:
-        // - /health: detailed JSON response with status and timestamp
-        // - /healthz: simple "ok" liveness probe (Kubernetes-style)
-        let router = router
-            .route("/health", get(web::health_check))
-            .route("/healthz", get(|| async { "ok" }));
-
-        // You may attach global middlewares here (trace, compression, cors), but do not start server.
-        tracing::debug!("REST host prepared base router with health check endpoints");
+        // Store for use when health routes are added in rest_finalize. A second set
+        // means rest_prepare ran twice — a lifecycle bug; fail fast rather than mask it.
+        if self.healthcheck_registry.set(hc_registry).is_err() {
+            anyhow::bail!("healthcheck_registry already set; rest_prepare called more than once");
+        }
+
+        tracing::debug!("REST host prepared base router");
         Ok(router)
     }
 
     fn rest_finalize(
         &self,
         _ctx: &toolkit::context::GearCtx,
         mut router: axum::Router,
+        hc_registry: Arc<toolkit::RestHealthcheckRegistry>,
     ) -> anyhow::Result<axum::Router> {
         let config = self.get_cached_config();
 
         if config.enable_docs {
             router = self.add_openapi_routes(router)?;
         }
 
-        // Apply middleware stack (including auth) to the final router
+        // Middleware on the main router before nesting (auth matching keyed on
+        // unprefixed OperationBuilder paths; layers run before nest() strips the prefix).
         tracing::debug!("Applying middleware stack to finalized router");
         let authn_client = self.authn_client.lock().clone();
-        router = self.apply_middleware_stack(router, authn_client)?;
+        router = self.apply_middleware_stack(router, authn_client.clone())?;
 
         let prefix = Self::normalize_prefix_path(&config.prefix_path)?;
-        router = Self::apply_prefix_nesting(router, &prefix);
+        let health_router = self.build_health_router(
+            hc_registry,
+            authn_client,
+            Duration::from_millis(config.healthcheck_timeout_ms),
+        )?;
+        router = Self::apply_prefix_nesting(router, &prefix, health_router);
 
         // Keep the finalized router to be used by `serve()`
         *self.final_router.lock() = Some(router.clone());
 
-        tracing::info!("REST host finalized router with OpenAPI endpoints and auth middleware");
+        tracing::info!(
+            "REST host finalized router with OpenAPI endpoints, health checks, and auth middleware"
+        );
         Ok(router)
     }
 

gears/system/api-gateway/src/web.rs

@@ -1,10 +1,19 @@
 use axum::{
+    extract::Extension,
     http::StatusCode,
-    response::{Html, Json},
+    response::{Html, IntoResponse, Json, Response},
     routing::{MethodRouter, get},
 };
 use chrono::{SecondsFormat, Utc};
-use serde_json::{Value, json};
+use serde_json::json;
+use std::sync::Arc;
+use std::time::Duration;
+use toolkit::RestHealthcheckRegistry;
+
+/// Per-check timeout (from `ApiGatewayConfig::healthcheck_timeout_ms`), injected as an
+/// `Extension`. A newtype avoids colliding with any other `Duration` extension.
+#[derive(Clone, Copy)]
+pub struct HealthcheckTimeout(pub Duration);
 
 /// Returns a 501 Not Implemented handler for operations without implementations
 #[allow(dead_code)]
@@ -20,11 +29,38 @@ pub fn placeholder_handler_501() -> MethodRouter {
     })
 }
 
-pub async fn health_check() -> Json<Value> {
-    Json(json!({
-        "status": "healthy",
-        "timestamp": Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true)
-    }))
+/// `GET /health`: runs all checks, returns JSON with per-component detail. Requires
+/// auth (detail not for public callers). 200 (Healthy/Degraded), 503 (Unhealthy).
+pub async fn health_detail(
+    Extension(registry): Extension<Arc<RestHealthcheckRegistry>>,
+    Extension(timeout): Extension<HealthcheckTimeout>,
+) -> Response {
+    let report = registry.report(timeout.0).await;
+    let status_code = status_for_report(report.is_ready());
+    let body = Json(json!({
+        "status": report.status,
+        "timestamp": Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true),
+        "components": report.components,
+    }));
+    (status_code, body).into_response()
+}
+
+/// `GET /readyz`: runs all checks, public, status code only (no detail).
+/// 200 (Healthy/Degraded), 503 (Unhealthy).
+pub async fn readyz_check(
+    Extension(registry): Extension<Arc<RestHealthcheckRegistry>>,
+    Extension(timeout): Extension<HealthcheckTimeout>,
+) -> StatusCode {
+    let report = registry.report(timeout.0).await;
+    status_for_report(report.is_ready())
+}
+
+fn status_for_report(report_ready: bool) -> StatusCode {
+    if report_ready {
+        StatusCode::OK
+    } else {
+        StatusCode::SERVICE_UNAVAILABLE
+    }
 }
 
 #[cfg(not(feature = "embed_elements"))]