Renew registrar leaves and reload TLS (#768)

CHANGELOG.md

@@ -48,6 +48,12 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
   CA, and its certificates and keys survive the restart byte-identically.
   An already-expired leaf is repaired at start, before the endpoint's TLS
   material loads, rather than at the first renewal tick.
+- An enabled registrar endpoint now renews both its server and registrar
+  client certificates on the daemon's regular maintenance cadence. The
+  endpoint applies renewed TLS material to new handshakes without
+  restarting its socket, while the in-repository registrar client reloads
+  its certificate and key for each new dial and safely retries the brief
+  atomic-replacement interval.
 - `bootroot-agent` now rotates `OpenBao`'s file audit device on a host
   whose registrar endpoint is enabled, so the deployment no longer needs
   an external rotator against it. Every 60 seconds the daemon renames the

docs/en/operations.md

@@ -1389,6 +1389,19 @@ is then wrapped in mutual TLS.
 - **The verbs see `registrar-client:<san>`** as the caller of record, and
   never a peer-credential value.
 
+#### Caller certificate reload contract
+
+The registrar client certificate and key are renewed by two atomic
+renames: the certificate is replaced first and the key second. A caller
+must load both files for every new dial and verify that the key belongs to
+the loaded leaf; it must never cache one half across dials or present a
+mismatched pair. The in-repository registrar endpoint client is the
+reference implementation: on a mismatch it reads the pair again up to
+five times, waiting 1 ms, 2 ms, 4 ms, then 8 ms between the first four
+failed reads, and returns a typed mismatch error if the fifth read is
+still torn. Existing connections retain their TLS configuration; a new
+dial picks up the next matching pair without a caller restart or signal.
+
 A connection that opens and never sends a `ClientHello` is dropped after
 five seconds, so an unauthenticated peer cannot hold one of the sixteen
 connection slots open.

docs/ko/operations.md

@@ -1321,6 +1321,18 @@ WantedBy=multi-user.target
 - **동사가 보는 호출자**는 `registrar-client:<san>`이며, 피어 자격증명 값이
   동사에 도달하는 일은 없습니다.
 
+#### 호출자 인증서 리로드 계약
+
+레지스트라 클라이언트 인증서와 키는 두 번의 atomic rename으로 갱신됩니다.
+인증서가 먼저 교체되고 키가 뒤따릅니다. 호출자는 새 dial마다 두 파일을 다시
+읽고 키가 읽은 리프에 속하는지 확인해야 하며, dial 사이에 한쪽을 캐시하거나
+서로 맞지 않는 쌍을 제시해서는 안 됩니다. 이 저장소의 레지스트라 엔드포인트
+클라이언트가 기준 구현입니다. 불일치가 나면 최대 다섯 번 쌍을 다시 읽고,
+앞의 네 번 실패 뒤에는 각각 1 ms, 2 ms, 4 ms, 8 ms를 기다립니다. 다섯 번째
+읽기에도 쌍이 찢어진 상태이면 형식화된 불일치 오류를 반환합니다. 이미 맺어진
+연결은 기존 TLS 구성을 유지하고, 새 dial은 호출자를 재시작하거나 신호를
+보내지 않아도 다음으로 일치하는 쌍을 사용합니다.
+
 연결을 열어 놓고 `ClientHello`를 보내지 않는 피어는 5초 뒤에 버려집니다.
 인증되지 않은 피어가 16개의 연결 슬롯 중 하나를 계속 붙잡고 있을 수 없게
 하기 위해서입니다.

src/daemon.rs

@@ -214,6 +214,27 @@ pub(crate) async fn run_daemon(invocation: DaemonInvocation) -> anyhow::Result<(
 
     #[cfg(target_os = "linux")]
     let registrar_maintenance = if let Some((endpoint, built)) = registrar_service {
+        // Start-time issuance completed before activation. Initialize the
+        // renewal observation from that verified on-disk material here,
+        // before the accept task below can serve its first connection.
+        let renewal_plan = crate::registrar_certs::resolve_surface_plan(&settings)?;
+        let renewal_states =
+            crate::registrar_certs::initialize_renewal_states(&renewal_plan.pairs).await?;
+        endpoint.set_renewal_states(Arc::clone(&renewal_states));
+        let renewal_endpoint = Arc::clone(&endpoint);
+        let renewal_settings = Arc::clone(&settings);
+        let renewal_shutdown = shutdown_rx.clone();
+        handles.push(tokio::spawn(async move {
+            crate::registrar_certs::run_surface_renewal_loop(
+                renewal_settings,
+                renewal_endpoint,
+                renewal_plan,
+                renewal_states,
+                insecure_mode,
+                renewal_shutdown,
+            )
+            .await
+        }));
         spawn_registrar_endpoint(
             &mut handles,
             endpoint,

src/fs_util.rs

@@ -1069,6 +1069,16 @@ impl FixedOwner {
         Self { uid: 0, gid: 0 }
     }
 
+    /// An already-observed owner that an atomic restore must re-establish.
+    ///
+    /// This remains crate-visible: configuration and external callers cannot
+    /// select an arbitrary owner for newly created files.
+    #[must_use]
+    #[cfg(target_os = "linux")]
+    pub(crate) const fn observed(uid: u32, gid: u32) -> Self {
+        Self { uid, gid }
+    }
+
     /// The test process's own effective uid and gid.
     ///
     /// The one way a protected publish is driven under an owner other

src/registrar/endpoint.rs

@@ -108,7 +108,7 @@ mod tests;
 
 use std::os::unix::io::{FromRawFd as _, RawFd};
 use std::path::PathBuf;
-use std::sync::Arc;
+use std::sync::{Arc, PoisonError, RwLock};
 use std::time::Duration;
 
 use anyhow::Context as _;
@@ -118,7 +118,6 @@ use tokio_rustls::TlsAcceptor;
 use tracing::{info, warn};
 
 use self::activation::{ActivationContract, ActivationValues};
-use self::tls::EndpointCertResolver;
 use crate::config::Settings;
 
 /// Largest declared request payload the endpoint will read.
@@ -217,14 +216,13 @@ pub(crate) struct ActivatedEndpoint {
     listener: UnixListener,
     socket_path: PathBuf,
     daemon_uid: u32,
-    acceptor: TlsAcceptor,
-    // The certificate resolver is retained for exactly one consumer,
-    // and that consumer — renewal for the endpoint leaf — is a sibling
-    // issue. It has to be retained now because there is no way back to
-    // the concrete resolver from a built `ServerConfig`, so a build
-    // that dropped it would leave renewal with nothing to swap
-    // through. Reachability is asserted through `cert_resolver`.
-    resolver: Arc<EndpointCertResolver>,
+    // A complete acceptor is replaced after a renewal publishes every
+    // input it was built from. Replacing only the certificate resolver
+    // would leave the old client verifier installed.
+    acceptor: RwLock<Arc<TlsAcceptor>>,
+    // The renewal task updates this source of truth; the reporting work
+    // reads the same handle without parsing certificate files on requests.
+    renewal_states: RwLock<Option<crate::registrar_certs::SurfaceRenewalStates>>,
     domain: String,
 }
 
@@ -245,31 +243,65 @@ impl ActivatedEndpoint {
     }
 
     /// Returns the acceptor every accepted connection is handed to.
-    pub(crate) fn tls_acceptor(&self) -> &TlsAcceptor {
-        &self.acceptor
+    pub(crate) fn tls_acceptor(&self) -> Arc<TlsAcceptor> {
+        Arc::clone(&self.acceptor.read().unwrap_or_else(PoisonError::into_inner))
     }
 
-    /// Returns the resolver that decides which certificate the next
-    /// handshake presents.
+    /// Exchanges the acceptor used by future handshakes.
     ///
-    /// This is the whole of the renewal seam. [`rustls::ServerConfig`]
-    /// keeps only an `Arc<dyn ResolvesServerCert>`, and the trait does
-    /// not extend `Any`, so code holding the configuration or the
-    /// acceptor cannot reach the concrete resolver at all — it survives
-    /// here or nowhere.
-    // The renewal work that calls this is a sibling issue, so until it
-    // lands nothing outside a test reaches the accessor. It exists now
-    // so that work needs nothing from this one.
+    /// Existing handshakes retain the clone they loaded before this
+    /// exchange and therefore continue under their original TLS policy.
+    pub(crate) fn replace_server_config(&self, config: Arc<rustls::ServerConfig>) {
+        let mut acceptor = self
+            .acceptor
+            .write()
+            .unwrap_or_else(PoisonError::into_inner);
+        *acceptor = Arc::new(TlsAcceptor::from(config));
+    }
+
+    /// Installs the daemon-owned registrar certificate renewal observations.
+    pub(crate) fn set_renewal_states(&self, states: crate::registrar_certs::SurfaceRenewalStates) {
+        let mut slot = self
+            .renewal_states
+            .write()
+            .unwrap_or_else(PoisonError::into_inner);
+        *slot = Some(states);
+    }
+
+    /// Returns the daemon-owned registrar certificate renewal observations.
+    // The reporting child reads this accessor once its response member lands;
+    // renewal owns the state now so it is not recomputed on a request path.
     #[allow(dead_code)]
-    pub(crate) fn cert_resolver(&self) -> &Arc<EndpointCertResolver> {
-        &self.resolver
+    pub(crate) fn renewal_states(&self) -> Option<crate::registrar_certs::SurfaceRenewalStates> {
+        self.renewal_states
+            .read()
+            .unwrap_or_else(PoisonError::into_inner)
+            .clone()
     }
 
     /// Returns the configured `network.domain` the presented client
     /// identity is recognized against.
     pub(crate) fn domain(&self) -> &str {
         &self.domain
     }
+
+    /// Creates an endpoint shell for renewal tests.
+    #[cfg(test)]
+    pub(crate) fn for_renewal_test(
+        listener: UnixListener,
+        socket_path: PathBuf,
+        server_config: Arc<rustls::ServerConfig>,
+        domain: String,
+    ) -> Arc<Self> {
+        Arc::new(Self {
+            listener,
+            socket_path,
+            daemon_uid: current_effective_uid(),
+            acceptor: RwLock::new(Arc::new(TlsAcceptor::from(server_config))),
+            renewal_states: RwLock::new(None),
+            domain,
+        })
+    }
 }
 
 impl std::fmt::Debug for ActivatedEndpoint {
@@ -346,7 +378,7 @@ pub(crate) fn activate(settings: &Settings) -> anyhow::Result<Option<Arc<Activat
     if warns_about_unprivileged_daemon(enabled, effective_uid) {
         warn!("{UNPRIVILEGED_DAEMON_WARNING}");
     }
-    let (server_config, resolver) = tls::build_server_config(
+    let (server_config, _) = tls::build_server_config(
         settings.registrar_endpoint.server_cert_path.as_deref(),
         settings.registrar_endpoint.server_key_path.as_deref(),
         settings.trust.ca_bundle_path.as_deref(),
@@ -361,7 +393,6 @@ pub(crate) fn activate(settings: &Settings) -> anyhow::Result<Option<Arc<Activat
         contract,
         effective_uid,
         server_config,
-        resolver,
         settings.domain.clone(),
     )
     .map(Some)
@@ -375,7 +406,7 @@ pub(crate) fn activate(settings: &Settings) -> anyhow::Result<Option<Arc<Activat
 /// against a listener its own harness bound, through the very code
 /// production runs. Nothing below this point binds, unlinks or chmods
 /// anything, and nothing below it reads certificate material: the
-/// already-built configuration and its resolver arrive as values.
+/// already-built configuration arrives as a value.
 ///
 /// # Errors
 ///
@@ -385,7 +416,6 @@ pub(crate) fn adopt(
     contract: ActivationContract,
     effective_uid: u32,
     server_config: Arc<rustls::ServerConfig>,
-    resolver: Arc<EndpointCertResolver>,
     domain: String,
 ) -> anyhow::Result<Arc<ActivatedEndpoint>> {
     let fd = contract.into_descriptor();
@@ -431,8 +461,8 @@ pub(crate) fn adopt(
         listener,
         socket_path,
         daemon_uid: effective_uid,
-        acceptor: TlsAcceptor::from(server_config),
-        resolver,
+        acceptor: RwLock::new(Arc::new(TlsAcceptor::from(server_config))),
+        renewal_states: RwLock::new(None),
         domain,
     }))
 }

src/registrar/endpoint/client.rs

@@ -96,6 +96,20 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
 /// deadline.
 const READ_TIMEOUT: Duration = Duration::from_secs(30);
 
+/// The bounded reader-side half of the certificate/key publication
+/// contract. Writers rename the certificate and then the key; a dial in
+/// that interval retries this many complete reads before reporting a
+/// persistent mismatch.
+const PAIR_LOAD_ATTEMPTS: usize = 5;
+
+/// Delays after the first four mismatches in a torn pair read.
+const PAIR_LOAD_DELAYS: [Duration; PAIR_LOAD_ATTEMPTS - 1] = [
+    Duration::from_millis(1),
+    Duration::from_millis(2),
+    Duration::from_millis(4),
+    Duration::from_millis(8),
+];
+
 /// The name handed to [`TlsConnector::connect`].
 ///
 /// Inert. [`endpoint_pin::RegistrarEndpointVerifier`] deliberately
@@ -189,6 +203,19 @@ pub(crate) enum ClientMaterialError {
         /// The path that was handed to the client.
         path: PathBuf,
     },
+    /// The certificate and key were both readable but belong to
+    /// different generations after every bounded retry.
+    #[error(
+        "registrar client certificate {} and key {} do not form a matching pair",
+        .certificate_path.display(),
+        .key_path.display()
+    )]
+    KeyMismatch {
+        /// The certificate path that was read.
+        certificate_path: PathBuf,
+        /// The key path that was read.
+        key_path: PathBuf,
+    },
     /// The pair parsed but `rustls` will not authenticate with it.
     #[error(
         "registrar client certificate {} and key {} are not usable client authentication \
@@ -761,10 +788,8 @@ pub(crate) fn build_client_config(
 /// Loads the registrar client leaf and its key from disk.
 ///
 /// The single seam every dial's material comes through. It does the
-/// plain load and nothing more: it does not verify that the key matches
-/// the leaf, does not retry a pair that is momentarily torn by a
-/// renewal, and does not inspect `notAfter`. Later work that owns those
-/// reasons adds them here rather than restructuring the dial path.
+/// checks the key against the leaf and retries the short certificate/key
+/// rename interval. It does not inspect `notAfter`.
 ///
 /// # Errors
 ///
@@ -773,6 +798,53 @@ pub(crate) fn build_client_config(
 fn load_client_material(
     certificate_path: &Path,
     key_path: &Path,
+) -> Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>), ClientMaterialError> {
+    load_client_material_with(
+        || load_client_material_once(certificate_path, key_path),
+        std::thread::sleep,
+    )
+}
+
+/// Applies the bounded torn-pair retry policy around one complete material
+/// read. Keeping the delay operation injectable makes the fixed reader-side
+/// contract deterministic without changing the production filesystem path.
+fn load_client_material_with<Load, Sleep>(
+    mut load_once: Load,
+    mut sleep: Sleep,
+) -> Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>), ClientMaterialError>
+where
+    Load: FnMut() -> Result<
+        (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>),
+        ClientMaterialError,
+    >,
+    Sleep: FnMut(Duration),
+{
+    for (attempt, delay) in PAIR_LOAD_DELAYS
+        .iter()
+        .copied()
+        .map(Some)
+        .chain(std::iter::once(None))
+        .enumerate()
+    {
+        match load_once() {
+            Ok(pair) => return Ok(pair),
+            Err(error @ ClientMaterialError::KeyMismatch { .. }) if delay.is_some() => {
+                sleep(delay.expect("retry delay exists before the final attempt"));
+                tracing::debug!(
+                    attempt,
+                    "Registrar client pair changed during a per-dial load; retrying."
+                );
+                let _ = error;
+            }
+            Err(error) => return Err(error),
+        }
+    }
+    unreachable!("the fixed pair-load attempt sequence always has a final attempt")
+}
+
+fn load_client_material_once(
+    certificate_path: &Path,
+    key_path: &Path,
 ) -> Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>), ClientMaterialError> {
     let certificate_bytes = std::fs::read(certificate_path).map_err(|source| {
         if source.kind() == io::ErrorKind::NotFound {
@@ -828,6 +900,23 @@ fn load_client_material(
             path: key_path.to_path_buf(),
         })?;
 
+    let leaf = chain
+        .first()
+        .ok_or_else(|| ClientMaterialError::NoCertificate {
+            path: certificate_path.to_path_buf(),
+        })?;
+    // An unsupported-but-well-formed key remains this loader's existing
+    // `Unusable` error when `rustls` builds the client configuration.
+    // Only a key this build can use is eligible for the torn-pair retry.
+    if let Ok(signing_key) = rustls::crypto::ring::sign::any_supported_type(&key)
+        && !crate::tls::cert_key_matches(leaf, signing_key.as_ref())
+    {
+        return Err(ClientMaterialError::KeyMismatch {
+            certificate_path: certificate_path.to_path_buf(),
+            key_path: key_path.to_path_buf(),
+        });
+    }
+
     Ok((chain, key))
 }