From 46ce7d46ab63fae29905a07807cc6ba69573b3e5 Mon Sep 17 00:00:00 2001 From: sehkone Date: Fri, 28 Aug 2026 22:23:42 +0900 Subject: [PATCH] Renew registrar certificates safely Keep registrar enrollment available as its short-lived TLS leaves approach expiry, without requiring a daemon or client restart. The daemon renews and validates staged registrar material before transactional publication; new handshakes get a complete replacement TLS configuration while existing connections remain intact. The client reloads a matching certificate/key pair for each new dial. Closes #768 --- CHANGELOG.md | 6 + docs/en/operations.md | 13 + docs/ko/operations.md | 12 + src/daemon.rs | 21 + src/fs_util.rs | 10 + src/registrar/endpoint.rs | 90 ++-- src/registrar/endpoint/client.rs | 97 +++- src/registrar/endpoint/client/tests.rs | 59 +++ src/registrar/endpoint/serve.rs | 6 +- src/registrar/endpoint/tests.rs | 75 ++- src/registrar/endpoint/tls.rs | 90 +++- src/registrar_certs.rs | 628 ++++++++++++++++++++++++- src/registrar_certs/tests.rs | 521 ++++++++++++++++++++ 13 files changed, 1519 insertions(+), 109 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac5fc0b2..5991c6d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,12 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm CA, and its certificates and keys survive the restart byte-identically. An already-expired leaf is repaired at start, before the endpoint's TLS material loads, rather than at the first renewal tick. +- An enabled registrar endpoint now renews both its server and registrar + client certificates on the daemon's regular maintenance cadence. The + endpoint applies renewed TLS material to new handshakes without + restarting its socket, while the in-repository registrar client reloads + its certificate and key for each new dial and safely retries the brief + atomic-replacement interval. - `bootroot-agent` now rotates `OpenBao`'s file audit device on a host whose registrar endpoint is enabled, so the deployment no longer needs an external rotator against it. Every 60 seconds the daemon renames the diff --git a/docs/en/operations.md b/docs/en/operations.md index b23bb1a5..fd3d037d 100644 --- a/docs/en/operations.md +++ b/docs/en/operations.md @@ -1389,6 +1389,19 @@ is then wrapped in mutual TLS. - **The verbs see `registrar-client:`** as the caller of record, and never a peer-credential value. +#### Caller certificate reload contract + +The registrar client certificate and key are renewed by two atomic +renames: the certificate is replaced first and the key second. A caller +must load both files for every new dial and verify that the key belongs to +the loaded leaf; it must never cache one half across dials or present a +mismatched pair. The in-repository registrar endpoint client is the +reference implementation: on a mismatch it reads the pair again up to +five times, waiting 1 ms, 2 ms, 4 ms, then 8 ms between the first four +failed reads, and returns a typed mismatch error if the fifth read is +still torn. Existing connections retain their TLS configuration; a new +dial picks up the next matching pair without a caller restart or signal. + A connection that opens and never sends a `ClientHello` is dropped after five seconds, so an unauthenticated peer cannot hold one of the sixteen connection slots open. diff --git a/docs/ko/operations.md b/docs/ko/operations.md index f7626815..166d298e 100644 --- a/docs/ko/operations.md +++ b/docs/ko/operations.md @@ -1321,6 +1321,18 @@ WantedBy=multi-user.target - **동사가 보는 호출자**는 `registrar-client:`이며, 피어 자격증명 값이 동사에 도달하는 일은 없습니다. +#### 호출자 인증서 리로드 계약 + +레지스트라 클라이언트 인증서와 키는 두 번의 atomic rename으로 갱신됩니다. +인증서가 먼저 교체되고 키가 뒤따릅니다. 호출자는 새 dial마다 두 파일을 다시 +읽고 키가 읽은 리프에 속하는지 확인해야 하며, dial 사이에 한쪽을 캐시하거나 +서로 맞지 않는 쌍을 제시해서는 안 됩니다. 이 저장소의 레지스트라 엔드포인트 +클라이언트가 기준 구현입니다. 불일치가 나면 최대 다섯 번 쌍을 다시 읽고, +앞의 네 번 실패 뒤에는 각각 1 ms, 2 ms, 4 ms, 8 ms를 기다립니다. 다섯 번째 +읽기에도 쌍이 찢어진 상태이면 형식화된 불일치 오류를 반환합니다. 이미 맺어진 +연결은 기존 TLS 구성을 유지하고, 새 dial은 호출자를 재시작하거나 신호를 +보내지 않아도 다음으로 일치하는 쌍을 사용합니다. + 연결을 열어 놓고 `ClientHello`를 보내지 않는 피어는 5초 뒤에 버려집니다. 인증되지 않은 피어가 16개의 연결 슬롯 중 하나를 계속 붙잡고 있을 수 없게 하기 위해서입니다. diff --git a/src/daemon.rs b/src/daemon.rs index 096eaef8..60c61234 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -214,6 +214,27 @@ pub(crate) async fn run_daemon(invocation: DaemonInvocation) -> anyhow::Result<( #[cfg(target_os = "linux")] let registrar_maintenance = if let Some((endpoint, built)) = registrar_service { + // Start-time issuance completed before activation. Initialize the + // renewal observation from that verified on-disk material here, + // before the accept task below can serve its first connection. + let renewal_plan = crate::registrar_certs::resolve_surface_plan(&settings)?; + let renewal_states = + crate::registrar_certs::initialize_renewal_states(&renewal_plan.pairs).await?; + endpoint.set_renewal_states(Arc::clone(&renewal_states)); + let renewal_endpoint = Arc::clone(&endpoint); + let renewal_settings = Arc::clone(&settings); + let renewal_shutdown = shutdown_rx.clone(); + handles.push(tokio::spawn(async move { + crate::registrar_certs::run_surface_renewal_loop( + renewal_settings, + renewal_endpoint, + renewal_plan, + renewal_states, + insecure_mode, + renewal_shutdown, + ) + .await + })); spawn_registrar_endpoint( &mut handles, endpoint, diff --git a/src/fs_util.rs b/src/fs_util.rs index a4ad9151..0cee03aa 100644 --- a/src/fs_util.rs +++ b/src/fs_util.rs @@ -1069,6 +1069,16 @@ impl FixedOwner { Self { uid: 0, gid: 0 } } + /// An already-observed owner that an atomic restore must re-establish. + /// + /// This remains crate-visible: configuration and external callers cannot + /// select an arbitrary owner for newly created files. + #[must_use] + #[cfg(target_os = "linux")] + pub(crate) const fn observed(uid: u32, gid: u32) -> Self { + Self { uid, gid } + } + /// The test process's own effective uid and gid. /// /// The one way a protected publish is driven under an owner other diff --git a/src/registrar/endpoint.rs b/src/registrar/endpoint.rs index 446ad746..dfbf5be9 100644 --- a/src/registrar/endpoint.rs +++ b/src/registrar/endpoint.rs @@ -108,7 +108,7 @@ mod tests; use std::os::unix::io::{FromRawFd as _, RawFd}; use std::path::PathBuf; -use std::sync::Arc; +use std::sync::{Arc, PoisonError, RwLock}; use std::time::Duration; use anyhow::Context as _; @@ -118,7 +118,6 @@ use tokio_rustls::TlsAcceptor; use tracing::{info, warn}; use self::activation::{ActivationContract, ActivationValues}; -use self::tls::EndpointCertResolver; use crate::config::Settings; /// Largest declared request payload the endpoint will read. @@ -217,14 +216,13 @@ pub(crate) struct ActivatedEndpoint { listener: UnixListener, socket_path: PathBuf, daemon_uid: u32, - acceptor: TlsAcceptor, - // The certificate resolver is retained for exactly one consumer, - // and that consumer — renewal for the endpoint leaf — is a sibling - // issue. It has to be retained now because there is no way back to - // the concrete resolver from a built `ServerConfig`, so a build - // that dropped it would leave renewal with nothing to swap - // through. Reachability is asserted through `cert_resolver`. - resolver: Arc, + // A complete acceptor is replaced after a renewal publishes every + // input it was built from. Replacing only the certificate resolver + // would leave the old client verifier installed. + acceptor: RwLock>, + // The renewal task updates this source of truth; the reporting work + // reads the same handle without parsing certificate files on requests. + renewal_states: RwLock>, domain: String, } @@ -245,24 +243,40 @@ impl ActivatedEndpoint { } /// Returns the acceptor every accepted connection is handed to. - pub(crate) fn tls_acceptor(&self) -> &TlsAcceptor { - &self.acceptor + pub(crate) fn tls_acceptor(&self) -> Arc { + Arc::clone(&self.acceptor.read().unwrap_or_else(PoisonError::into_inner)) } - /// Returns the resolver that decides which certificate the next - /// handshake presents. + /// Exchanges the acceptor used by future handshakes. /// - /// This is the whole of the renewal seam. [`rustls::ServerConfig`] - /// keeps only an `Arc`, and the trait does - /// not extend `Any`, so code holding the configuration or the - /// acceptor cannot reach the concrete resolver at all — it survives - /// here or nowhere. - // The renewal work that calls this is a sibling issue, so until it - // lands nothing outside a test reaches the accessor. It exists now - // so that work needs nothing from this one. + /// Existing handshakes retain the clone they loaded before this + /// exchange and therefore continue under their original TLS policy. + pub(crate) fn replace_server_config(&self, config: Arc) { + let mut acceptor = self + .acceptor + .write() + .unwrap_or_else(PoisonError::into_inner); + *acceptor = Arc::new(TlsAcceptor::from(config)); + } + + /// Installs the daemon-owned registrar certificate renewal observations. + pub(crate) fn set_renewal_states(&self, states: crate::registrar_certs::SurfaceRenewalStates) { + let mut slot = self + .renewal_states + .write() + .unwrap_or_else(PoisonError::into_inner); + *slot = Some(states); + } + + /// Returns the daemon-owned registrar certificate renewal observations. + // The reporting child reads this accessor once its response member lands; + // renewal owns the state now so it is not recomputed on a request path. #[allow(dead_code)] - pub(crate) fn cert_resolver(&self) -> &Arc { - &self.resolver + pub(crate) fn renewal_states(&self) -> Option { + self.renewal_states + .read() + .unwrap_or_else(PoisonError::into_inner) + .clone() } /// Returns the configured `network.domain` the presented client @@ -270,6 +284,24 @@ impl ActivatedEndpoint { pub(crate) fn domain(&self) -> &str { &self.domain } + + /// Creates an endpoint shell for renewal tests. + #[cfg(test)] + pub(crate) fn for_renewal_test( + listener: UnixListener, + socket_path: PathBuf, + server_config: Arc, + domain: String, + ) -> Arc { + Arc::new(Self { + listener, + socket_path, + daemon_uid: current_effective_uid(), + acceptor: RwLock::new(Arc::new(TlsAcceptor::from(server_config))), + renewal_states: RwLock::new(None), + domain, + }) + } } impl std::fmt::Debug for ActivatedEndpoint { @@ -346,7 +378,7 @@ pub(crate) fn activate(settings: &Settings) -> anyhow::Result anyhow::Result anyhow::Result, - resolver: Arc, domain: String, ) -> anyhow::Result> { let fd = contract.into_descriptor(); @@ -431,8 +461,8 @@ pub(crate) fn adopt( listener, socket_path, daemon_uid: effective_uid, - acceptor: TlsAcceptor::from(server_config), - resolver, + acceptor: RwLock::new(Arc::new(TlsAcceptor::from(server_config))), + renewal_states: RwLock::new(None), domain, })) } diff --git a/src/registrar/endpoint/client.rs b/src/registrar/endpoint/client.rs index 397fe2c5..a8ac5d52 100644 --- a/src/registrar/endpoint/client.rs +++ b/src/registrar/endpoint/client.rs @@ -96,6 +96,20 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); /// deadline. const READ_TIMEOUT: Duration = Duration::from_secs(30); +/// The bounded reader-side half of the certificate/key publication +/// contract. Writers rename the certificate and then the key; a dial in +/// that interval retries this many complete reads before reporting a +/// persistent mismatch. +const PAIR_LOAD_ATTEMPTS: usize = 5; + +/// Delays after the first four mismatches in a torn pair read. +const PAIR_LOAD_DELAYS: [Duration; PAIR_LOAD_ATTEMPTS - 1] = [ + Duration::from_millis(1), + Duration::from_millis(2), + Duration::from_millis(4), + Duration::from_millis(8), +]; + /// The name handed to [`TlsConnector::connect`]. /// /// Inert. [`endpoint_pin::RegistrarEndpointVerifier`] deliberately @@ -189,6 +203,19 @@ pub(crate) enum ClientMaterialError { /// The path that was handed to the client. path: PathBuf, }, + /// The certificate and key were both readable but belong to + /// different generations after every bounded retry. + #[error( + "registrar client certificate {} and key {} do not form a matching pair", + .certificate_path.display(), + .key_path.display() + )] + KeyMismatch { + /// The certificate path that was read. + certificate_path: PathBuf, + /// The key path that was read. + key_path: PathBuf, + }, /// The pair parsed but `rustls` will not authenticate with it. #[error( "registrar client certificate {} and key {} are not usable client authentication \ @@ -761,10 +788,8 @@ pub(crate) fn build_client_config( /// Loads the registrar client leaf and its key from disk. /// /// The single seam every dial's material comes through. It does the -/// plain load and nothing more: it does not verify that the key matches -/// the leaf, does not retry a pair that is momentarily torn by a -/// renewal, and does not inspect `notAfter`. Later work that owns those -/// reasons adds them here rather than restructuring the dial path. +/// checks the key against the leaf and retries the short certificate/key +/// rename interval. It does not inspect `notAfter`. /// /// # Errors /// @@ -773,6 +798,53 @@ pub(crate) fn build_client_config( fn load_client_material( certificate_path: &Path, key_path: &Path, +) -> Result<(Vec>, PrivateKeyDer<'static>), ClientMaterialError> { + load_client_material_with( + || load_client_material_once(certificate_path, key_path), + std::thread::sleep, + ) +} + +/// Applies the bounded torn-pair retry policy around one complete material +/// read. Keeping the delay operation injectable makes the fixed reader-side +/// contract deterministic without changing the production filesystem path. +fn load_client_material_with( + mut load_once: Load, + mut sleep: Sleep, +) -> Result<(Vec>, PrivateKeyDer<'static>), ClientMaterialError> +where + Load: FnMut() -> Result< + (Vec>, PrivateKeyDer<'static>), + ClientMaterialError, + >, + Sleep: FnMut(Duration), +{ + for (attempt, delay) in PAIR_LOAD_DELAYS + .iter() + .copied() + .map(Some) + .chain(std::iter::once(None)) + .enumerate() + { + match load_once() { + Ok(pair) => return Ok(pair), + Err(error @ ClientMaterialError::KeyMismatch { .. }) if delay.is_some() => { + sleep(delay.expect("retry delay exists before the final attempt")); + tracing::debug!( + attempt, + "Registrar client pair changed during a per-dial load; retrying." + ); + let _ = error; + } + Err(error) => return Err(error), + } + } + unreachable!("the fixed pair-load attempt sequence always has a final attempt") +} + +fn load_client_material_once( + certificate_path: &Path, + key_path: &Path, ) -> Result<(Vec>, PrivateKeyDer<'static>), ClientMaterialError> { let certificate_bytes = std::fs::read(certificate_path).map_err(|source| { if source.kind() == io::ErrorKind::NotFound { @@ -828,6 +900,23 @@ fn load_client_material( path: key_path.to_path_buf(), })?; + let leaf = chain + .first() + .ok_or_else(|| ClientMaterialError::NoCertificate { + path: certificate_path.to_path_buf(), + })?; + // An unsupported-but-well-formed key remains this loader's existing + // `Unusable` error when `rustls` builds the client configuration. + // Only a key this build can use is eligible for the torn-pair retry. + if let Ok(signing_key) = rustls::crypto::ring::sign::any_supported_type(&key) + && !crate::tls::cert_key_matches(leaf, signing_key.as_ref()) + { + return Err(ClientMaterialError::KeyMismatch { + certificate_path: certificate_path.to_path_buf(), + key_path: key_path.to_path_buf(), + }); + } + Ok((chain, key)) } diff --git a/src/registrar/endpoint/client/tests.rs b/src/registrar/endpoint/client/tests.rs index 2a5e6dbb..81654492 100644 --- a/src/registrar/endpoint/client/tests.rs +++ b/src/registrar/endpoint/client/tests.rs @@ -33,6 +33,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex as StdMutex}; +use std::time::Duration; use rustls::server::WebPkiClientVerifier; use rustls::{RootCertStore, ServerConfig}; @@ -717,6 +718,64 @@ async fn a_key_that_is_not_a_private_key_fails_before_the_dial() { assert_eq!(double.observed().connections, 0); } +#[test] +fn a_permanent_torn_pair_returns_the_typed_mismatch_error() { + let deployment = Deployment::new(); + let (_other_leaf, other_key) = + issue_leaf(&deployment.pki.ca, vec![dns_san(®istrar_client_name())]); + std::fs::write(&deployment.key_path, other_key.serialize_pem()) + .expect("write a different private key beside the client leaf"); + + let mut reads = 0; + let mut observed_delays = Vec::new(); + let error = load_client_material_with( + || { + reads += 1; + load_client_material_once(&deployment.certificate_path, &deployment.key_path) + }, + |delay| observed_delays.push(delay), + ) + .expect_err("a permanent torn pair must not be presented"); + assert!( + matches!(error, ClientMaterialError::KeyMismatch { .. }), + "expected the typed torn-pair error, got {error:?}" + ); + assert_eq!(reads, PAIR_LOAD_ATTEMPTS); + assert_eq!(observed_delays, PAIR_LOAD_DELAYS); +} + +#[test] +fn a_torn_pair_retries_until_the_second_rename_completes() { + let deployment = Deployment::new(); + let original_key = std::fs::read(&deployment.key_path).expect("read the original key"); + let (_other_leaf, other_key) = + issue_leaf(&deployment.pki.ca, vec![dns_san(®istrar_client_name())]); + std::fs::write(&deployment.key_path, other_key.serialize_pem()) + .expect("write a torn pair after the certificate rename"); + + let mut reads = 0; + let mut observed_delays = Vec::new(); + let material = load_client_material_with( + || { + reads += 1; + if reads == 3 { + std::fs::write(&deployment.key_path, &original_key) + .expect("complete the key rename before the third read"); + } + load_client_material_once(&deployment.certificate_path, &deployment.key_path) + }, + |delay| observed_delays.push(delay), + ) + .expect("the matching pair after the second rename loads"); + + assert!(!material.0.is_empty()); + assert_eq!(reads, 3); + assert_eq!( + observed_delays, + [Duration::from_millis(1), Duration::from_millis(2)] + ); +} + #[tokio::test] async fn a_pem_key_rustls_will_not_load_fails_before_the_dial() { let deployment = Deployment::new(); diff --git a/src/registrar/endpoint/serve.rs b/src/registrar/endpoint/serve.rs index fb4102d8..c6bde590 100644 --- a/src/registrar/endpoint/serve.rs +++ b/src/registrar/endpoint/serve.rs @@ -381,7 +381,11 @@ async fn handshake( accepted_at: Instant, ) -> Option> { let deadline = accepted_at + HANDSHAKE_TIMEOUT; - match timeout_at(deadline, endpoint.tls_acceptor().accept(stream)).await { + // Load immediately before the handshake. The lock is released before + // awaiting, and a successful renewal therefore affects the next + // handshake without disturbing one already in progress. + let acceptor = endpoint.tls_acceptor(); + match timeout_at(deadline, acceptor.accept(stream)).await { Ok(Ok(tls)) => Some(tls), Ok(Err(err)) => { warn!( diff --git a/src/registrar/endpoint/tests.rs b/src/registrar/endpoint/tests.rs index 7f58c5a8..fb258bbb 100644 --- a/src/registrar/endpoint/tests.rs +++ b/src/registrar/endpoint/tests.rs @@ -64,8 +64,8 @@ use super::test_support::{ endpoint_name, generate_ca, registrar_client_name, valid_ca, }; use super::tls::{ - CA_BUNDLE_SETTING, EndpointCertResolver, EndpointTlsError, SERVER_CERT_SETTING, - SERVER_KEY_SETTING, TRUSTED_CA_SETTING, build_server_config, + CA_BUNDLE_SETTING, EndpointTlsError, SERVER_CERT_SETTING, SERVER_KEY_SETTING, + TRUSTED_CA_SETTING, build_server_config, }; use super::{ ActivatedEndpoint, CONNECTION_DRAIN_TIMEOUT, HANDSHAKE_TIMEOUT, HEADER_IDLE_TIMEOUT, @@ -1683,15 +1683,11 @@ impl Harness { } fn bind_over(pki: Pki) -> anyhow::Result { - let (config, resolver) = pki.conforming(); - Self::bind_with(pki, config, resolver) + let (config, _) = pki.conforming(); + Self::bind_with(pki, config) } - fn bind_with( - pki: Pki, - config: Arc, - resolver: Arc, - ) -> anyhow::Result { + fn bind_with(pki: Pki, config: Arc) -> anyhow::Result { let dir = conforming_tempdir()?; let socket_path = dir.path().join("registrar.sock"); let listener = std::os::unix::net::UnixListener::bind(&socket_path)?; @@ -1703,7 +1699,6 @@ impl Harness { activation_descriptor(listener), current_effective_uid(), config, - resolver, TEST_DOMAIN.to_string(), )?; Ok(Self { @@ -1743,14 +1738,8 @@ fn adopt_for_test( effective_uid: u32, ) -> anyhow::Result> { let pki = Pki::new(); - let (config, resolver) = pki.conforming(); - super::adopt( - contract, - effective_uid, - config, - resolver, - TEST_DOMAIN.to_string(), - ) + let (config, _) = pki.conforming(); + super::adopt(contract, effective_uid, config, TEST_DOMAIN.to_string()) } /// Runs one accept loop until the returned sender is told to stop, and @@ -2835,20 +2824,19 @@ fn the_activated_endpoint_carries_no_handler() { !fields.iter().any(|field| field.contains("handler")), "the adopted endpoint holds no handler: {fields:?}" ); - assert_eq!( - fields, - vec![ - "listener: UnixListener,", - "socket_path: PathBuf,", - "daemon_uid: u32,", - "acceptor: TlsAcceptor,", - "// The certificate resolver is retained for exactly one consumer,", - "resolver: Arc,", - "domain: String,", - ], - "the adopted endpoint holds the socket, its path, the daemon uid and the TLS material — \ - and no handler" - ); + for expected in [ + "listener: UnixListener,", + "socket_path: PathBuf,", + "daemon_uid: u32,", + "acceptor: RwLock>,", + "renewal_states: RwLock>", + "domain: String,", + ] { + assert!( + fields.iter().any(|field| field.contains(expected)), + "the adopted endpoint must retain {expected}; fields: {fields:?}" + ); + } assert!( !source.contains("fn production_handler"), "the missing-handler refusal is gone, not relocated" @@ -3228,12 +3216,11 @@ async fn a_mismatched_peer_is_refused_even_with_the_registrar_certificate() { ) .expect("chmod"); let pki = Pki::new(); - let (config, resolver) = pki.conforming(); + let (config, _) = pki.conforming(); let endpoint = super::adopt( activation_descriptor(listener), current_effective_uid(), config, - resolver, TEST_DOMAIN.to_string(), ) .expect("adoption"); @@ -3349,17 +3336,17 @@ async fn a_pinned_caller_is_served_and_an_unpinned_one_refuses_before_a_request( running.stop().await; } -/// The renewal seam is reachable from what the daemon holds. +/// The complete renewal seam is reachable from what the daemon holds. /// -/// The resolver is taken from the [`ActivatedEndpoint`] the harness -/// produced — not from one the test built — because that is the whole -/// point: a test that swapped its own resolver would pass while renewal -/// remained unable to reach one. After the swap the endpoint presents a +/// The replacement configuration is installed through the +/// [`ActivatedEndpoint`] the harness produced — not merely built beside +/// it — because renewal must replace the client verifier as well as the +/// presented certificate. After the exchange the endpoint presents a /// chain under a different anchor, which the caller pinned to the first /// anchor now refuses and a caller pinned to the second now accepts, /// with no restart in between. #[tokio::test] -async fn swapping_the_resolver_through_the_activated_endpoint_changes_the_presented_chain() { +async fn replacing_the_acceptor_through_the_activated_endpoint_changes_the_presented_chain() { let harness = Harness::bind().expect("harness"); let running = RunningEndpoint::start( &harness.endpoint, @@ -3374,10 +3361,8 @@ async fn swapping_the_resolver_through_the_activated_endpoint_changes_the_presen // A second deployment PKI, and material for the same endpoint name // under its anchor. let renewed = Pki::new(); - let (cert_path, key_path) = renewed.server_material(); - let certified_key = super::tls::load_certified_key(&cert_path, &key_path) - .expect("the renewed material must load"); - harness.endpoint.cert_resolver().swap(certified_key); + let (renewed_config, _) = renewed.conforming(); + harness.endpoint.replace_server_config(renewed_config); // The caller pinned to the first anchor no longer accepts what is // presented, and the caller pinned to the second one does. @@ -3389,7 +3374,7 @@ async fn swapping_the_resolver_through_the_activated_endpoint_changes_the_presen let config = client_config_pinning( &renewed.pin_file_path(), - Some(harness.pki.registrar_client_material()), + Some(renewed.registrar_client_material()), ); let served = tls_round_trip(&harness.socket_path, config, &frame_of(b"mint", b"")).await; assert!( diff --git a/src/registrar/endpoint/tls.rs b/src/registrar/endpoint/tls.rs index b3be3880..41fa6e78 100644 --- a/src/registrar/endpoint/tls.rs +++ b/src/registrar/endpoint/tls.rs @@ -31,13 +31,11 @@ //! the outbound path applies. It is never built with //! `allow_unauthenticated()`: a caller presenting no certificate //! fails the handshake rather than arriving unauthenticated. -//! - **The swap seam.** [`EndpointCertResolver`] holds the presented -//! [`CertifiedKey`] behind an `RwLock` and swaps it under the next -//! handshake with no restart. [`rustls::ServerConfig`] keeps only an -//! `Arc` and the trait does not extend `Any`, -//! so the typed handle is returned alongside the configuration and -//! retained by [`super::ActivatedEndpoint`]; there is no way back to -//! it from a built configuration. +//! - **The swap seam.** A complete [`ServerConfig`] is built before +//! publication and exchanged by [`super::ActivatedEndpoint`] only +//! after every live file write succeeds. This replaces both the +//! presented key and the incoming client verifier for the next +//! handshake without disturbing an established connection. //! //! # Where a refusal is decided //! @@ -68,6 +66,10 @@ pub(crate) const SERVER_CERT_SETTING: &str = "[registrar_endpoint] server_cert_p /// diagnostic. pub(crate) const SERVER_KEY_SETTING: &str = "[registrar_endpoint] server_key_path"; +/// How the `[registrar_endpoint]` client-certificate key is spelled in a +/// diagnostic. +pub(crate) const CLIENT_CERT_SETTING: &str = "[registrar_endpoint] client_cert_path"; + /// How the trust bundle setting is spelled in a diagnostic. pub(crate) const CA_BUNDLE_SETTING: &str = "trust.ca_bundle_path"; @@ -233,10 +235,19 @@ pub(crate) enum EndpointTlsError { /// What the verifier builder reported. detail: String, }, + /// The registrar client leaf would not authenticate to the endpoint. + #[error("{setting} at {} is not a valid registrar client certificate: {detail}", .path.display())] + ClientCertificate { + /// The setting at fault. + setting: &'static str, + /// The staged client certificate path. + path: PathBuf, + /// What the verifier reported. + detail: String, + }, } -/// The endpoint's server-certificate resolver, and the one point a -/// renewal reaches to replace the presented material. +/// The endpoint's server-certificate resolver. /// /// The `RwLock` is taken and released inside the synchronous `resolve` /// and `swap`, so no guard is ever held across an `.await`. A poisoned @@ -258,10 +269,8 @@ impl EndpointCertResolver { /// Replaces the certificate and key presented from the next /// handshake onwards, with no restart and without disturbing a /// connection already established. - // Certificate renewal for the endpoint leaf is a sibling issue, so - // until it lands the only caller of the seam is the test that - // proves it is reachable from the `ActivatedEndpoint` the daemon - // holds. + // Kept for endpoint unit tests. Production renewal replaces the + // complete acceptor so that its client verifier changes too. #[allow(dead_code)] pub(crate) fn swap(&self, certified_key: CertifiedKey) { let mut guard = self @@ -361,6 +370,61 @@ pub(crate) fn build_server_config( Ok((Arc::new(config), resolver)) } +/// Verifies staged registrar client material against the incoming mTLS rule. +/// +/// This is the same pinned subset of the staged CA bundle that +/// [`build_server_config`] installs for incoming handshakes. Running it before +/// publication refuses an expired certificate or one without `clientAuth`, +/// rather than leaving the next registrar dial to discover it. +/// +/// # Errors +/// +/// Returns an error if no configured pin is present in the bundle, the pinned +/// subset cannot construct a verifier, or the client leaf is not currently +/// valid client-authentication material under that verifier. +pub(crate) fn validate_client_certificate( + certified_key: &CertifiedKey, + cert_path: &Path, + bundle_path: &Path, + pins: &[String], +) -> Result<(), EndpointTlsError> { + tls::install_crypto_provider(); + + if pins.is_empty() { + return Err(EndpointTlsError::MissingSetting { + setting: TRUSTED_CA_SETTING, + }); + } + let pin_set: HashSet = pins.iter().map(|pin| pin.to_ascii_lowercase()).collect(); + let anchors = pinned_bundle_anchors(bundle_path, &pin_set)?; + let roots = + tls::certs_to_root_store(&anchors).map_err(|err| EndpointTlsError::ClientVerifier { + setting: CA_BUNDLE_SETTING, + path: bundle_path.to_path_buf(), + detail: format!("{err:#}"), + })?; + let verifier = WebPkiClientVerifier::builder(Arc::new(roots)) + .build() + .map_err(|err| EndpointTlsError::ClientVerifier { + setting: CA_BUNDLE_SETTING, + path: bundle_path.to_path_buf(), + detail: err.to_string(), + })?; + let leaf = leaf_of(certified_key, cert_path)?; + verifier + .verify_client_cert( + leaf, + certified_key.cert.get(1..).unwrap_or_default(), + UnixTime::now(), + ) + .map_err(|err| EndpointTlsError::ClientCertificate { + setting: CLIENT_CERT_SETTING, + path: cert_path.to_path_buf(), + detail: err.to_string(), + })?; + Ok(()) +} + /// Reads the loaded leaf's single DNS SAN and holds it to the endpoint /// server name rule in the configured domain. /// diff --git a/src/registrar_certs.rs b/src/registrar_certs.rs index eb64aa72..8d327f3f 100644 --- a/src/registrar_certs.rs +++ b/src/registrar_certs.rs @@ -42,10 +42,10 @@ //! that process a new key on every restart. The two pairs are evaluated //! independently. //! -//! This is not a scheduler. There is no registration point, no lead-time -//! constant and no retry policy, and nothing here changes how the -//! per-service loop or the internal profile's own renewal is scheduled, -//! credentialed or triggered. +//! The start-time path is not a scheduler. The daemon-owned renewal adapter +//! below registers the same two leaves separately, using the rendered +//! internal profile's cadence, lead time and retry policy; it does not alter +//! the per-service loop or the internal profile's own renewal process. //! //! # Why this is not a module of [`crate::registrar`] //! @@ -78,8 +78,20 @@ //! strictly stronger than this one. use std::path::{Path, PathBuf}; +#[cfg(target_os = "linux")] +use std::{ + collections::BTreeMap, + future::Future, + os::unix::fs::{MetadataExt as _, PermissionsExt as _}, + sync::{Arc, Mutex}, + time::Duration, +}; use anyhow::{Context, Result}; +#[cfg(target_os = "linux")] +use tokio::sync::watch; +#[cfg(target_os = "linux")] +use tracing::error; use tracing::{info, warn}; use x509_parser::prelude::ASN1Time; @@ -98,6 +110,60 @@ use crate::registrar::{ use crate::secret::HmacSecret; use crate::{cert_chain, tls}; +/// The outcome recorded for one registrar-surface renewal attempt. +#[cfg(target_os = "linux")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum RenewalAttempt { + /// No renewal has been attempted since daemon start. + NeverAttempted, + /// The most recent attempt published a replacement. + Succeeded, + /// The most recent attempt failed before a replacement was active. + Failed(String), +} + +/// In-process observation of one registrar-surface leaf. +#[cfg(target_os = "linux")] +#[derive(Debug, Clone)] +pub(crate) struct SurfaceRenewalState { + /// The currently active certificate expiry time. + pub(crate) not_after: time::OffsetDateTime, + /// The most recent attempt outcome. + pub(crate) attempt: RenewalAttempt, + /// The time the most recent attempt started, if there was one. + pub(crate) attempted_at: Option, +} + +/// The sole in-process source of registrar leaf renewal observations. +#[cfg(target_os = "linux")] +pub(crate) type SurfaceRenewalStates = Arc>>; + +/// A staged registrar private key whose debug representation is redacted. +#[cfg(target_os = "linux")] +struct SurfacePrivateKeyPem(String); + +#[cfg(target_os = "linux")] +impl SurfacePrivateKeyPem { + /// Wraps key material as it enters the renewal publication path. + #[must_use] + fn new(pem: String) -> Self { + Self(pem) + } + + /// Borrows the key only for the established certificate/key writer. + #[must_use] + fn expose(&self) -> &str { + &self.0 + } +} + +#[cfg(target_os = "linux")] +impl std::fmt::Debug for SurfacePrivateKeyPem { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("") + } +} + /// The KV v2 path the deployment's shared agent EAB is stored at. /// /// Read here through the internal credential, whose policy already @@ -126,7 +192,7 @@ const SURFACE_LEAF_PUBLICATION: LeafPublication = LeafPublication::LeafWithChain /// The two are evaluated and issued independently: one being usable is /// never a reason to leave the other unusable, and one needing issuance /// is never a reason to re-issue the other. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub(crate) enum SurfaceLeaf { /// The leaf the endpoint presents. A server certificate, so it takes /// the ordinary CSR shape and requests no extended key usage. @@ -567,6 +633,15 @@ pub(crate) struct SurfacePlan { pub(crate) kv_mount: String, /// The bootroot host's own label, from the rendered internal config. pub(crate) host: String, + /// The rendered internal credential profile. Its daemon and retry + /// settings are the registrar surface renewal policy. + #[cfg(target_os = "linux")] + pub(crate) renewal_profile: DaemonProfileSettings, + /// The internal credential configuration's effective issuance retry + /// policy. The profile may override it; otherwise its own top-level + /// `[retry]` applies, never the outer daemon configuration. + #[cfg(target_os = "linux")] + pub(crate) renewal_retry_backoff: Vec, /// Both configured pairs, in evaluation order. pub(crate) pairs: Vec, } @@ -608,17 +683,22 @@ pub(crate) fn resolve_surface_plan(settings: &Settings) -> Result { internal_paths.agent_config().display() ) })?; - let host = internal - .profiles - .first() - .map(|profile| profile.hostname.clone()) - .ok_or_else(|| { - anyhow::anyhow!( - "the rendered internal agent config at {} carries no profile to take the host \ + let internal_profile = internal.profiles.first().cloned().ok_or_else(|| { + anyhow::anyhow!( + "the rendered internal agent config at {} carries no profile to take the host \ label from", - internal_paths.agent_config().display() - ) - })?; + internal_paths.agent_config().display() + ) + })?; + let host = internal_profile.hostname.clone(); + #[cfg(target_os = "linux")] + let (renewal_profile, renewal_retry_backoff) = ( + internal_profile.clone(), + internal_profile.retry.as_ref().map_or_else( + || internal.retry.backoff_secs.clone(), + |retry| retry.backoff_secs.clone(), + ), + ); let pairs = surface_pairs(&settings.registrar_endpoint, &host, &settings.domain)?; Ok(SurfacePlan { @@ -626,10 +706,511 @@ pub(crate) fn resolve_surface_plan(settings: &Settings) -> Result { openbao_url: state.openbao_url, kv_mount: state.kv_mount, host, + #[cfg(target_os = "linux")] + renewal_profile, + #[cfg(target_os = "linux")] + renewal_retry_backoff, pairs, }) } +/// Initializes the shared state after start-time issuance and before +/// the endpoint accept task can begin serving. +#[cfg(target_os = "linux")] +pub(crate) async fn initialize_renewal_states( + pairs: &[SurfacePairPaths], +) -> Result { + let mut entries = BTreeMap::new(); + for pair in pairs { + let bytes = tokio::fs::read(&pair.cert_path).await.with_context(|| { + format!( + "reading registrar certificate at {}", + pair.cert_path.display() + ) + })?; + let not_after = crate::daemon::parse_cert_not_after(&bytes)?; + entries.insert( + pair.leaf, + SurfaceRenewalState { + not_after, + attempt: RenewalAttempt::NeverAttempted, + attempted_at: None, + }, + ); + } + Ok(Arc::new(Mutex::new(entries))) +} + +/// Runs the registrar leaves on the internal credential profile's +/// cadence. It is deliberately independent of ordinary service profiles: +/// this path authenticates using the internal certificate and never reads +/// `AppRole` material. +#[cfg(target_os = "linux")] +pub(crate) async fn run_surface_renewal_loop( + settings: Arc, + endpoint: Arc, + plan: SurfacePlan, + states: SurfaceRenewalStates, + insecure_mode: bool, + mut shutdown: watch::Receiver, +) -> Result<()> { + let profile = plan.renewal_profile.clone(); + let mut first_tick = true; + loop { + if *shutdown.borrow_and_update() { + return Ok(()); + } + let delay = if first_tick { + first_tick = false; + Duration::ZERO + } else { + crate::utils::jittered_delay(profile.daemon.check_interval, profile.daemon.check_jitter) + }; + tokio::select! { + _ = shutdown.changed() => return Ok(()), + () = tokio::time::sleep(delay) => {} + } + let renewal_settings = Arc::clone(&settings); + let renewal_plan = plan.clone(); + let renewal_endpoint = Arc::clone(&endpoint); + run_surface_renewal_pass(&settings, &plan, &states, move |pair| { + let settings = Arc::clone(&renewal_settings); + let plan = renewal_plan.clone(); + let endpoint = Arc::clone(&renewal_endpoint); + async move { renew_surface_pair(&settings, &plan, &pair, &endpoint, insecure_mode).await } + }) + .await; + } +} + +/// Drives one deterministic registrar renewal pass. +/// +/// The loop above owns cadence and shutdown; this unit owns per-leaf +/// eligibility and state transitions so tests can exercise a pass without +/// waiting for its interval. The closure retains the issuance and publication +/// boundary, leaving a no-op pass unable to reach `OpenBao` or the CA. +#[cfg(target_os = "linux")] +async fn run_surface_renewal_pass( + settings: &Settings, + plan: &SurfacePlan, + states: &SurfaceRenewalStates, + mut renew: Renew, +) where + Renew: FnMut(SurfacePairPaths) -> RenewFuture, + RenewFuture: Future>, +{ + for pair in &plan.pairs { + let mut eligibility = plan.renewal_profile.clone(); + eligibility.paths.cert = pair.cert_path.clone(); + eligibility.paths.key = pair.key_path.clone(); + let due = match crate::daemon::should_renew( + &eligibility, + &settings.trust, + plan.renewal_profile.daemon.renew_before, + ) + .await + { + Ok(due) => due, + Err(err) => { + // This is an eligibility check, not a renewal attempt. + // Preserve the last attempt observation until issuance + // actually starts on a later tick. + warn!(leaf = %pair.name, "Checking registrar renewal eligibility failed: {err:#}"); + continue; + } + }; + if !due { + continue; + } + let attempted_at = time::OffsetDateTime::now_utc(); + match renew(pair.clone()).await { + Ok(not_after) => record_success(states, pair.leaf, not_after, attempted_at), + Err(err) => { + error!(leaf = %pair.name, "Registrar surface renewal failed: {err:#}"); + record_failure_at(states, pair.leaf, attempted_at, err.to_string()); + } + } + } +} + +#[cfg(target_os = "linux")] +fn record_success( + states: &SurfaceRenewalStates, + leaf: SurfaceLeaf, + not_after: time::OffsetDateTime, + attempted_at: time::OffsetDateTime, +) { + let mut states = states + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(state) = states.get_mut(&leaf) { + state.not_after = not_after; + state.attempt = RenewalAttempt::Succeeded; + state.attempted_at = Some(attempted_at); + } +} + +#[cfg(target_os = "linux")] +fn record_failure_at( + states: &SurfaceRenewalStates, + leaf: SurfaceLeaf, + attempted_at: time::OffsetDateTime, + reason: String, +) { + let mut states = states + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(state) = states.get_mut(&leaf) { + state.attempt = RenewalAttempt::Failed(reason); + state.attempted_at = Some(attempted_at); + } +} + +#[cfg(target_os = "linux")] +async fn renew_surface_pair( + settings: &Settings, + plan: &SurfacePlan, + pair: &SurfacePairPaths, + endpoint: &Arc, + insecure_mode: bool, +) -> Result { + let bundle_path = settings.trust.ca_bundle_path.as_deref().ok_or_else(|| { + anyhow::anyhow!("trust.ca_bundle_path is required for an enabled registrar endpoint") + })?; + let staging = + tempfile::tempdir().context("creating private registrar renewal staging directory")?; + let candidate = SurfacePairPaths { + leaf: pair.leaf, + cert_path: staging.path().join("certificate.pem"), + key_path: staging.path().join("key.pem"), + name: pair.name.clone(), + }; + let candidate_bundle = staging.path().join("ca-bundle.pem"); + let live_bundle = tokio::fs::read_to_string(bundle_path) + .await + .with_context(|| { + format!( + "reading CA bundle at {} before registrar renewal", + bundle_path.display() + ) + })?; + crate::fs_util::write_ca_bundle( + &candidate_bundle, + &live_bundle, + crate::cert_group::CertGroupPolicy::none(), + ) + .await + .context("seeding staged registrar CA bundle")?; + + let inputs = read_acme_inputs(&plan.secrets_dir, &plan.openbao_url, &plan.kv_mount).await?; + // The ACME flow takes its output paths from the single profile in the + // settings it receives. Build that profile from the staged pair, rather + // than cloning the daemon settings, so issuance cannot reach a live + // registrar pair before validation and publication have completed. + let mut candidate_settings = + issuance_settings(settings, &candidate, &plan.host, &inputs.responder_hmac); + candidate_settings.trust.ca_bundle_path = Some(candidate_bundle.clone()); + crate::daemon::issue_with_retry_inner( + || issue_surface_candidate(&candidate_settings, &candidate, &inputs, insecure_mode), + tokio::time::sleep, + &plan.renewal_retry_backoff, + ) + .await?; + + let client_pair = plan + .pairs + .iter() + .find(|configured| configured.leaf == SurfaceLeaf::RegistrarClient) + .ok_or_else(|| anyhow::anyhow!("registrar renewal plan has no client certificate pair"))?; + validate_candidate( + &candidate, + &candidate_bundle, + client_pair, + &settings.trust.trusted_ca_sha256, + )?; + let server_pair = if pair.leaf == SurfaceLeaf::EndpointServer { + (&candidate.cert_path, &candidate.key_path) + } else { + let current_server = plan + .pairs + .iter() + .find(|configured| configured.leaf == SurfaceLeaf::EndpointServer) + .ok_or_else(|| { + anyhow::anyhow!("registrar renewal plan has no server certificate pair") + })?; + (¤t_server.cert_path, ¤t_server.key_path) + }; + let (next_config, _) = crate::registrar::endpoint::tls::build_server_config( + Some(server_pair.0), + Some(server_pair.1), + Some(&candidate_bundle), + &settings.trust.trusted_ca_sha256, + &settings.domain, + ) + .context("building staged registrar endpoint TLS configuration")?; + + let snapshot = RenewalSnapshot::capture(bundle_path, pair).await?; + publish_surface_renewal_transaction( + endpoint, + next_config, + || publish_candidate(bundle_path, pair, &candidate, &candidate_bundle), + || snapshot.restore(), + ) + .await?; + let bytes = tokio::fs::read(&pair.cert_path).await?; + crate::daemon::parse_cert_not_after(&bytes) +} + +/// Publishes a fully validated registrar renewal and exchanges its already +/// built TLS configuration only after publication succeeds. +/// +/// The two operations are parameters so the transaction's failure and +/// rollback paths remain deterministically testable without weakening the +/// production publication primitive. +#[cfg(target_os = "linux")] +async fn publish_surface_renewal_transaction( + endpoint: &crate::registrar::endpoint::ActivatedEndpoint, + next_config: Arc, + publish: Publish, + restore: Restore, +) -> Result<()> +where + Publish: FnOnce() -> PublishFuture, + PublishFuture: Future>, + Restore: FnOnce() -> RestoreFuture, + RestoreFuture: Future>, +{ + if let Err(publish_error) = publish().await { + return match restore().await { + Ok(()) => { + Err(publish_error.context("publishing registrar renewal; restored prior material")) + } + Err(rollback_error) => Err(publish_error.context(format!( + "publishing registrar renewal; rollback also failed: {rollback_error:#}" + ))), + }; + } + endpoint.replace_server_config(next_config); + Ok(()) +} + +#[cfg(target_os = "linux")] +fn validate_candidate( + candidate: &SurfacePairPaths, + bundle: &Path, + client_pair: &SurfacePairPaths, + trusted_ca_sha256: &[String], +) -> Result<()> { + let certified = crate::registrar::endpoint::tls::load_certified_key( + &candidate.cert_path, + &candidate.key_path, + ) + .context("loading staged registrar certificate and key")?; + if pair_name(&certified, &candidate.cert_path)? != candidate.name { + anyhow::bail!("staged registrar certificate does not carry the expected identity"); + } + if candidate.leaf == SurfaceLeaf::EndpointServer { + let pins = crate::registrar::endpoint_pin::load_anchor_pins( + &crate::registrar::endpoint_pin::anchor_pin_path_for_client_certificate( + &client_pair.cert_path, + ), + )?; + let leaf = certified + .cert + .first() + .ok_or_else(|| anyhow::anyhow!("staged endpoint certificate has no leaf"))?; + crate::registrar::endpoint_pin::RegistrarEndpointVerifier::new( + pins.into_iter().collect(), + &candidate.name, + )? + .verify( + leaf, + certified.cert.get(1..).unwrap_or_default(), + rustls::pki_types::UnixTime::now(), + )?; + } else { + crate::registrar::endpoint::tls::validate_client_certificate( + &certified, + &candidate.cert_path, + bundle, + trusted_ca_sha256, + )?; + } + let cert = std::fs::read(&candidate.cert_path)?; + if !cert_chain::leaf_chains_to_bundle(&cert, &std::fs::read(bundle)?)? { + anyhow::bail!("staged registrar certificate does not chain to the staged CA bundle"); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn pair_name(certified: &rustls::sign::CertifiedKey, path: &Path) -> Result { + let leaf = certified + .cert + .first() + .ok_or_else(|| anyhow::anyhow!("certificate at {} has no leaf", path.display()))?; + single_dns_san(leaf.as_ref()) + .map_err(|error| anyhow::anyhow!("reading candidate SAN at {}: {error}", path.display())) +} + +#[cfg(target_os = "linux")] +struct RenewalSnapshot { + bundle: SnapshotFile, + cert: SnapshotFile, + key: SnapshotFile, +} + +#[cfg(target_os = "linux")] +impl RenewalSnapshot { + async fn capture(bundle_path: &Path, pair: &SurfacePairPaths) -> Result { + Ok(Self { + bundle: SnapshotFile::capture(bundle_path).await?, + cert: SnapshotFile::capture(&pair.cert_path).await?, + key: SnapshotFile::capture(&pair.key_path).await?, + }) + } + + async fn restore(&self) -> Result<()> { + let mut failures = Vec::new(); + for snapshot in [&self.bundle, &self.cert, &self.key] { + if let Err(error) = snapshot.restore().await { + failures.push(format!("{}: {error:#}", snapshot.path.display())); + } + } + if !failures.is_empty() { + anyhow::bail!( + "restoring registrar renewal snapshots failed: {}", + failures.join("; ") + ); + } + Ok(()) + } +} + +/// One live path saved before a registrar renewal starts publication. +#[cfg(target_os = "linux")] +struct SnapshotFile { + bytes: Vec, + mode: u32, + uid: u32, + gid: u32, + path: PathBuf, +} + +#[cfg(target_os = "linux")] +impl SnapshotFile { + async fn capture(path: &Path) -> Result { + let metadata = tokio::fs::metadata(path).await.with_context(|| { + format!( + "reading metadata for registrar renewal snapshot at {}", + path.display() + ) + })?; + Ok(Self { + bytes: tokio::fs::read(path).await.with_context(|| { + format!("reading registrar renewal snapshot at {}", path.display()) + })?, + mode: metadata.permissions().mode() & 0o7777, + uid: metadata.uid(), + gid: metadata.gid(), + path: path.to_path_buf(), + }) + } + + async fn restore(&self) -> Result<()> { + crate::fs_util::atomic_write_fixed_owner( + crate::fs_util::Destination::operator_named(&self.path), + &self.bytes, + crate::fs_util::StagedMode::Policy(self.mode), + crate::fs_util::FixedOwner::observed(self.uid, self.gid), + ) + .await + .with_context(|| format!("restoring registrar renewal path {}", self.path.display()))?; + let metadata = tokio::fs::metadata(&self.path).await.with_context(|| { + format!( + "checking restored registrar renewal ownership at {}", + self.path.display() + ) + })?; + if metadata.uid() != self.uid || metadata.gid() != self.gid { + anyhow::bail!( + "restored registrar renewal path has owner {}:{}, expected {}:{}", + metadata.uid(), + metadata.gid(), + self.uid, + self.gid + ); + } + Ok(()) + } +} + +#[cfg(target_os = "linux")] +async fn publish_candidate( + bundle_path: &Path, + live: &SurfacePairPaths, + candidate: &SurfacePairPaths, + candidate_bundle: &Path, +) -> Result<()> { + publish_candidate_after_stage(bundle_path, live, candidate, candidate_bundle, |_| Ok(())).await +} + +/// A completed live publication step in the registrar renewal transaction. +#[cfg(target_os = "linux")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CandidatePublicationStage { + Bundle, + Certificate, + Key, +} + +/// Publishes a staged candidate, notifying `after_stage` after each live +/// rename. +/// +/// The callback keeps publication-failure tests at the real writer boundary: +/// each preceding file has been written with the same production primitive, +/// and any error still follows the ordinary transaction rollback path. +#[cfg(target_os = "linux")] +async fn publish_candidate_after_stage( + bundle_path: &Path, + live: &SurfacePairPaths, + candidate: &SurfacePairPaths, + candidate_bundle: &Path, + mut after_stage: AfterStage, +) -> Result<()> +where + AfterStage: FnMut(CandidatePublicationStage) -> Result<()>, +{ + let bundle = tokio::fs::read_to_string(candidate_bundle).await?; + let cert = tokio::fs::read_to_string(&candidate.cert_path).await?; + let key = SurfacePrivateKeyPem::new(tokio::fs::read_to_string(&candidate.key_path).await?); + crate::fs_util::write_ca_bundle( + bundle_path, + &bundle, + crate::cert_group::CertGroupPolicy::none(), + ) + .await?; + after_stage(CandidatePublicationStage::Bundle)?; + + // Keep the established certificate-then-key two-rename contract, while + // making the intermediate state observable to the transaction tests. + let cert_dir = live + .cert_path + .parent() + .ok_or_else(|| anyhow::anyhow!("certificate path has no parent directory"))?; + let key_dir = live + .key_path + .parent() + .ok_or_else(|| anyhow::anyhow!("key path has no parent directory"))?; + let policy = crate::cert_group::CertGroupPolicy::none(); + crate::cert_group::ensure_key_parent_dir(key_dir, policy).await?; + crate::cert_group::ensure_cert_parent_dir(cert_dir, key_dir, policy).await?; + crate::cert_group::write_cert_file(&live.cert_path, &cert, policy).await?; + after_stage(CandidatePublicationStage::Certificate)?; + crate::cert_group::write_key_file(&live.key_path, key.expose(), policy).await?; + after_stage(CandidatePublicationStage::Key) +} + /// Evaluates both pairs and returns the ones an issuance has to replace. /// /// The two are judged independently, so one being usable is never a @@ -780,6 +1361,21 @@ pub(crate) async fn issue_surface_pair( insecure_mode: bool, ) -> Result<()> { let issuance = issuance_settings(settings, pair, host, &inputs.responder_hmac); + issue_surface_candidate(&issuance, pair, inputs, insecure_mode).await +} + +/// Issues a registrar surface pair into the paths already embedded in +/// `issuance`. +/// +/// Start-time issuance hands this helper the configured output paths. Renewal +/// builds the same issuance settings around private staging paths, so no +/// renewal candidate can write a configured pair or the live CA bundle. +async fn issue_surface_candidate( + issuance: &Settings, + pair: &SurfacePairPaths, + inputs: &SurfaceAcmeInputs, + insecure_mode: bool, +) -> Result<()> { let profile = issuance .profiles .first() @@ -789,7 +1385,7 @@ pub(crate) async fn issue_surface_pair( // existing post-issuance merge gate does. let bootstrap_pins = bootstrap_pins_for_mode(&issuance.trust, insecure_mode); crate::acme::issue_certificate_with_bootstrap( - &issuance, + issuance, profile, inputs.eab.clone(), insecure_mode, diff --git a/src/registrar_certs/tests.rs b/src/registrar_certs/tests.rs index 534c2f39..01ab43ff 100644 --- a/src/registrar_certs/tests.rs +++ b/src/registrar_certs/tests.rs @@ -324,6 +324,35 @@ impl Host { } } +/// Builds the activated-endpoint holder renewal exchanges after a successful +/// publication, over the host's current live material. +#[cfg(target_os = "linux")] +fn renewal_test_endpoint(host: &Host) -> Arc { + let (server_cert, server_key) = host.server_pair(); + let bundle = host + .settings + .trust + .ca_bundle_path + .as_deref() + .expect("the renewal host configures a CA bundle"); + let (config, _) = crate::registrar::endpoint::tls::build_server_config( + Some(&server_cert), + Some(&server_key), + Some(bundle), + &host.settings.trust.trusted_ca_sha256, + &host.settings.domain, + ) + .expect("the host's provisioned material builds an endpoint configuration"); + let socket_path = host.dir.path().join("renewal-test.sock"); + let listener = tokio::net::UnixListener::bind(&socket_path).expect("bind test listener"); + crate::registrar::endpoint::ActivatedEndpoint::for_renewal_test( + listener, + socket_path, + config, + host.settings.domain.clone(), + ) +} + fn base_settings() -> Settings { Settings { email: TEST_EMAIL.to_string(), @@ -414,6 +443,36 @@ fn the_issuance_profile_composes_the_reserved_name_for_its_pair() { } } +/// Renewal passes an issuance profile built around private staging paths, so +/// the ACME writer cannot alter a live pair before validation succeeds. +#[test] +#[cfg(target_os = "linux")] +fn a_renewal_issuance_profile_targets_only_the_staged_pair() { + let host = Host::new(); + let live = surface_pairs(host.endpoint(), TEST_HOST, TEST_DOMAIN) + .expect("both live pairs resolve") + .into_iter() + .next() + .expect("the endpoint server pair exists"); + let staged = SurfacePairPaths { + leaf: live.leaf, + cert_path: host.dir.path().join("renewal/certificate.pem"), + key_path: host.dir.path().join("renewal/key.pem"), + name: live.name.clone(), + }; + + let issuance = issuance_settings(&host.settings, &staged, TEST_HOST, &"unused".into()); + let profile = issuance + .profiles + .first() + .expect("the staged profile exists"); + + assert_eq!(profile.paths.cert, staged.cert_path); + assert_eq!(profile.paths.key, staged.key_path); + assert_ne!(profile.paths.cert, live.cert_path); + assert_ne!(profile.paths.key, live.key_path); +} + /// The client leaf selects the `clientAuth` shape and the server leaf /// does not: the endpoint's own leaf is a server certificate and needs /// no added extended key usage. @@ -3096,3 +3155,465 @@ fn no_superseded_passage_survives_in_the_configuration_surface() { ); } } + +/// Renewal initialization observes the already-validated start-time pairs +/// without turning that observation into an issuance attempt. +#[tokio::test] +#[cfg(target_os = "linux")] +async fn renewal_initialization_records_both_leaves_without_an_attempt() { + let host = Host::new(); + host.provision_both_pairs(); + let plan = resolve_surface_plan(&host.settings).expect("the plan resolves"); + + let states = initialize_renewal_states(&plan.pairs) + .await + .expect("the usable pairs initialize their observations"); + let states = states + .lock() + .expect("the renewal observation lock is not poisoned"); + + assert_eq!(states.len(), 2); + for leaf in [SurfaceLeaf::EndpointServer, SurfaceLeaf::RegistrarClient] { + let state = states + .get(&leaf) + .expect("every enabled leaf has a state entry"); + assert_eq!(state.attempt, RenewalAttempt::NeverAttempted); + assert!(state.attempted_at.is_none()); + assert!(state.not_after > time::OffsetDateTime::now_utc()); + } +} + +/// A driven pass over usable leaves is a true no-op: it neither reaches the +/// issuance boundary nor changes the start-time observations. +#[tokio::test] +#[cfg(target_os = "linux")] +async fn renewal_pass_leaves_usable_pairs_unattempted() { + let host = Host::new(); + host.provision_both_pairs(); + let plan = resolve_surface_plan(&host.settings).expect("the plan resolves"); + let states = initialize_renewal_states(&plan.pairs) + .await + .expect("the usable pairs initialize their observations"); + let before = states + .lock() + .expect("the renewal observation lock is not poisoned") + .clone(); + let issuances = AtomicUsize::new(0); + + run_surface_renewal_pass(&host.settings, &plan, &states, |_| { + issuances.fetch_add(1, Ordering::SeqCst); + async { Ok(time::OffsetDateTime::UNIX_EPOCH) } + }) + .await; + + assert_eq!(issuances.load(Ordering::SeqCst), 0); + let states = states + .lock() + .expect("the renewal observation lock is not poisoned"); + for leaf in [SurfaceLeaf::EndpointServer, SurfaceLeaf::RegistrarClient] { + let before = before.get(&leaf).expect("the initial observation exists"); + let after = states.get(&leaf).expect("the current observation exists"); + assert_eq!(after.not_after, before.not_after); + assert_eq!(after.attempt, RenewalAttempt::NeverAttempted); + assert!(after.attempted_at.is_none()); + } +} + +/// A driven ordinary issuance failure records only the affected leaf and +/// retains its observed expiry for the next retry. +#[tokio::test] +#[cfg(target_os = "linux")] +async fn renewal_pass_records_an_injected_issuance_failure() { + let host = Host::new(); + host.provision_both_pairs(); + let plan = resolve_surface_plan(&host.settings).expect("the plan resolves"); + let states = initialize_renewal_states(&plan.pairs) + .await + .expect("the usable pairs initialize their observations"); + let original_server_not_after = states + .lock() + .expect("the renewal observation lock is not poisoned") + .get(&SurfaceLeaf::EndpointServer) + .expect("the server observation exists") + .not_after; + let (server_cert, _) = host.server_pair(); + std::fs::remove_file(server_cert).expect("remove the server certificate to make it due"); + let issuances = AtomicUsize::new(0); + + run_surface_renewal_pass(&host.settings, &plan, &states, |_| { + issuances.fetch_add(1, Ordering::SeqCst); + async { anyhow::bail!("injected registrar issuance failure") } + }) + .await; + + assert_eq!(issuances.load(Ordering::SeqCst), 1); + let states = states + .lock() + .expect("the renewal observation lock is not poisoned"); + let server = states + .get(&SurfaceLeaf::EndpointServer) + .expect("the server observation exists"); + assert_eq!(server.not_after, original_server_not_after); + assert!(matches!( + &server.attempt, + RenewalAttempt::Failed(reason) if reason.contains("injected registrar issuance failure") + )); + assert!(server.attempted_at.is_some()); + let client = states + .get(&SurfaceLeaf::RegistrarClient) + .expect("the client observation exists"); + assert_eq!(client.attempt, RenewalAttempt::NeverAttempted); + assert!(client.attempted_at.is_none()); +} + +/// Renewal takes its retry policy from the rendered internal credential +/// configuration, whose top-level `[retry]` applies when its sole profile +/// has no override. The outer daemon's retry setting is intentionally +/// distinct in this fixture. +#[test] +#[cfg(target_os = "linux")] +fn renewal_uses_the_internal_configurations_effective_retry_policy() { + let host = Host::new(); + let plan = resolve_surface_plan(&host.settings).expect("the plan resolves"); + + assert_eq!(host.settings.retry.backoff_secs, vec![1]); + assert!(plan.renewal_profile.retry.is_none()); + assert_eq!(plan.renewal_retry_backoff, vec![5, 15, 60]); +} + +/// A due client renewal exercises the production issuance and publication +/// paths: all candidate material remains off-live until validation and the +/// staged TLS configuration have completed, then the live pair and active +/// acceptor change together. +#[tokio::test] +#[cfg(target_os = "linux")] +async fn client_renewal_publishes_only_after_staging_and_swaps_the_acceptor() { + let log = Arc::new(OpenBaoLog::default()); + let openbao = start_openbao(TEST_KV_MOUNT, &log).await; + let mut host = Host::with_openbao_url(&openbao.uri()); + host.provision_both_pairs(); + host.provision_internal_credential(); + let acme = start_acme(Arc::clone(&host.ca)).await; + aim_at(&mut host.settings, &acme); + let plan = resolve_surface_plan(&host.settings).expect("the plan resolves"); + let client = plan + .pairs + .iter() + .find(|pair| pair.leaf == SurfaceLeaf::RegistrarClient) + .expect("the client pair exists"); + let endpoint = renewal_test_endpoint(&host); + let previous_acceptor = endpoint.tls_acceptor(); + let (client_cert, client_key) = host.client_pair(); + let previous_cert = digest_of(&client_cert); + let previous_key = digest_of(&client_key); + + renew_surface_pair(&host.settings, &plan, client, &endpoint, false) + .await + .expect("the staged client renewal publishes"); + + assert_ne!( + digest_of(&client_cert), + previous_cert, + "a new leaf is published" + ); + assert_ne!( + digest_of(&client_key), + previous_key, + "a fresh key is published" + ); + assert!( + !Arc::ptr_eq(&previous_acceptor, &endpoint.tls_acceptor()), + "the active acceptor changes only after publication succeeds" + ); + let paths = log.paths.lock().expect("the OpenBao log is not poisoned"); + assert!( + paths.iter().any(|path| path == "/v1/auth/cert/login"), + "renewal reads its ACME inputs with the internal certificate credential: {paths:?}" + ); +} + +/// A server candidate under an anchor absent from the endpoint pin file is +/// refused before publication, retaining both disk material and the active +/// endpoint configuration. +#[tokio::test] +#[cfg(target_os = "linux")] +async fn unpinned_server_renewal_preserves_live_material_and_active_acceptor() { + let log = Arc::new(OpenBaoLog::default()); + let openbao = start_openbao(TEST_KV_MOUNT, &log).await; + let mut host = Host::with_openbao_url(&openbao.uri()); + host.provision_both_pairs(); + host.provision_internal_credential(); + let acme = start_acme(Arc::clone(&host.ca)).await; + aim_at(&mut host.settings, &acme); + let plan = resolve_surface_plan(&host.settings).expect("the plan resolves"); + let server = plan + .pairs + .iter() + .find(|pair| pair.leaf == SurfaceLeaf::EndpointServer) + .expect("the server pair exists"); + let (client_cert, _) = host.client_pair(); + let pin_path = + crate::registrar::endpoint_pin::anchor_pin_path_for_client_certificate(&client_cert); + let foreign = TestCa::new("Unpinned Renewal Root"); + std::fs::write(&pin_path, format!("{}\n", foreign.root_fingerprint())) + .expect("write an unpinned anchor"); + let endpoint = renewal_test_endpoint(&host); + let previous_acceptor = endpoint.tls_acceptor(); + let paths = [ + host.settings + .trust + .ca_bundle_path + .clone() + .expect("the bundle path is configured"), + host.server_pair().0, + host.server_pair().1, + ]; + let before: Vec = paths.iter().map(|path| digest_of(path)).collect(); + + let error = renew_surface_pair(&host.settings, &plan, server, &endpoint, false) + .await + .expect_err("an unpinned server candidate must be refused"); + + assert!(format!("{error:#}").contains("pin"), "{error:#}"); + assert_eq!( + paths.iter().map(|path| digest_of(path)).collect::>(), + before, + "a pin refusal must not touch the live bundle or server pair" + ); + assert!( + Arc::ptr_eq(&previous_acceptor, &endpoint.tls_acceptor()), + "a pin refusal must leave the active acceptor installed" + ); +} + +/// Each failure point in the real bundle/certificate/key publication seam +/// restores every live path and leaves the active acceptor alone. A rollback +/// failure after a partial publication is reported alongside that failure and +/// still cannot exchange the active configuration. +#[tokio::test] +#[cfg(target_os = "linux")] +async fn publication_failures_roll_back_or_report_the_rollback_failure() { + let host = Host::new(); + host.provision_both_pairs(); + let plan = resolve_surface_plan(&host.settings).expect("the plan resolves"); + let client = plan + .pairs + .iter() + .find(|pair| pair.leaf == SurfaceLeaf::RegistrarClient) + .expect("the client pair exists"); + let endpoint = renewal_test_endpoint(&host); + let previous_acceptor = endpoint.tls_acceptor(); + let bundle = host + .settings + .trust + .ca_bundle_path + .as_deref() + .expect("the renewal host configures a bundle"); + let live_paths = [ + bundle.to_path_buf(), + client.cert_path.clone(), + client.key_path.clone(), + ]; + let originals: Vec> = live_paths + .iter() + .map(|path| std::fs::read(path).expect("read original live material")) + .collect(); + let staging = tempfile::tempdir().expect("create candidate staging directory"); + let candidate = SurfacePairPaths { + leaf: client.leaf, + cert_path: staging.path().join("candidate.crt"), + key_path: staging.path().join("candidate.key"), + name: client.name.clone(), + }; + let (candidate_cert, candidate_key) = host.ca.issue(&leaf_params(&candidate.name, -1, 30)); + write_pair( + &candidate.cert_path, + &candidate.key_path, + &candidate_cert, + &candidate_key, + ); + let candidate_bundle = staging.path().join("candidate-ca-bundle.pem"); + std::fs::write(&candidate_bundle, &originals[0]).expect("write staged candidate bundle"); + + for failed_stage in [ + CandidatePublicationStage::Bundle, + CandidatePublicationStage::Certificate, + CandidatePublicationStage::Key, + ] { + let snapshot = RenewalSnapshot::capture(bundle, client) + .await + .expect("capture publication restore artifacts"); + let (next_config, _) = crate::registrar::endpoint::tls::build_server_config( + Some(&host.server_pair().0), + Some(&host.server_pair().1), + Some(bundle), + &host.settings.trust.trusted_ca_sha256, + &host.settings.domain, + ) + .expect("build a complete next configuration before publication"); + let error = publish_surface_renewal_transaction( + &endpoint, + next_config, + || { + publish_candidate_after_stage( + bundle, + client, + &candidate, + &candidate_bundle, + |stage| { + if stage == failed_stage { + anyhow::bail!("injected {stage:?} publication failure"); + } + Ok(()) + }, + ) + }, + || snapshot.restore(), + ) + .await + .expect_err("the injected publication failure must be reported"); + assert!(format!("{error:#}").contains("restored prior material")); + assert_eq!( + live_paths + .iter() + .map(|path| std::fs::read(path).expect("read restored live material")) + .collect::>(), + originals, + "a {failed_stage:?} failure restores bundle, certificate and key" + ); + assert!(Arc::ptr_eq(&previous_acceptor, &endpoint.tls_acceptor())); + } + + let _snapshot = RenewalSnapshot::capture(bundle, client) + .await + .expect("capture publication restore artifacts"); + let (next_config, _) = crate::registrar::endpoint::tls::build_server_config( + Some(&host.server_pair().0), + Some(&host.server_pair().1), + Some(bundle), + &host.settings.trust.trusted_ca_sha256, + &host.settings.domain, + ) + .expect("build a complete next configuration before publication"); + let error = publish_surface_renewal_transaction( + &endpoint, + next_config, + || { + publish_candidate_after_stage(bundle, client, &candidate, &candidate_bundle, |stage| { + if stage == CandidatePublicationStage::Certificate { + anyhow::bail!("injected certificate publication failure"); + } + Ok(()) + }) + }, + || async { anyhow::bail!("injected rollback failure") }, + ) + .await + .expect_err("the injected rollback failure must be reported"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("injected certificate publication failure"), + "{rendered}" + ); + assert!(rendered.contains("injected rollback failure"), "{rendered}"); + assert_ne!( + std::fs::read(&client.cert_path).expect("read the partially published certificate"), + originals[1], + "a failed rollback after the certificate rename must not claim the old material returned" + ); + assert!(Arc::ptr_eq(&previous_acceptor, &endpoint.tls_acceptor())); +} + +/// A failed publication restores bytes, mode and the pre-existing owner from +/// each saved path before another tick is allowed to retry. +#[tokio::test] +#[cfg(target_os = "linux")] +async fn renewal_snapshot_restores_bytes_mode_and_owner() { + let directory = tempfile::tempdir().expect("create renewal snapshot directory"); + let path = directory.path().join("client.key"); + std::fs::write(&path, b"original private key").expect("write original key"); + std::fs::set_permissions(&path, std::os::unix::fs::PermissionsExt::from_mode(0o600)) + .expect("set original restrictive key mode"); + let Some(original_gid) = crate::cert_group::one_supplementary_test_gid() else { + // The host cannot exercise a real ownership transition without a + // supplementary group the process is allowed to establish. + return; + }; + std::os::unix::fs::chown(&path, None, Some(original_gid)) + .expect("seed an owner distinct from the publishing process"); + let snapshot = SnapshotFile::capture(&path) + .await + .expect("capture the live key before publication"); + + std::fs::write(&path, b"partially published key").expect("overwrite live key"); + std::fs::set_permissions(&path, std::os::unix::fs::PermissionsExt::from_mode(0o644)) + .expect("simulate an incorrect published mode"); + std::os::unix::fs::chown(&path, None, Some(crate::cert_group::current_process_egid())) + .expect("simulate a publication that changed ownership"); + snapshot.restore().await.expect("restore the saved key"); + + let metadata = std::fs::metadata(&path).expect("read restored key metadata"); + assert_eq!( + std::fs::read(&path).expect("read restored key"), + b"original private key" + ); + assert_eq!( + std::os::unix::fs::PermissionsExt::mode(&metadata.permissions()) & 0o7777, + 0o600 + ); + assert_eq!(std::os::unix::fs::MetadataExt::uid(&metadata), snapshot.uid); + assert_eq!(std::os::unix::fs::MetadataExt::gid(&metadata), snapshot.gid); +} + +/// A staged client leaf must be valid for the exact mTLS verifier the next +/// endpoint configuration would install; a matching key and reserved SAN do +/// not make an expired certificate safe to publish. +#[test] +#[cfg(target_os = "linux")] +fn renewal_refuses_an_expired_registrar_client_candidate_before_publication() { + let host = Host::new(); + let candidate = SurfacePairPaths { + leaf: SurfaceLeaf::RegistrarClient, + cert_path: host.dir.path().join("staging/client.crt"), + key_path: host.dir.path().join("staging/client.key"), + name: Host::client_name(), + }; + let params = crate::acme::build_registrar_client_csr_params( + REGISTRAR_SURFACE_INSTANCE, + TEST_HOST, + TEST_DOMAIN, + ) + .expect("the registrar client CSR builds"); + let key = KeyPair::generate().expect("create a staged client key"); + let csr = params + .serialize_request(&key) + .expect("serialize the staged client CSR"); + host.ca.sign_inside(-2, -1); + let certificate = host.ca.sign_csr(csr.der()); + write_pair( + &candidate.cert_path, + &candidate.key_path, + &certificate, + &key.serialize_pem(), + ); + + let bundle = host + .settings + .trust + .ca_bundle_path + .as_deref() + .expect("the endpoint host configures a CA bundle"); + let error = validate_candidate( + &candidate, + bundle, + &candidate, + &host.settings.trust.trusted_ca_sha256, + ) + .expect_err("an expired staged registrar client certificate must not publish"); + assert!( + error + .to_string() + .contains("not a valid registrar client certificate"), + "unexpected validation error: {error:#}" + ); +}