From 5585d01494644c58f54ce57fb303fb7921070e15 Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Tue, 14 Jul 2026 23:58:16 -0700 Subject: [PATCH] feat(nras): audit and measure device attestation nras -- GPU/DPU attestation, security-critical -- had neither logs nor metrics, so a remote verifier failure surfaced nowhere. This adds an audit trail and outbound measurement. - The attestation call in `attest_gpu` runs through `red::instrumented("nras", "attest_gpu", ...)`, recording `carbide_external_call_duration_milliseconds{backend = "nras", operation, outcome}` and a WARN on transport failure -- reusing the outbound-RED family, so no new row for it. - A new `carbide_attestation_total{device_type, outcome}` counter is the audit trail: emitted once per attestation at the spdm-controller dispatch, INFO on success and WARN on a failed verifier call. Labels are bounded to `device_type` and `outcome`; the machine and device ids ride as log-only context, and no attestation evidence, certificate, or nonce ever reaches a label or log. - `attest_cx7` is not implemented yet, so a Cx7 attestation records `outcome = error` -- deliberately, to surface that a device we cannot yet attest was asked to. This supports https://github.com/NVIDIA/infra-controller/issues/3173 Signed-off-by: Chet Nichols III --- Cargo.lock | 2 + crates/nras/Cargo.toml | 2 + crates/nras/src/client.rs | 40 ++++++++----- crates/nras/src/keystore.rs | 31 ++++++---- crates/spdm-controller/Cargo.toml | 1 + crates/spdm-controller/src/handler.rs | 83 ++++++++++++++++++++++++++- docs/observability/core_metrics.md | 1 + 7 files changed, 131 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f61320fa9c..df407c41eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3025,6 +3025,7 @@ dependencies = [ "async-trait", "carbide-api-db", "carbide-api-model", + "carbide-instrument", "carbide-redfish", "carbide-utils", "config-version", @@ -7528,6 +7529,7 @@ version = "0.1.0" dependencies = [ "async-trait", "base64", + "carbide-instrument", "clap", "fmt", "jsonwebtoken", diff --git a/crates/nras/Cargo.toml b/crates/nras/Cargo.toml index 613e29daa6..7e30afa770 100644 --- a/crates/nras/Cargo.toml +++ b/crates/nras/Cargo.toml @@ -22,6 +22,8 @@ license.workspace = true authors.workspace = true [dependencies] +carbide-instrument = { path = "../instrument" } + async-trait.workspace = true base64 = { workspace = true } clap = { features = ["derive", "env"], workspace = true } diff --git a/crates/nras/src/client.rs b/crates/nras/src/client.rs index 0979e18bd8..c43d67bb19 100644 --- a/crates/nras/src/client.rs +++ b/crates/nras/src/client.rs @@ -18,6 +18,7 @@ use std::collections as stdcol; use async_trait::async_trait; +use carbide_instrument::red; use serde_json as sj; use crate::{DeviceAttestationInfo, NrasError, RawAttestationOutcome}; @@ -67,8 +68,7 @@ impl VerifierClient for NrasVerifierClient { device_attestation_info: &DeviceAttestationInfo, ) -> Result { // prepare the request - // submit to NRAS (the http client propagates W3C trace context via middleware) - let att_response = self + let request = self .http_client .post(format!( "{}{}", @@ -77,19 +77,29 @@ impl VerifierClient for NrasVerifierClient { .header("Content-Type", "application/json") .body(serde_json::to_string(device_attestation_info).map_err(|e| { NrasError::Serde(format!("Error Serializing Attestation Request: {}", e)) - })?) - .send() - .await?; - - let status_code = att_response.status(); - let response_text = att_response.text().await?; - - if status_code != reqwest::StatusCode::OK { - return Err(NrasError::Communication(format!( - "NRAS returned status code {} and message {}.\n Config is {:?}", - status_code, response_text, self.config - ))); - } + })?); + + // Submit to NRAS (the http client propagates W3C trace context via middleware). + // The send AND the HTTP status check run inside the RED wrapper, so a non-success + // status is recorded as outcome="error" (and logged WARN), not a silent "ok": + // carbide_external_call_duration_milliseconds{backend="nras", operation="attest_gpu", + // outcome}. Only the static backend/operation tags are recorded -- never the request + // or response body (evidence, certificate, nonce). + let response_text = red::instrumented("nras", "attest_gpu", async { + let att_response = request.send().await?; + let status_code = att_response.status(); + if status_code != reqwest::StatusCode::OK { + // Status only: the response body and the config can carry sensitive + // attestation material, so neither is placed in the error or its log. + return Err(NrasError::Communication(format!( + "NRAS returned status code {}", + status_code + ))); + } + // Read the body only after confirming success (the happy path needs it). + Ok(att_response.text().await?) + }) + .await?; // read the response and map to the RawAttestationOutcome let verifier_response: RawAttestationOutcome = diff --git a/crates/nras/src/keystore.rs b/crates/nras/src/keystore.rs index e65f18e423..93ef355016 100644 --- a/crates/nras/src/keystore.rs +++ b/crates/nras/src/keystore.rs @@ -17,6 +17,7 @@ use std::collections as stdcol; +use carbide_instrument::red; use jsonwebtoken as jst; use crate::NrasError; @@ -54,17 +55,25 @@ impl KeyStore for NrasKeyStore { impl NrasKeyStore { pub async fn new_with_config(config: &crate::Config) -> Result { - let jwks_response = reqwest::get(&config.nras_jwks_url).await?; - - let status_code = jwks_response.status(); - let response_text = jwks_response.text().await?; - - if status_code != reqwest::StatusCode::OK { - return Err(NrasError::Communication(format!( - "NRAS KeyStore returned status code {} and message {}", - status_code, response_text - ))); - } + // The JWKS fetch is a required NRAS call on the attestation path; wrap the + // GET and its status check in the RED triad so it is no longer dark and a + // non-success status records outcome = error. Reuses the + // carbide_external_call_duration_milliseconds family (operation="fetch_jwks"). + let response_text = red::instrumented("nras", "fetch_jwks", async { + let jwks_response = reqwest::get(&config.nras_jwks_url).await?; + let status_code = jwks_response.status(); + if status_code != reqwest::StatusCode::OK { + // Status only: the response body can carry sensitive material, so it is + // never placed in the error or its log. + return Err(NrasError::Communication(format!( + "NRAS KeyStore returned status code {}", + status_code + ))); + } + // Read the body only after confirming success. + Ok(jwks_response.text().await?) + }) + .await?; // parse JWKS and find matching JWK let jwks: Jwks = serde_json::from_str(&response_text) diff --git a/crates/spdm-controller/Cargo.toml b/crates/spdm-controller/Cargo.toml index b97a4ccafe..4220e3d833 100644 --- a/crates/spdm-controller/Cargo.toml +++ b/crates/spdm-controller/Cargo.toml @@ -24,6 +24,7 @@ authors.workspace = true [dependencies] carbide-api-db = { path = "../api-db", default-features = false } carbide-api-model = { path = "../api-model", default-features = false } +carbide-instrument = { path = "../instrument" } carbide-redfish = { path = "../redfish" } carbide-utils = { path = "../utils", default-features = false } config-version = { path = "../config-version", default-features = false } diff --git a/crates/spdm-controller/src/handler.rs b/crates/spdm-controller/src/handler.rs index cb23a80636..c921e0aa77 100644 --- a/crates/spdm-controller/src/handler.rs +++ b/crates/spdm-controller/src/handler.rs @@ -17,6 +17,7 @@ use std::sync::Arc; +use carbide_instrument::{DynamicLog, Event, LabelValue, LogAt, Outcome, emit}; use carbide_redfish::libredfish::conv::IntoModel; use carbide_redfish::libredfish::error::state_handler_redfish_error as redfish_error; use itertools::Itertools; @@ -312,6 +313,56 @@ impl StateHandler for SpdmAttestationStateHandler { } } } + +/// The device kind an attestation covered, as a bounded metric label. Only the +/// kinds an attestation is actually dispatched for appear here; an unknown +/// device type returns before any attestation runs, so it never reaches a label. +#[derive(Debug, Clone, Copy, PartialEq, Eq, LabelValue)] +enum AttestedDeviceType { + Gpu, + Cx7, +} + +/// A device attestation ran to completion -- the security audit record. +/// +/// `nras`'s RED wrapper (`carbide_external_call_duration_milliseconds`) times +/// the verifier call and logs only its transport failures, counting successes +/// silently. Attestation needs a real audit trail, so this event logs every +/// attestation -- INFO on success, WARN on a failed call -- and counts it by +/// `device_type` and `outcome`. Those two bounded enums are the only labels; the +/// machine and device ids are log-only context. Attestation evidence, JWTs, +/// certificates, and the nonce are never put on a label or in the log line. +#[derive(Event)] +#[event( + name = "carbide_attestation_total", + component = "carbide-spdm-controller", + log = dynamic, + metric = counter, + message = "Device attestation performed", + describe = "Number of device attestations performed, by device type and outcome." +)] +struct AttestationPerformed { + #[label] + device_type: AttestedDeviceType, + #[label] + outcome: Outcome, + #[context] + machine_id: String, + #[context] + device_id: String, +} + +impl DynamicLog for AttestationPerformed { + fn log_at(&self) -> LogAt { + match self.outcome { + // Every attestation is audited: a successful call at INFO, a failed + // call elevated to WARN (RED already logged the transport cause). + Outcome::Ok => LogAt::Level(tracing::Level::INFO), + Outcome::Error => LogAt::Level(tracing::Level::WARN), + } + } +} + async fn perform_attestation( client: &dyn VerifierClient, device: &SpdmDeviceAttestation, @@ -353,9 +404,21 @@ async fn perform_attestation( }; let device_type: DeviceType = device.device_id.parse()?; - let response = match device_type { - DeviceType::Gpu => client.attest_gpu(&device_attestation_info).await, - DeviceType::Cx7 => client.attest_cx7(&device_attestation_info).await, + let (attested_device_type, response) = match device_type { + DeviceType::Gpu => ( + AttestedDeviceType::Gpu, + client.attest_gpu(&device_attestation_info).await, + ), + // `attest_cx7` is not implemented yet, so every Cx7 call records + // `outcome = error` -- deliberately, so the audit trail surfaces that we + // were asked to attest a Cx7 device and couldn't. TODO: once Cx7 + // attestation lands, this becomes a real outcome; if the not-implemented + // errors prove noisy before then, skip the emit for Cx7 as we do for + // `Unknown`. + DeviceType::Cx7 => ( + AttestedDeviceType::Cx7, + client.attest_cx7(&device_attestation_info).await, + ), DeviceType::Unknown => { return Err(SpdmHandlerError::VerifierNotImplemented { module: "state_handler".to_string(), @@ -365,6 +428,20 @@ async fn perform_attestation( } }; + // Security audit trail. `nras`'s RED wrapper times the verifier call and warns + // on transport failures but counts successes silently; attestation needs a + // real audit record, so emit one for every call -- the one place a successful + // attestation is logged (INFO), and a failed call is re-surfaced (WARN) with + // its device type and outcome. device_type and outcome are the only labels; + // the machine and device ids are log-only context. Attestation evidence, JWTs, + // certificates, and the nonce never reach a label or the log line. + emit(AttestationPerformed { + device_type: attested_device_type, + outcome: Outcome::from(&response), + machine_id: device.machine_id.to_string(), + device_id: device.device_id.clone(), + }); + match response { Ok(res) => Ok(res), Err(nras::NrasError::NotImplemented) => Err(SpdmHandlerError::VerifierNotImplemented { diff --git a/docs/observability/core_metrics.md b/docs/observability/core_metrics.md index 60769dfcc0..c3ba4ec00b 100644 --- a/docs/observability/core_metrics.md +++ b/docs/observability/core_metrics.md @@ -19,6 +19,7 @@ This file contains a list of metrics exported by NVIDIA Infra Controller (NICo). carbide_api_vault_requests_succeeded_totalcounterNumber of successful Vault requests carbide_api_vault_token_time_until_refresh_secondsgaugeThe amount of time, in seconds, until the Vault token is required to be refreshed carbide_api_versiongaugeVersion (git sha, build date, etc) of this service +carbide_attestation_totalcounterNumber of device attestations performed, by device type and outcome. carbide_authn_client_cert_rejected_totalcounterNumber of client certificates rejected during authentication carbide_available_ips_countgaugeNumber of available IPs per network segment carbide_client_tcp_connect_attempts_totalcounterNumber of outbound TCP connect attempts across all HTTP connectors