feat(nras): audit and measure device attestation

crates/nras/Cargo.toml

@@ -22,6 +22,8 @@ license.workspace = true
 authors.workspace = true
 
 [dependencies]
+carbide-instrument = { path = "../instrument" }
+
 async-trait.workspace = true
 base64 = { workspace = true }
 clap = { features = ["derive", "env"], workspace = true }

crates/nras/src/client.rs

@@ -18,6 +18,7 @@
 use std::collections as stdcol;
 
 use async_trait::async_trait;
+use carbide_instrument::red;
 use serde_json as sj;
 
 use crate::{DeviceAttestationInfo, NrasError, RawAttestationOutcome};
@@ -67,8 +68,7 @@ impl VerifierClient for NrasVerifierClient {
         device_attestation_info: &DeviceAttestationInfo,
     ) -> Result<RawAttestationOutcome, NrasError> {
         // prepare the request
-        // submit to NRAS (the http client propagates W3C trace context via middleware)
-        let att_response = self
+        let request = self
             .http_client
             .post(format!(
                 "{}{}",
@@ -77,19 +77,29 @@ impl VerifierClient for NrasVerifierClient {
             .header("Content-Type", "application/json")
             .body(serde_json::to_string(device_attestation_info).map_err(|e| {
                 NrasError::Serde(format!("Error Serializing Attestation Request: {}", e))
-            })?)
-            .send()
-            .await?;
-
-        let status_code = att_response.status();
-        let response_text = att_response.text().await?;
-
-        if status_code != reqwest::StatusCode::OK {
-            return Err(NrasError::Communication(format!(
-                "NRAS returned status code {} and message {}.\n Config is {:?}",
-                status_code, response_text, self.config
-            )));
-        }
+            })?);
+
+        // Submit to NRAS (the http client propagates W3C trace context via middleware).
+        // The send AND the HTTP status check run inside the RED wrapper, so a non-success
+        // status is recorded as outcome="error" (and logged WARN), not a silent "ok":
+        // carbide_external_call_duration_milliseconds{backend="nras", operation="attest_gpu",
+        // outcome}. Only the static backend/operation tags are recorded -- never the request
+        // or response body (evidence, certificate, nonce).
+        let response_text = red::instrumented("nras", "attest_gpu", async {
+            let att_response = request.send().await?;
+            let status_code = att_response.status();
+            if status_code != reqwest::StatusCode::OK {
+                // Status only: the response body and the config can carry sensitive
+                // attestation material, so neither is placed in the error or its log.
+                return Err(NrasError::Communication(format!(
+                    "NRAS returned status code {}",
+                    status_code
+                )));
+            }
+            // Read the body only after confirming success (the happy path needs it).
+            Ok(att_response.text().await?)
+        })
+        .await?;
 
         // read the response and map to the RawAttestationOutcome
         let verifier_response: RawAttestationOutcome =

crates/nras/src/keystore.rs

@@ -17,6 +17,7 @@
 
 use std::collections as stdcol;
 
+use carbide_instrument::red;
 use jsonwebtoken as jst;
 
 use crate::NrasError;
@@ -54,17 +55,25 @@ impl KeyStore for NrasKeyStore {
 
 impl NrasKeyStore {
     pub async fn new_with_config(config: &crate::Config) -> Result<NrasKeyStore, crate::NrasError> {
-        let jwks_response = reqwest::get(&config.nras_jwks_url).await?;
-
-        let status_code = jwks_response.status();
-        let response_text = jwks_response.text().await?;
-
-        if status_code != reqwest::StatusCode::OK {
-            return Err(NrasError::Communication(format!(
-                "NRAS KeyStore returned status code {} and message {}",
-                status_code, response_text
-            )));
-        }
+        // The JWKS fetch is a required NRAS call on the attestation path; wrap the
+        // GET and its status check in the RED triad so it is no longer dark and a
+        // non-success status records outcome = error. Reuses the
+        // carbide_external_call_duration_milliseconds family (operation="fetch_jwks").
+        let response_text = red::instrumented("nras", "fetch_jwks", async {
+            let jwks_response = reqwest::get(&config.nras_jwks_url).await?;
+            let status_code = jwks_response.status();
+            if status_code != reqwest::StatusCode::OK {
+                // Status only: the response body can carry sensitive material, so it is
+                // never placed in the error or its log.
+                return Err(NrasError::Communication(format!(
+                    "NRAS KeyStore returned status code {}",
+                    status_code
+                )));
+            }
+            // Read the body only after confirming success.
+            Ok(jwks_response.text().await?)
+        })
+        .await?;
 
         // parse JWKS and find matching JWK
         let jwks: Jwks = serde_json::from_str(&response_text)

crates/spdm-controller/Cargo.toml

@@ -24,6 +24,7 @@ authors.workspace = true
 [dependencies]
 carbide-api-db = { path = "../api-db", default-features = false }
 carbide-api-model = { path = "../api-model", default-features = false }
+carbide-instrument = { path = "../instrument" }
 carbide-redfish = { path = "../redfish" }
 carbide-utils = { path = "../utils", default-features = false }
 config-version = { path = "../config-version", default-features = false }

crates/spdm-controller/src/handler.rs

@@ -17,6 +17,7 @@
 
 use std::sync::Arc;
 
+use carbide_instrument::{DynamicLog, Event, LabelValue, LogAt, Outcome, emit};
 use carbide_redfish::libredfish::conv::IntoModel;
 use carbide_redfish::libredfish::error::state_handler_redfish_error as redfish_error;
 use itertools::Itertools;
@@ -312,6 +313,56 @@ impl StateHandler for SpdmAttestationStateHandler {
         }
     }
 }
+
+/// The device kind an attestation covered, as a bounded metric label. Only the
+/// kinds an attestation is actually dispatched for appear here; an unknown
+/// device type returns before any attestation runs, so it never reaches a label.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, LabelValue)]
+enum AttestedDeviceType {
+    Gpu,
+    Cx7,
+}
+
+/// A device attestation ran to completion -- the security audit record.
+///
+/// `nras`'s RED wrapper (`carbide_external_call_duration_milliseconds`) times
+/// the verifier call and logs only its transport failures, counting successes
+/// silently. Attestation needs a real audit trail, so this event logs every
+/// attestation -- INFO on success, WARN on a failed call -- and counts it by
+/// `device_type` and `outcome`. Those two bounded enums are the only labels; the
+/// machine and device ids are log-only context. Attestation evidence, JWTs,
+/// certificates, and the nonce are never put on a label or in the log line.
+#[derive(Event)]
+#[event(
+    name = "carbide_attestation_total",
+    component = "carbide-spdm-controller",
+    log = dynamic,
+    metric = counter,
+    message = "Device attestation performed",
+    describe = "Number of device attestations performed, by device type and outcome."
+)]
+struct AttestationPerformed {
+    #[label]
+    device_type: AttestedDeviceType,
+    #[label]
+    outcome: Outcome,
+    #[context]
+    machine_id: String,
+    #[context]
+    device_id: String,
+}
+
+impl DynamicLog for AttestationPerformed {
+    fn log_at(&self) -> LogAt {
+        match self.outcome {
+            // Every attestation is audited: a successful call at INFO, a failed
+            // call elevated to WARN (RED already logged the transport cause).
+            Outcome::Ok => LogAt::Level(tracing::Level::INFO),
+            Outcome::Error => LogAt::Level(tracing::Level::WARN),
+        }
+    }
+}
+
 async fn perform_attestation(
     client: &dyn VerifierClient,
     device: &SpdmDeviceAttestation,
@@ -353,9 +404,21 @@ async fn perform_attestation(
     };
 
     let device_type: DeviceType = device.device_id.parse()?;
-    let response = match device_type {
-        DeviceType::Gpu => client.attest_gpu(&device_attestation_info).await,
-        DeviceType::Cx7 => client.attest_cx7(&device_attestation_info).await,
+    let (attested_device_type, response) = match device_type {
+        DeviceType::Gpu => (
+            AttestedDeviceType::Gpu,
+            client.attest_gpu(&device_attestation_info).await,
+        ),
+        // `attest_cx7` is not implemented yet, so every Cx7 call records
+        // `outcome = error` -- deliberately, so the audit trail surfaces that we
+        // were asked to attest a Cx7 device and couldn't. TODO: once Cx7
+        // attestation lands, this becomes a real outcome; if the not-implemented
+        // errors prove noisy before then, skip the emit for Cx7 as we do for
+        // `Unknown`.
+        DeviceType::Cx7 => (
+            AttestedDeviceType::Cx7,
+            client.attest_cx7(&device_attestation_info).await,
+        ),
         DeviceType::Unknown => {
             return Err(SpdmHandlerError::VerifierNotImplemented {
                 module: "state_handler".to_string(),
@@ -365,6 +428,20 @@ async fn perform_attestation(
         }
     };
 
+    // Security audit trail. `nras`'s RED wrapper times the verifier call and warns
+    // on transport failures but counts successes silently; attestation needs a
+    // real audit record, so emit one for every call -- the one place a successful
+    // attestation is logged (INFO), and a failed call is re-surfaced (WARN) with
+    // its device type and outcome. device_type and outcome are the only labels;
+    // the machine and device ids are log-only context. Attestation evidence, JWTs,
+    // certificates, and the nonce never reach a label or the log line.
+    emit(AttestationPerformed {
+        device_type: attested_device_type,
+        outcome: Outcome::from(&response),
+        machine_id: device.machine_id.to_string(),
+        device_id: device.device_id.clone(),
+    });
+
     match response {
         Ok(res) => Ok(res),
         Err(nras::NrasError::NotImplemented) => Err(SpdmHandlerError::VerifierNotImplemented {

docs/observability/core_metrics.md

@@ -19,6 +19,7 @@ This file contains a list of metrics exported by NVIDIA Infra Controller (NICo).
 <tr><td>carbide_api_vault_requests_succeeded_total</td><td>counter</td><td>Number of successful Vault requests</td></tr>
 <tr><td>carbide_api_vault_token_time_until_refresh_seconds</td><td>gauge</td><td>The amount of time, in seconds, until the Vault token is required to be refreshed</td></tr>
 <tr><td>carbide_api_version</td><td>gauge</td><td>Version (git sha, build date, etc) of this service</td></tr>
+<tr><td>carbide_attestation_total</td><td>counter</td><td>Number of device attestations performed, by device type and outcome.</td></tr>
 <tr><td>carbide_authn_client_cert_rejected_total</td><td>counter</td><td>Number of client certificates rejected during authentication</td></tr>
 <tr><td>carbide_available_ips_count</td><td>gauge</td><td>Number of available IPs per network segment</td></tr>
 <tr><td>carbide_client_tcp_connect_attempts_total</td><td>counter</td><td>Number of outbound TCP connect attempts across all HTTP connectors</td></tr>