feat(scout): give the on-host discovery agent a /metrics endpoint

crates/scout/Cargo.toml

@@ -34,12 +34,14 @@ path = "src/main.rs"
 # [local-dependencies]
 carbide-rpc = { path = "../rpc" }
 carbide-host-support = { path = "../host-support" }
+carbide-instrument = { path = "../instrument" }
 carbide-tls = { path = "../tls" }
 carbide-uuid = { path = "../uuid" }
 carbide-version = { path = "../version" }
 carbide-utils = { path = "../utils" }
 carbide-machine-validation = { path = "../machine-validation" }
 carbide-libmlx = { path = "../libmlx" }
+metrics-endpoint = { path = "../metrics-endpoint" }
 
 chrono = { workspace = true }
 clap = { workspace = true }
@@ -73,6 +75,8 @@ x509-parser = { workspace = true }
 
 [dev-dependencies]
 axum = { workspace = true }
+carbide-instrument = { path = "../instrument", features = ["test-support"] }
+carbide-test-support = { path = "../test-support" }
 
 [build-dependencies]
 carbide-version = { path = "../version" }

crates/scout/src/cfg/command_line.rs

@@ -14,6 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+use std::net::SocketAddr;
+
 use carbide_uuid::machine_validation::MachineValidationId;
 use clap::{Parser, Subcommand, ValueEnum};
 use forge_tls::default as tls_default;
@@ -108,6 +110,13 @@ pub(crate) struct Options {
     )]
     pub tpm_path: String,
 
+    #[clap(
+        long,
+        help = "HTTP listen address for the metrics/health endpoint (e.g. 127.0.0.1:9091). \
+                When omitted the endpoint is not served and no metrics are collected."
+    )]
+    pub metrics_listen_addr: Option<SocketAddr>,
+
     #[clap(subcommand)]
     pub subcmd: Option<Command>,
 }

crates/scout/src/main.rs

@@ -22,6 +22,7 @@ use std::time::Duration;
 
 use carbide_host_support::dpa_cmds::{DpaCommand, OpCode};
 use carbide_host_support::registration;
+use carbide_instrument::{Outcome, emit};
 use carbide_uuid::machine::MachineId;
 use cfg::{AutoDetect, Command, MlxAction, Mode, Options};
 use chrono::{DateTime, Days, TimeDelta, Utc};
@@ -54,6 +55,7 @@ mod deprovision;
 mod discovery;
 mod firmware_upgrade;
 mod machine_validation;
+mod metrics;
 mod mlx_device;
 mod platform;
 mod register;
@@ -167,6 +169,35 @@ async fn initial_setup(config: &Options) -> Result<(uuid::Uuid, MachineId), eyre
 }
 
 async fn run_as_service(config: &Options) -> Result<(), eyre::Report> {
+    // Stand up the metrics/scrape endpoint when it is configured. The meter
+    // provider must stay alive for the lifetime of the service: dropping it
+    // shuts down the Prometheus exporter. The endpoint is opt-in -- without
+    // --metrics-listen-addr scout installs no meter and the counters below are
+    // no-ops.
+    let _metrics_guard = match config.metrics_listen_addr {
+        Some(address) => {
+            let metrics_setup =
+                metrics_endpoint::new_metrics_setup("nico-scout", "forge-system", true)?;
+            carbide_instrument::log_events::register(&metrics_setup.meter);
+            let metrics_config = metrics_endpoint::MetricsEndpointConfig {
+                address,
+                registry: metrics_setup.registry,
+                health_controller: Some(metrics_setup.health_controller),
+                additional_prefix: None,
+            };
+            // The endpoint's /health and /ready report process liveness (the
+            // default HealthController state), not scout readiness.
+            tokio::spawn(async move {
+                tracing::info!("Spawning metrics endpoint on {}", metrics_config.address);
+                if let Err(e) = metrics_endpoint::run_metrics_endpoint(&metrics_config).await {
+                    tracing::error!("Metrics endpoint error: {e}");
+                }
+            });
+            Some(metrics_setup.meter_provider)
+        }
+        None => None,
+    };
+
     // Implement the logic to run as a service here
     let (machine_interface_id, machine_id) = initial_setup(config).await?;
 
@@ -210,7 +241,14 @@ async fn run_as_service(config: &Options) -> Result<(), eyre::Report> {
         };
         if let Some(action) = controller_response.action {
             let action_str = action.as_str_name().to_owned();
-            match handle_action(action, &machine_id, machine_interface_id, config).await {
+            // Capture the action label before handle_action consumes `action`.
+            let scout_action = metrics::ScoutAction::from(&action);
+            let result = handle_action(action, &machine_id, machine_interface_id, config).await;
+            emit(metrics::ScoutActionHandled {
+                action: scout_action,
+                outcome: Outcome::from(&result),
+            });
+            match result {
                 Ok(_) => tracing::info!("Successfully served {}", action_str),
                 Err(e) => tracing::info!("Failed to serve {}: Err {}", action_str, e),
             };
@@ -348,10 +386,13 @@ async fn handle_action(
             unimplemented!("Rebuild not written yet");
         }
         fac::Action::Noop(_) => {}
-        fac::Action::LogError(_) => match logerror_to_carbide(config, machine_interface_id).await {
-            Ok(()) => (),
-            Err(e) => tracing::info!("Forge Scout logerror_to_carbide error: {}", e),
-        },
+        fac::Action::LogError(_) => logerror_to_carbide(config, machine_interface_id)
+            .await
+            // Propagate the failure so `carbide_scout_actions_total` records this
+            // as `outcome = error`, not a silent success.
+            .map_err(|e| {
+                CarbideClientError::GenericError(format!("logerror_to_carbide failed: {e}"))
+            })?,
         fac::Action::Retry(_) => {
             panic!(
                 "Retrieved Retry action, which should be handled internally by query_api_with_retries"

crates/scout/src/metrics.rs

@@ -0,0 +1,205 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//! Control-loop and stream counters for scout. Every event here is metric-only
+//! (`log = off`): the human-readable side is already carried by the existing
+//! `tracing` lines at each site, and the process-wide log rate rides on
+//! `carbide_log_events_total{level, component}` once `log_events::register`
+//! runs. These counters add the per-action and per-stream-outcome rates that a
+//! log line alone can't total.
+
+use carbide_instrument::{Event, LabelValue, Outcome};
+use rpc::forge_agent_control_response as fac;
+
+/// Which control-loop action scout handled, as a bounded metric label: one
+/// variant per [`fac::Action`] arm the service loop can dispatch.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, LabelValue)]
+pub enum ScoutAction {
+    Noop,
+    Reset,
+    Discovery,
+    Rebuild,
+    Retry,
+    Measure,
+    LogError,
+    MachineValidation,
+    MlxAction,
+    FirmwareUpgrade,
+}
+
+impl From<&fac::Action> for ScoutAction {
+    fn from(action: &fac::Action) -> Self {
+        match action {
+            fac::Action::Noop(_) => Self::Noop,
+            fac::Action::Reset(_) => Self::Reset,
+            fac::Action::Discovery(_) => Self::Discovery,
+            fac::Action::Rebuild(_) => Self::Rebuild,
+            fac::Action::Retry(_) => Self::Retry,
+            fac::Action::Measure(_) => Self::Measure,
+            fac::Action::LogError(_) => Self::LogError,
+            fac::Action::MachineValidation(_) => Self::MachineValidation,
+            fac::Action::MlxAction(_) => Self::MlxAction,
+            fac::Action::FirmwareUpgrade(_) => Self::FirmwareUpgrade,
+        }
+    }
+}
+
+/// Scout finished handling one control-loop action, whatever the result.
+#[derive(Event)]
+#[event(
+    name = "carbide_scout_actions_total",
+    component = "nico-scout",
+    log = off,
+    metric = counter,
+    describe = "Number of scout control-loop actions handled, by action and outcome."
+)]
+pub struct ScoutActionHandled {
+    #[label]
+    pub action: ScoutAction,
+    #[label]
+    pub outcome: Outcome,
+}
+
+/// A scout stream connection attempt resolved -- `ok` once the bidirectional
+/// stream is established, `error` when the client could not be built.
+#[derive(Event)]
+#[event(
+    name = "carbide_scout_stream_connections_total",
+    component = "nico-scout",
+    log = off,
+    metric = counter,
+    describe = "Number of scout stream connection attempts, by outcome."
+)]
+pub struct ScoutStreamConnection {
+    #[label]
+    pub outcome: Outcome,
+}
+
+/// The scout stream closed or errored and the loop looped back to re-establish
+/// it after the reconnect delay.
+#[derive(Event)]
+#[event(
+    name = "carbide_scout_stream_reconnects_total",
+    component = "nico-scout",
+    log = off,
+    metric = counter,
+    describe = "Number of scout stream reconnect cycles after a stream closed or errored."
+)]
+pub struct ScoutStreamReconnect {}
+
+#[cfg(test)]
+mod tests {
+    use carbide_instrument::emit;
+    use carbide_instrument::testing::MetricsCapture;
+    use carbide_test_support::{Check, check_values};
+
+    use super::*;
+
+    #[test]
+    fn scout_action_maps_every_dispatchable_action() {
+        check_values(
+            [
+                Check {
+                    scenario: "noop",
+                    input: fac::Action::Noop(fac::Noop {}),
+                    expect: ScoutAction::Noop,
+                },
+                Check {
+                    scenario: "reset",
+                    input: fac::Action::Reset(fac::Reset {}),
+                    expect: ScoutAction::Reset,
+                },
+                Check {
+                    scenario: "discovery",
+                    input: fac::Action::Discovery(fac::Discovery {}),
+                    expect: ScoutAction::Discovery,
+                },
+                Check {
+                    scenario: "rebuild",
+                    input: fac::Action::Rebuild(fac::Rebuild {}),
+                    expect: ScoutAction::Rebuild,
+                },
+                Check {
+                    scenario: "retry",
+                    input: fac::Action::Retry(fac::Retry {}),
+                    expect: ScoutAction::Retry,
+                },
+                Check {
+                    scenario: "measure",
+                    input: fac::Action::Measure(fac::Measure {}),
+                    expect: ScoutAction::Measure,
+                },
+                Check {
+                    scenario: "log error",
+                    input: fac::Action::LogError(fac::LogError {}),
+                    expect: ScoutAction::LogError,
+                },
+                Check {
+                    scenario: "machine validation",
+                    input: fac::Action::MachineValidation(fac::MachineValidation::default()),
+                    expect: ScoutAction::MachineValidation,
+                },
+                Check {
+                    scenario: "mlx action",
+                    input: fac::Action::MlxAction(fac::MlxAction::default()),
+                    expect: ScoutAction::MlxAction,
+                },
+                Check {
+                    scenario: "firmware upgrade",
+                    input: fac::Action::FirmwareUpgrade(fac::FirmwareUpgrade::default()),
+                    expect: ScoutAction::FirmwareUpgrade,
+                },
+            ],
+            |action| ScoutAction::from(&action),
+        );
+    }
+
+    #[test]
+    fn scout_counters_move_per_label() {
+        let metrics = MetricsCapture::start();
+
+        // Labels chosen so no other test in this binary shares them.
+        emit(ScoutActionHandled {
+            action: ScoutAction::FirmwareUpgrade,
+            outcome: Outcome::Error,
+        });
+        emit(ScoutStreamConnection {
+            outcome: Outcome::Ok,
+        });
+        emit(ScoutStreamReconnect {});
+        emit(ScoutStreamReconnect {});
+
+        assert_eq!(
+            metrics.counter_delta(
+                "carbide_scout_actions_total",
+                &[("action", "firmware_upgrade"), ("outcome", "error")],
+            ),
+            1.0
+        );
+        assert_eq!(
+            metrics.counter_delta(
+                "carbide_scout_stream_connections_total",
+                &[("outcome", "ok")],
+            ),
+            1.0
+        );
+        assert_eq!(
+            metrics.counter_delta("carbide_scout_stream_reconnects_total", &[]),
+            2.0
+        );
+    }
+}