nvlink: use switch state machine to rotate certs

crates/api-core/src/handlers/component_manager.rs

@@ -28,7 +28,7 @@ use carbide_uuid::machine::MachineId;
 use carbide_uuid::power_shelf::PowerShelfId;
 use carbide_uuid::rack::RackId;
 use carbide_uuid::switch::SwitchId;
-use component_manager::component_manager::ComponentManager;
+use component_manager::component_manager::{ComponentManager, SwitchMaintenanceRequestResult};
 use component_manager::compute_tray_manager::{ComputeTrayEndpoint, ComputeTrayVendor};
 use component_manager::error::ComponentManagerError;
 use component_manager::nv_switch_manager::SwitchEndpoint;
@@ -47,7 +47,6 @@ use model::rack::{FirmwareUpgradeJob, MaintenanceActivity};
 use model::switch::SwitchMaintenanceOperation;
 use tonic::{Code, Request, Response, Status};
 
-use crate::CarbideError;
 use crate::api::{Api, log_request_data, log_request_data_redacted};
 
 const MACHINE_POWER_OVERRIDE_SOURCE: &str = "component_power_control";
@@ -65,7 +64,7 @@ fn unsupported_from_json_firmware_versions(target: &str) -> Status {
     ))
 }
 
-fn component_manager_error_to_status(err: ComponentManagerError) -> Status {
+pub(crate) fn component_manager_error_to_status(err: ComponentManagerError) -> Status {
     match err {
         ComponentManagerError::Unavailable(msg) => Status::unavailable(msg),
         ComponentManagerError::NotFound(msg) => Status::not_found(msg),
@@ -303,61 +302,43 @@ fn map_switch_maintenance_operation(action: PowerAction) -> SwitchMaintenanceOpe
 
 async fn queue_switch_power_control_via_state_controller(
     api: &Api,
+    cm: &ComponentManager,
     switch_ids: &[SwitchId],
     action: PowerAction,
 ) -> Result<Vec<rpc::ComponentResult>, Status> {
     let operation = map_switch_maintenance_operation(action);
-    queue_switch_maintenance_via_state_controller(api, switch_ids, operation).await
+    queue_switch_maintenance_via_state_controller(api, cm, switch_ids, operation).await
 }
 
 async fn queue_switch_maintenance_via_state_controller(
     api: &Api,
+    cm: &ComponentManager,
     switch_ids: &[SwitchId],
     operation: SwitchMaintenanceOperation,
 ) -> Result<Vec<rpc::ComponentResult>, Status> {
-    let mut txn = api.txn_begin().await?;
-    let existing = db::switch::find_by(
-        &mut txn,
-        db::ObjectColumnFilter::List(db::switch::IdColumn, switch_ids),
-    )
-    .await
-    .map_err(CarbideError::from)?;
-
-    let by_id: HashMap<SwitchId, model::switch::Switch> =
-        existing.into_iter().map(|sw| (sw.id, sw)).collect();
-    let mut results = Vec::with_capacity(switch_ids.len());
-
-    for switch_id in switch_ids {
-        let Some(switch) = by_id.get(switch_id) else {
-            results.push(error_result(
-                &switch_id.to_string(),
-                format!("switch {switch_id} not found"),
-            ));
-            continue;
-        };
-
-        if switch.is_marked_as_deleted() {
-            results.push(error_result(
-                &switch_id.to_string(),
-                format!("switch {switch_id} is marked for deletion"),
-            ));
-            continue;
-        }
-
-        db::switch::set_switch_maintenance_requested(
-            &mut txn,
-            *switch_id,
-            "component-manager",
+    let results = cm
+        .request_switch_maintenance_via_state_controller(
+            &api.database_connection,
+            switch_ids,
             operation,
+            "component-manager",
         )
         .await
-        .map_err(CarbideError::from)?;
+        .map_err(component_manager_error_to_status)?;
 
-        results.push(success_result(&switch_id.to_string()));
-    }
+    Ok(results
+        .iter()
+        .map(switch_maintenance_request_result_to_component_result)
+        .collect())
+}
 
-    txn.commit().await?;
-    Ok(results)
+fn switch_maintenance_request_result_to_component_result(
+    result: &SwitchMaintenanceRequestResult,
+) -> rpc::ComponentResult {
+    match &result.error {
+        Some(error) => error_result(&result.switch_id.to_string(), error.clone()),
+        None => success_result(&result.switch_id.to_string()),
+    }
 }
 
 /// Maps raw proto `ComputeTrayComponent` values to display-name strings.
@@ -1241,7 +1222,8 @@ pub(crate) async fn component_power_control(
         rpc::component_power_control_request::Target::SwitchIds(list) => {
             if cm.nv_switch_use_state_controller && !bypass_state_controller {
                 let results =
-                    queue_switch_power_control_via_state_controller(api, &list.ids, action).await?;
+                    queue_switch_power_control_via_state_controller(api, cm, &list.ids, action)
+                        .await?;
                 (results, Vec::new())
             } else {
                 let endpoints = resolve_switch_endpoints(api, &list.ids).await?;
@@ -1458,6 +1440,7 @@ pub(crate) async fn component_configure_switch_certificate(
     if cm.nv_switch_use_state_controller && !bypass_state_controller {
         let results = queue_switch_maintenance_via_state_controller(
             api,
+            cm,
             &switch_ids.ids,
             SwitchMaintenanceOperation::ReconfigureCertificate,
         )

crates/api-core/src/handlers/rack.rs

@@ -22,11 +22,14 @@ use ::rpc::forge::{self as rpc, HealthReportEntry};
 use carbide_rack::firmware_object::{
     rack_maintenance_access_token_key, rms_access_token_or_noauth,
 };
-use carbide_secrets::credentials::Credentials;
 use carbide_uuid::machine::MachineId;
 use carbide_uuid::power_shelf::PowerShelfId;
 use carbide_uuid::rack::RackId;
 use carbide_uuid::switch::SwitchId;
+use component_manager::component_manager::{
+    RackMaintenanceAccessToken, RackMaintenanceEligibility, RackMaintenanceRequestOutcome,
+    request_rack_maintenance_via_state_controller,
+};
 use db::{
     ObjectColumnFilter, WithTransaction, machine as db_machine, power_shelf as db_power_shelf,
     rack as db_rack, switch as db_switch,
@@ -41,6 +44,7 @@ use tonic::{Request, Response, Status};
 use crate::CarbideError;
 use crate::api::{Api, log_request_data, log_request_data_redacted};
 use crate::auth::AuthContext;
+use crate::handlers::component_manager::component_manager_error_to_status;
 
 pub async fn get_rack(
     api: &Api,
@@ -766,58 +770,44 @@ pub(crate) async fn on_demand_rack_maintenance(
         }
     }
 
-    let access_token_stored = maintenance_access_token.is_some();
-    if let Some(token) = maintenance_access_token {
-        api.credential_manager
-            .set_credentials(
-                &rack_maintenance_access_token_key(&rack_id),
-                &Credentials::UsernamePassword {
-                    username: "access_token".into(),
-                    password: token,
-                },
-            )
-            .await
-            .map_err(|error| CarbideError::Internal {
-                message: format!("failed to store rack maintenance access token: {error}"),
-            })?;
-    }
-
-    let mut updated_config = rack.config.clone();
-    updated_config.maintenance_requested = Some(scope);
+    let maintenance_access_token =
+        maintenance_access_token.map(|token| RackMaintenanceAccessToken {
+            credential_manager: api.credential_manager.as_ref(),
+            token,
+        });
 
-    let db_result: Result<(), Status> = async {
-        let mut txn = api.txn_begin().await?;
-        db_rack::update(&mut txn, &rack_id, &updated_config).await?;
-        if updated_config
-            .maintenance_requested
-            .as_ref()
-            .is_some_and(|scope| {
-                scope.should_run(&MaintenanceActivity::FirmwareUpgrade {
-                    firmware_version: None,
-                    components: vec![],
-                    force_update: false,
-                })
-            })
-        {
-            db_rack::update_firmware_upgrade_job(txn.as_mut(), &rack_id, None).await?;
-        }
-        txn.commit().await?;
-        Ok(())
-    }
+    let schedule_result = request_rack_maintenance_via_state_controller(
+        &api.database_connection,
+        &rack_id,
+        scope,
+        RackMaintenanceEligibility::AllowErrorRecovery,
+        maintenance_access_token,
+    )
     .await;
-    if let Err(status) = db_result {
-        if access_token_stored
-            && let Err(error) = api
-                .credential_manager
-                .delete_credentials(&rack_maintenance_access_token_key(&rack_id))
-                .await
-        {
-            tracing::warn!(
-                rack_id = %rack_id,
-                error = %error,
-                "failed to delete rack maintenance access token after DB error",
-            );
-        }
+
+    let scheduling_error = match schedule_result {
+        Ok(
+            RackMaintenanceRequestOutcome::Scheduled
+            | RackMaintenanceRequestOutcome::AlreadyPending,
+        ) => None,
+        Ok(RackMaintenanceRequestOutcome::Busy) => Some(
+            CarbideError::InvalidArgument(format!(
+                "On-demand maintenance for rack {} is already scheduled.",
+                rack_id,
+            ))
+            .into(),
+        ),
+        Ok(RackMaintenanceRequestOutcome::Deferred { state }) => Some(
+            CarbideError::InvalidArgument(format!(
+                "Rack {} is not in Ready or Error state (current: {:?}). Maintenance can only be requested when the rack is Ready or in Error.",
+                rack_id, state,
+            ))
+            .into(),
+        ),
+        Err(error) => Some(component_manager_error_to_status(error)),
+    };
+
+    if let Some(status) = scheduling_error {
         return Err(status);
     }
 

crates/api-core/src/setup.rs

@@ -1493,9 +1493,7 @@ async fn initialize_and_start_controllers<'a>(
         meter: meter.clone(),
         config: carbide_config.nvlink_config.clone().unwrap_or_default(),
         host_health: carbide_config.host_health,
-        rms_client: api_service.rms_client.clone(),
-        credential_manager: api_service.credential_manager.clone(),
-        rack_profiles: carbide_config.rack_profiles.clone(),
+        component_manager: api_service.component_manager.clone().map(Arc::new),
         work_lock_manager_handle: work_lock_manager_handle.clone(),
     })
     .start(join_set, cancel_token.clone())?;

crates/api-core/src/tests/common/api_fixtures/mod.rs

@@ -1455,9 +1455,7 @@ pub async fn create_test_env_with_overrides(
         db_pool.clone(),
         test_meter.meter(),
         config.nvlink_config.clone().unwrap(),
-        rms_sim.as_rms_client(),
-        composite_manager.clone(),
-        config.rack_profiles.clone(),
+        test_component_manager.clone(),
         api.work_lock_manager_handle.clone(),
     );