diff --git a/Cargo.lock b/Cargo.lock index 5beaa66a83..1ab335d0f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2711,6 +2711,7 @@ dependencies = [ "tokio-util", "tonic", "tracing", + "uuid", ] [[package]] @@ -2740,6 +2741,7 @@ dependencies = [ "state-controller", "tonic", "tracing", + "uuid", ] [[package]] @@ -2957,6 +2959,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tracing", + "uuid", "vaultrs", ] diff --git a/crates/api-core/src/handlers/component_manager.rs b/crates/api-core/src/handlers/component_manager.rs index 2ce59fbadc..07f843abc3 100644 --- a/crates/api-core/src/handlers/component_manager.rs +++ b/crates/api-core/src/handlers/component_manager.rs @@ -68,7 +68,7 @@ fn unsupported_from_json_firmware_versions(target: &str) -> Status { )) } -fn component_manager_error_to_status(err: ComponentManagerError) -> Status { +pub(crate) fn component_manager_error_to_status(err: ComponentManagerError) -> Status { match err { ComponentManagerError::Unavailable(msg) => Status::unavailable(msg), ComponentManagerError::NotFound(msg) => Status::not_found(msg), diff --git a/crates/api-core/src/handlers/rack.rs b/crates/api-core/src/handlers/rack.rs index da3bacbe1d..8ede448077 100644 --- a/crates/api-core/src/handlers/rack.rs +++ b/crates/api-core/src/handlers/rack.rs @@ -22,11 +22,14 @@ use ::rpc::forge::{self as rpc, HealthReportEntry}; use carbide_rack::firmware_object::{ rack_maintenance_access_token_key, rms_access_token_or_noauth, }; -use carbide_secrets::credentials::Credentials; use carbide_uuid::machine::MachineId; use carbide_uuid::power_shelf::PowerShelfId; use carbide_uuid::rack::RackId; use carbide_uuid::switch::SwitchId; +use component_manager::component_manager::{ + RackMaintenanceAccessToken, RackMaintenanceEligibility, RackMaintenanceRequestOutcome, + request_rack_maintenance_via_state_controller, +}; use db::{ ObjectColumnFilter, WithTransaction, machine as db_machine, power_shelf as db_power_shelf, rack as db_rack, switch as db_switch, @@ -41,6 +44,7 @@ use tonic::{Request, Response, Status}; use crate::CarbideError; use crate::api::{Api, log_request_data, log_request_data_redacted}; use crate::auth::AuthContext; +use crate::handlers::component_manager::component_manager_error_to_status; pub async fn get_rack( api: &Api, @@ -242,6 +246,8 @@ pub async fn admin_force_delete_rack( .into()); } + let maintenance_request_id = rack_list[0].config.maintenance_request_id; + db_rack::final_delete(&mut txn, &rack_id) .await .map_err(CarbideError::from)?; @@ -250,7 +256,10 @@ pub async fn admin_force_delete_rack( if let Err(error) = api .credential_manager - .delete_credentials(&rack_maintenance_access_token_key(&rack_id)) + .delete_credentials(&rack_maintenance_access_token_key( + &rack_id, + maintenance_request_id, + )) .await { tracing::warn!( @@ -766,58 +775,44 @@ pub(crate) async fn on_demand_rack_maintenance( } } - let access_token_stored = maintenance_access_token.is_some(); - if let Some(token) = maintenance_access_token { - api.credential_manager - .set_credentials( - &rack_maintenance_access_token_key(&rack_id), - &Credentials::UsernamePassword { - username: "access_token".into(), - password: token, - }, - ) - .await - .map_err(|error| CarbideError::Internal { - message: format!("failed to store rack maintenance access token: {error}"), - })?; - } - - let mut updated_config = rack.config.clone(); - updated_config.maintenance_requested = Some(scope); + let maintenance_access_token = + maintenance_access_token.map(|token| RackMaintenanceAccessToken { + credential_manager: api.credential_manager.as_ref(), + token, + }); - let db_result: Result<(), Status> = async { - let mut txn = api.txn_begin().await?; - db_rack::update(&mut txn, &rack_id, &updated_config).await?; - if updated_config - .maintenance_requested - .as_ref() - .is_some_and(|scope| { - scope.should_run(&MaintenanceActivity::FirmwareUpgrade { - firmware_version: None, - components: vec![], - force_update: false, - }) - }) - { - db_rack::update_firmware_upgrade_job(txn.as_mut(), &rack_id, None).await?; - } - txn.commit().await?; - Ok(()) - } + let schedule_result = request_rack_maintenance_via_state_controller( + &api.database_connection, + &rack_id, + scope, + RackMaintenanceEligibility::AllowError, + maintenance_access_token, + ) .await; - if let Err(status) = db_result { - if access_token_stored - && let Err(error) = api - .credential_manager - .delete_credentials(&rack_maintenance_access_token_key(&rack_id)) - .await - { - tracing::warn!( - rack_id = %rack_id, - error = %error, - "failed to delete rack maintenance access token after DB error", - ); - } + + let scheduling_error = match schedule_result { + Ok( + RackMaintenanceRequestOutcome::Scheduled { .. } + | RackMaintenanceRequestOutcome::AlreadyPending { .. }, + ) => None, + Ok(RackMaintenanceRequestOutcome::Busy) => Some( + CarbideError::InvalidArgument(format!( + "On-demand maintenance for rack {} is already scheduled.", + rack_id, + )) + .into(), + ), + Ok(RackMaintenanceRequestOutcome::Deferred { state }) => Some( + CarbideError::InvalidArgument(format!( + "Rack {} is not in Ready or Error state (current: {:?}). Maintenance can only be requested when the rack is Ready or in Error.", + rack_id, state, + )) + .into(), + ), + Err(error) => Some(component_manager_error_to_status(error)), + }; + + if let Some(status) = scheduling_error { return Err(status); } diff --git a/crates/api-core/src/tests/rack_state_controller/handler.rs b/crates/api-core/src/tests/rack_state_controller/handler.rs index 38503ced54..c2ef4912b8 100644 --- a/crates/api-core/src/tests/rack_state_controller/handler.rs +++ b/crates/api-core/src/tests/rack_state_controller/handler.rs @@ -36,7 +36,8 @@ use model::rack::{ ConfigureNmxClusterCertificateState, ConfigureNmxClusterState, FirmwareUpgradeDeviceStatus, FirmwareUpgradeJob, FirmwareUpgradeState, MaintenanceActivity, MaintenanceScope, NvosUpdateState, NvosUpdateSwitchStatus, Rack, RackConfig, RackFirmwareUpgradeState, - RackMaintenanceState, RackPowerState, RackState, RackValidationState, + RackMaintenanceRequestStatus, RackMaintenanceState, RackPowerState, RackState, + RackValidationState, }; use model::rack_type::{ RackCapabilitiesSet, RackCapabilityCompute, RackCapabilityPowerShelf, RackCapabilitySwitch, @@ -341,6 +342,49 @@ pub(crate) fn new_rack_id() -> RackId { RackId::new(uuid::Uuid::new_v4().to_string()) } +async fn create_ready_rack_with_maintenance_request( + pool: &sqlx::PgPool, + scope: MaintenanceScope, +) -> Result<(RackId, uuid::Uuid), Box> { + let rack_id = new_rack_id(); + let request_id = uuid::Uuid::new_v4(); + let config = RackConfig { + maintenance_requested: Some(scope.clone()), + maintenance_request_id: Some(request_id), + ..Default::default() + }; + let mut txn = pool.begin().await?; + let rack = db_rack::create( + txn.as_mut(), + &rack_id, + Some(&RackProfileId::new("Empty")), + &config, + None, + ) + .await?; + db::rack_maintenance_request::insert_preparing( + txn.as_mut(), + request_id, + &rack_id, + &scope, + false, + ) + .await?; + assert!(db::rack_maintenance_request::mark_ready(txn.as_mut(), request_id).await?); + assert!( + db_rack::try_update_controller_state( + txn.as_mut(), + &rack_id, + rack.controller_state.version, + rack.controller_state.version.increment(), + &RackState::Ready, + ) + .await? + ); + txn.commit().await?; + Ok((rack_id, request_id)) +} + async fn create_ready_rack_with_switch( env: &TestEnv, pool: &sqlx::PgPool, @@ -1181,6 +1225,147 @@ async fn test_error_state_does_nothing( Ok(()) } +#[crate::sqlx_test] +async fn test_first_class_maintenance_request_runs_and_completes( + pool: sqlx::PgPool, +) -> Result<(), Box> { + let env = create_test_env_with_overrides(pool.clone(), TestEnvOverrides::default()).await; + let scope = MaintenanceScope { + activities: vec![MaintenanceActivity::ConfigureNmxCluster], + ..Default::default() + }; + let (rack_id, request_id) = create_ready_rack_with_maintenance_request(&pool, scope).await?; + let mut rack = get_db_rack(env.db_reader().as_mut(), &rack_id).await; + + let handler = RackStateHandler::default(); + let mut services = env.rack_state_handler_services(); + let mut metrics = RackMetrics::default(); + let mut db_writes = DbWriteBatch::default(); + let mut ctx = StateHandlerContext:: { + services: &mut services, + metrics: &mut metrics, + pending_db_writes: &mut db_writes, + }; + + let mut outcome = handler + .handle_object_state(&rack_id, &mut rack, &RackState::Ready, &mut ctx) + .await?; + assert!(matches!( + &outcome, + StateHandlerOutcome::Transition { + next_state: RackState::Maintenance { .. }, + .. + } + )); + if let Some(txn) = outcome.take_transaction() { + txn.commit().await?; + } + + let mut conn = pool.acquire().await?; + let request = db::rack_maintenance_request::find_by_id(conn.as_mut(), request_id) + .await? + .unwrap(); + assert_eq!(request.status, RackMaintenanceRequestStatus::Running); + assert!(request.started.is_some()); + drop(conn); + + let completed_state = RackState::Maintenance { + maintenance_state: RackMaintenanceState::Completed, + }; + let mut outcome = handler + .handle_object_state(&rack_id, &mut rack, &completed_state, &mut ctx) + .await?; + assert!(matches!( + &outcome, + StateHandlerOutcome::Transition { + next_state: RackState::Validating { + validating_state: RackValidationState::Pending, + }, + .. + } + )); + if let Some(txn) = outcome.take_transaction() { + txn.commit().await?; + } + + let mut conn = pool.acquire().await?; + let request = db::rack_maintenance_request::find_by_id(conn.as_mut(), request_id) + .await? + .unwrap(); + assert_eq!(request.status, RackMaintenanceRequestStatus::Completed); + assert!(request.completed.is_some()); + let rack = get_db_rack(conn.as_mut(), &rack_id).await; + assert!(rack.config.maintenance_requested.is_none()); + assert!(rack.config.maintenance_request_id.is_none()); + + Ok(()) +} + +#[crate::sqlx_test] +async fn test_first_class_maintenance_request_fails_with_maintenance( + pool: sqlx::PgPool, +) -> Result<(), Box> { + let env = create_test_env_with_overrides(pool.clone(), TestEnvOverrides::default()).await; + let scope = MaintenanceScope { + activities: vec![MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }], + ..Default::default() + }; + let (rack_id, request_id) = create_ready_rack_with_maintenance_request(&pool, scope).await?; + let mut rack = get_db_rack(env.db_reader().as_mut(), &rack_id).await; + + let handler = RackStateHandler::default(); + let mut services = env.rack_state_handler_services(); + let mut metrics = RackMetrics::default(); + let mut db_writes = DbWriteBatch::default(); + let mut ctx = StateHandlerContext:: { + services: &mut services, + metrics: &mut metrics, + pending_db_writes: &mut db_writes, + }; + + let mut outcome = handler + .handle_object_state(&rack_id, &mut rack, &RackState::Ready, &mut ctx) + .await?; + if let Some(txn) = outcome.take_transaction() { + txn.commit().await?; + } + + let firmware_state = RackState::Maintenance { + maintenance_state: RackMaintenanceState::FirmwareUpgrade { + rack_firmware_upgrade: FirmwareUpgradeState::Start, + }, + }; + let mut outcome = handler + .handle_object_state(&rack_id, &mut rack, &firmware_state, &mut ctx) + .await?; + assert!(matches!( + &outcome, + StateHandlerOutcome::Transition { + next_state: RackState::Error { .. }, + .. + } + )); + if let Some(txn) = outcome.take_transaction() { + txn.commit().await?; + } + + let mut conn = pool.acquire().await?; + let request = db::rack_maintenance_request::find_by_id(conn.as_mut(), request_id) + .await? + .unwrap(); + assert_eq!(request.status, RackMaintenanceRequestStatus::Failed); + assert!(request.error_message.is_some()); + let rack = get_db_rack(conn.as_mut(), &rack_id).await; + assert!(rack.config.maintenance_requested.is_none()); + assert!(rack.config.maintenance_request_id.is_none()); + + Ok(()) +} + /// test_maintenance_completed_transitions_to_validation verifies that /// Maintenance::Completed transitions to Validation(Pending). #[crate::sqlx_test] diff --git a/crates/api-db/migrations/20260714120000_rack_maintenance_requests.sql b/crates/api-db/migrations/20260714120000_rack_maintenance_requests.sql new file mode 100644 index 0000000000..5d8081ba2e --- /dev/null +++ b/crates/api-db/migrations/20260714120000_rack_maintenance_requests.sql @@ -0,0 +1,31 @@ +-- First-class lifecycle for on-demand rack maintenance. +-- +-- The request row is the durable coordination point between API/monitor +-- callers, the external credential store, and the rack state controller. +-- Secret material remains outside Postgres. + +CREATE TYPE rack_maintenance_request_status AS ENUM + ('preparing', 'ready', 'running', 'completed', 'failed', 'cancelled'); + +CREATE TABLE rack_maintenance_requests ( + id uuid PRIMARY KEY, + rack_id varchar(64) NOT NULL REFERENCES racks(id) ON DELETE CASCADE, + scope jsonb NOT NULL, + status rack_maintenance_request_status NOT NULL, + requires_access_token boolean NOT NULL DEFAULT false, + error_message text, + created timestamptz NOT NULL DEFAULT now(), + updated timestamptz NOT NULL DEFAULT now(), + started timestamptz, + completed timestamptz +); + +-- A rack can have at most one request being prepared, queued, or executed. +-- Terminal rows are retained for audit and do not block future requests. +CREATE UNIQUE INDEX rack_maintenance_requests_one_active_per_rack + ON rack_maintenance_requests (rack_id) + WHERE status IN ('preparing', 'ready', 'running'); + +CREATE INDEX rack_maintenance_requests_rack_created + ON rack_maintenance_requests (rack_id, created DESC); + diff --git a/crates/api-db/src/lib.rs b/crates/api-db/src/lib.rs index d55ae36ae6..ce73195f77 100644 --- a/crates/api-db/src/lib.rs +++ b/crates/api-db/src/lib.rs @@ -80,6 +80,7 @@ pub mod power_shelf; pub mod predicted_machine_interface; pub mod queries; pub mod rack; +pub mod rack_maintenance_request; pub mod redfish_actions; pub mod resource_pool; pub mod retained_boot_interface; diff --git a/crates/api-db/src/rack.rs b/crates/api-db/src/rack.rs index c2b3cb043f..7b575848fc 100644 --- a/crates/api-db/src/rack.rs +++ b/crates/api-db/src/rack.rs @@ -58,6 +58,25 @@ where .map_err(|e| DatabaseError::new(query.sql(), e)) } +/// Load one active rack while holding a row lock until the caller's transaction +/// completes. +/// +/// Callers that make a read/check/write decision about rack state or config +/// must use this instead of [`find_by`]. In particular, it prevents two +/// concurrent maintenance schedulers from both observing an empty +/// `maintenance_requested` field and then overwriting each other. +pub async fn find_by_id_for_update( + txn: &mut PgConnection, + rack_id: &RackId, +) -> DatabaseResult> { + let query = "SELECT * FROM racks WHERE id = $1 AND deleted IS NULL FOR UPDATE"; + sqlx::query_as(query) + .bind(rack_id) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::new(query, e)) +} + pub async fn find_ids( txn: impl DbReader<'_>, filter: model::rack::RackSearchFilter, diff --git a/crates/api-db/src/rack_maintenance_request.rs b/crates/api-db/src/rack_maintenance_request.rs new file mode 100644 index 0000000000..182cf7c036 --- /dev/null +++ b/crates/api-db/src/rack_maintenance_request.rs @@ -0,0 +1,178 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use carbide_uuid::rack::RackId; +use model::rack::{MaintenanceScope, RackMaintenanceRequest, RackMaintenanceRequestStatus}; +use sqlx::PgConnection; +use uuid::Uuid; + +use crate::{DatabaseError, DatabaseResult}; + +pub async fn find_by_id( + conn: &mut PgConnection, + request_id: Uuid, +) -> DatabaseResult> { + let query = "SELECT * FROM rack_maintenance_requests WHERE id = $1"; + sqlx::query_as(query) + .bind(request_id) + .fetch_optional(conn) + .await + .map_err(|error| DatabaseError::new(query, error)) +} + +pub async fn find_active_for_rack_for_update( + conn: &mut PgConnection, + rack_id: &RackId, +) -> DatabaseResult> { + let query = "SELECT * FROM rack_maintenance_requests \ + WHERE rack_id = $1 AND status IN ('preparing', 'ready', 'running') \ + FOR UPDATE"; + sqlx::query_as(query) + .bind(rack_id) + .fetch_optional(conn) + .await + .map_err(|error| DatabaseError::new(query, error)) +} + +pub async fn insert_preparing( + conn: &mut PgConnection, + request_id: Uuid, + rack_id: &RackId, + scope: &MaintenanceScope, + requires_access_token: bool, +) -> DatabaseResult { + let query = "INSERT INTO rack_maintenance_requests \ + (id, rack_id, scope, status, requires_access_token) \ + VALUES ($1, $2, $3, 'preparing', $4) RETURNING *"; + sqlx::query_as(query) + .bind(request_id) + .bind(rack_id) + .bind(sqlx::types::Json(scope)) + .bind(requires_access_token) + .fetch_one(conn) + .await + .map_err(|error| DatabaseError::new(query, error)) +} + +pub async fn mark_ready(conn: &mut PgConnection, request_id: Uuid) -> DatabaseResult { + transition( + conn, + request_id, + RackMaintenanceRequestStatus::Preparing, + RackMaintenanceRequestStatus::Ready, + None, + ) + .await +} + +pub async fn mark_running(conn: &mut PgConnection, request_id: Uuid) -> DatabaseResult { + transition( + conn, + request_id, + RackMaintenanceRequestStatus::Ready, + RackMaintenanceRequestStatus::Running, + None, + ) + .await +} + +pub async fn mark_completed(conn: &mut PgConnection, request_id: Uuid) -> DatabaseResult { + transition( + conn, + request_id, + RackMaintenanceRequestStatus::Running, + RackMaintenanceRequestStatus::Completed, + None, + ) + .await +} + +pub async fn mark_failed( + conn: &mut PgConnection, + request_id: Uuid, + error_message: &str, +) -> DatabaseResult { + let query = "UPDATE rack_maintenance_requests \ + SET status = 'failed', error_message = $2, updated = now(), completed = now() \ + WHERE id = $1 AND status IN ('preparing', 'ready', 'running')"; + sqlx::query(query) + .bind(request_id) + .bind(error_message) + .execute(conn) + .await + .map(|result| result.rows_affected() == 1) + .map_err(|error| DatabaseError::new(query, error)) +} + +pub async fn mark_preparing_failed( + conn: &mut PgConnection, + request_id: Uuid, + error_message: &str, +) -> DatabaseResult { + let query = "UPDATE rack_maintenance_requests \ + SET status = 'failed', error_message = $2, updated = now(), completed = now() \ + WHERE id = $1 AND status = 'preparing'"; + sqlx::query(query) + .bind(request_id) + .bind(error_message) + .execute(conn) + .await + .map(|result| result.rows_affected() == 1) + .map_err(|error| DatabaseError::new(query, error)) +} + +pub async fn mark_cancelled( + conn: &mut PgConnection, + request_id: Uuid, + reason: &str, +) -> DatabaseResult { + let query = "UPDATE rack_maintenance_requests \ + SET status = 'cancelled', error_message = $2, updated = now(), completed = now() \ + WHERE id = $1 AND status IN ('preparing', 'ready')"; + sqlx::query(query) + .bind(request_id) + .bind(reason) + .execute(conn) + .await + .map(|result| result.rows_affected() == 1) + .map_err(|error| DatabaseError::new(query, error)) +} + +async fn transition( + conn: &mut PgConnection, + request_id: Uuid, + expected: RackMaintenanceRequestStatus, + next: RackMaintenanceRequestStatus, + error_message: Option<&str>, +) -> DatabaseResult { + let terminal = next.is_terminal(); + let query = "UPDATE rack_maintenance_requests \ + SET status = $3, error_message = $4, updated = now(), \ + started = CASE WHEN $3 = 'running' THEN now() ELSE started END, \ + completed = CASE WHEN $5 THEN now() ELSE completed END \ + WHERE id = $1 AND status = $2"; + sqlx::query(query) + .bind(request_id) + .bind(expected) + .bind(next) + .bind(error_message) + .bind(terminal) + .execute(conn) + .await + .map(|result| result.rows_affected() == 1) + .map_err(|error| DatabaseError::new(query, error)) +} diff --git a/crates/api-model/src/rack.rs b/crates/api-model/src/rack.rs index 55ef4a187b..829d39a9a1 100644 --- a/crates/api-model/src/rack.rs +++ b/crates/api-model/src/rack.rs @@ -79,6 +79,59 @@ pub struct Rack { pub version: ConfigVersion, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, sqlx::Type)] +#[serde(rename_all = "snake_case")] +#[sqlx( + type_name = "rack_maintenance_request_status", + rename_all = "snake_case" +)] +pub enum RackMaintenanceRequestStatus { + Preparing, + Ready, + Running, + Completed, + Failed, + Cancelled, +} + +impl RackMaintenanceRequestStatus { + pub fn is_terminal(self) -> bool { + matches!(self, Self::Completed | Self::Failed | Self::Cancelled) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RackMaintenanceRequest { + pub id: uuid::Uuid, + pub rack_id: RackId, + pub scope: MaintenanceScope, + pub status: RackMaintenanceRequestStatus, + pub requires_access_token: bool, + pub error_message: Option, + pub created: DateTime, + pub updated: DateTime, + pub started: Option>, + pub completed: Option>, +} + +impl<'r> FromRow<'r, PgRow> for RackMaintenanceRequest { + fn from_row(row: &'r PgRow) -> Result { + let scope: sqlx::types::Json = row.try_get("scope")?; + Ok(Self { + id: row.try_get("id")?, + rack_id: row.try_get("rack_id")?, + scope: scope.0, + status: row.try_get("status")?, + requires_access_token: row.try_get("requires_access_token")?, + error_message: row.try_get("error_message")?, + created: row.try_get("created")?, + updated: row.try_get("updated")?, + started: row.try_get("started")?, + completed: row.try_get("completed")?, + }) + } +} + #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct FirmwareUpgradeJob { pub job_id: Option, @@ -745,7 +798,7 @@ impl std::fmt::Display for MaintenanceActivity { /// Specifies which devices in the rack should be included in an on-demand /// maintenance cycle. When all three device-id lists are empty, the full rack /// is maintained. -#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)] pub struct MaintenanceScope { #[serde(default)] pub machine_ids: Vec, @@ -787,6 +840,12 @@ pub struct RackConfig { /// selects full-rack vs partial-rack and which activities to run. #[serde(default)] pub maintenance_requested: Option, + + /// Identity of the first-class request backing `maintenance_requested`. + /// `None` preserves compatibility with maintenance queued before request + /// lifecycle tracking was introduced. + #[serde(default)] + pub maintenance_request_id: Option, } /// Reason a rack will not accept a new on-demand maintenance request. diff --git a/crates/component-manager/Cargo.toml b/crates/component-manager/Cargo.toml index f0c630f37d..700549eb30 100644 --- a/crates/component-manager/Cargo.toml +++ b/crates/component-manager/Cargo.toml @@ -44,15 +44,16 @@ tonic = { workspace = true } tonic-prost = { workspace = true } serde = { workspace = true, features = ["derive"] } tracing = { workspace = true } +uuid = { workspace = true } [dev-dependencies] carbide-api-test-helper = { path = "../api-test-helper" } carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-macros = { path = "../macros" } +carbide-secrets = { path = "../secrets", features = ["test-support"] } carbide-sqlx-testing = { path = "../sqlx-testing" } carbide-test-support = { path = "../test-support" } toml = { workspace = true } -uuid = { workspace = true } [build-dependencies] tonic-prost-build = { workspace = true } diff --git a/crates/component-manager/src/component_manager.rs b/crates/component-manager/src/component_manager.rs index e8bb0f7a91..a3d7783056 100644 --- a/crates/component-manager/src/component_manager.rs +++ b/crates/component-manager/src/component_manager.rs @@ -3,8 +3,13 @@ use std::sync::Arc; +use carbide_rack::firmware_object::rack_maintenance_access_token_key; use carbide_redfish::libredfish::RedfishClientPool; +use carbide_secrets::credentials::{CredentialManager, Credentials}; +use carbide_uuid::rack::RackId; +use db::WithTransaction; use librms::RmsApi; +use model::rack::{MaintenanceActivity, MaintenanceScope, RackMaintenanceRequestStatus, RackState}; use model::rack_type::RackProfileConfig; use sqlx::PgPool; @@ -41,6 +46,437 @@ pub struct ComponentManager { pub compute_tray_use_state_controller: bool, } +/// Which rack states a maintenance caller is willing to schedule from. +/// +/// Automatic maintenance uses [`RequireReady`](Self::RequireReady), while the +/// operator-facing API retains its existing ability to recover racks in +/// `Error` via [`AllowError`](Self::AllowError). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RackMaintenanceEligibility { + RequireReady, + AllowError, +} + +/// The complete, non-error result of attempting to schedule rack maintenance. +/// +/// Keeping contention and eligibility as outcomes (rather than string errors) +/// lets periodic callers retry without marking certificate rotation failed, +/// while operator-facing callers can map the same result to their public API. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RackMaintenanceRequestOutcome { + /// This call persisted the requested scope. + Scheduled { request_id: uuid::Uuid }, + /// The exact same scope is already pending. This is an idempotent retry. + AlreadyPending { request_id: Option }, + /// A different maintenance request is pending and was left untouched. + Busy, + /// No request is pending, but the rack's current state is not eligible. + Deferred { state: RackState }, +} + +/// Credential to stage for a rack-maintenance request before it is published +/// to the rack state controller. +pub struct RackMaintenanceAccessToken<'a> { + pub credential_manager: &'a dyn CredentialManager, + pub token: String, +} + +/// Atomically request rack maintenance through the rack state controller. +/// +/// The request is first reserved as `Preparing` under the rack row lock. Any +/// access token is then staged at a request-specific credential path without a +/// database transaction. A second transaction atomically publishes the scope +/// in the rack config and marks the request `Ready`. The rack controller never +/// observes a request whose credential failed to stage. +/// +/// This is a free function because the API supports rack maintenance even when +/// no `ComponentManager` backend is configured. +pub async fn request_rack_maintenance_via_state_controller( + db_pool: &PgPool, + rack_id: &RackId, + scope: MaintenanceScope, + eligibility: RackMaintenanceEligibility, + maintenance_access_token: Option>, +) -> Result { + enum Preparation { + Prepared { request_id: uuid::Uuid }, + Complete(RackMaintenanceRequestOutcome), + } + + let credential_manager = maintenance_access_token + .as_ref() + .map(|credential| credential.credential_manager); + let access_token = maintenance_access_token.map(|credential| credential.token); + let requires_access_token = access_token.is_some(); + let prepare_rack_id = rack_id.clone(); + let prepare_scope = scope.clone(); + let preparation = db_pool + .with_txn(|txn| { + Box::pin(async move { + let rack = db::rack::find_by_id_for_update(txn.as_mut(), &prepare_rack_id) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))? + .ok_or_else(|| { + ComponentManagerError::NotFound(format!("rack {prepare_rack_id} not found")) + })?; + + if let Some(existing_scope) = rack.config.maintenance_requested.as_ref() { + return Ok(Preparation::Complete(if existing_scope == &prepare_scope { + RackMaintenanceRequestOutcome::AlreadyPending { + request_id: rack.config.maintenance_request_id, + } + } else { + RackMaintenanceRequestOutcome::Busy + })); + } + + if let Some(existing) = + db::rack_maintenance_request::find_active_for_rack_for_update( + txn.as_mut(), + &prepare_rack_id, + ) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))? + { + if existing.status != RackMaintenanceRequestStatus::Preparing { + return Ok(Preparation::Complete(if existing.scope == prepare_scope { + RackMaintenanceRequestOutcome::AlreadyPending { + request_id: Some(existing.id), + } + } else { + RackMaintenanceRequestOutcome::Busy + })); + } + + let preparation_age = + chrono::Utc::now().signed_duration_since(existing.updated); + if preparation_age < chrono::Duration::minutes(5) { + return Ok(Preparation::Complete(if existing.scope == prepare_scope { + RackMaintenanceRequestOutcome::AlreadyPending { + request_id: Some(existing.id), + } + } else { + RackMaintenanceRequestOutcome::Busy + })); + } + + if !db::rack_maintenance_request::mark_preparing_failed( + txn.as_mut(), + existing.id, + "rack maintenance request preparation timed out", + ) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))? + { + return Err(ComponentManagerError::Internal(format!( + "stale rack maintenance request {} could not be failed", + existing.id + ))); + } + tracing::warn!( + rack_id = %prepare_rack_id, + request_id = %existing.id, + "Superseding stale rack maintenance request preparation" + ); + } + + let state = rack.controller_state.value.clone(); + let eligible = match eligibility { + RackMaintenanceEligibility::RequireReady => state == RackState::Ready, + RackMaintenanceEligibility::AllowError => { + matches!(&state, RackState::Ready | RackState::Error { .. }) + } + }; + if !eligible { + return Ok(Preparation::Complete( + RackMaintenanceRequestOutcome::Deferred { state }, + )); + } + + let request_id = uuid::Uuid::new_v4(); + db::rack_maintenance_request::insert_preparing( + txn.as_mut(), + request_id, + &prepare_rack_id, + &prepare_scope, + requires_access_token, + ) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))?; + Ok(Preparation::Prepared { request_id }) + }) + }) + .await; + + let request_id = match preparation { + Ok(Ok(Preparation::Prepared { request_id })) => request_id, + Ok(Ok(Preparation::Complete(outcome))) => return Ok(outcome), + Ok(Err(error)) => return Err(error), + Err(error) => return Err(ComponentManagerError::Internal(error.to_string())), + }; + + if let (Some(credential_manager), Some(token)) = (credential_manager, access_token) { + let staging_result = tokio::time::timeout( + std::time::Duration::from_secs(30), + ensure_rack_maintenance_access_token(credential_manager, rack_id, request_id, token), + ) + .await + .map_err(|_| "credential store operation timed out".to_string()) + .and_then(|result| result); + if let Err(error) = staging_result { + if fail_preparing_rack_maintenance_request( + db_pool, + request_id, + "failed to stage rack maintenance access token", + ) + .await + { + cleanup_rack_maintenance_access_token(credential_manager, rack_id, request_id) + .await; + } + return Err(ComponentManagerError::Internal(format!( + "failed to stage rack maintenance access token: {error}" + ))); + } + } + + let finalize_rack_id = rack_id.clone(); + let finalize_scope = scope.clone(); + let finalization = db_pool + .with_txn(|txn| { + Box::pin(async move { + let rack = db::rack::find_by_id_for_update(txn.as_mut(), &finalize_rack_id) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))? + .ok_or_else(|| { + ComponentManagerError::NotFound(format!( + "rack {finalize_rack_id} not found" + )) + })?; + + let Some(active) = db::rack_maintenance_request::find_active_for_rack_for_update( + txn.as_mut(), + &finalize_rack_id, + ) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))? + else { + return Err(ComponentManagerError::Internal(format!( + "prepared rack maintenance request {request_id} disappeared" + ))); + }; + + if active.id != request_id { + return Ok((RackMaintenanceRequestOutcome::Busy, true)); + } + if active.status != RackMaintenanceRequestStatus::Preparing { + return Ok(( + RackMaintenanceRequestOutcome::AlreadyPending { + request_id: Some(request_id), + }, + false, + )); + } + + if rack.config.maintenance_requested.is_some() { + if !db::rack_maintenance_request::mark_cancelled( + txn.as_mut(), + request_id, + "another maintenance request was published first", + ) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))? + { + return Err(ComponentManagerError::Internal(format!( + "rack maintenance request {request_id} could not be cancelled" + ))); + } + return Ok((RackMaintenanceRequestOutcome::Busy, true)); + } + + let state = rack.controller_state.value.clone(); + let eligible = match eligibility { + RackMaintenanceEligibility::RequireReady => state == RackState::Ready, + RackMaintenanceEligibility::AllowError => { + matches!(&state, RackState::Ready | RackState::Error { .. }) + } + }; + if !eligible { + if !db::rack_maintenance_request::mark_cancelled( + txn.as_mut(), + request_id, + "rack became ineligible before request publication", + ) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))? + { + return Err(ComponentManagerError::Internal(format!( + "rack maintenance request {request_id} could not be cancelled" + ))); + } + return Ok((RackMaintenanceRequestOutcome::Deferred { state }, true)); + } + + if !db::rack_maintenance_request::mark_ready(txn.as_mut(), request_id) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))? + { + return Err(ComponentManagerError::Internal(format!( + "rack maintenance request {request_id} could not transition to ready" + ))); + } + + let reset_firmware_upgrade_job = + finalize_scope.should_run(&MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }); + let mut config = rack.config; + config.maintenance_requested = Some(finalize_scope); + config.maintenance_request_id = Some(request_id); + db::rack::update(txn.as_mut(), &finalize_rack_id, &config) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))?; + + if reset_firmware_upgrade_job { + db::rack::update_firmware_upgrade_job(txn.as_mut(), &finalize_rack_id, None) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))?; + } + + Ok(( + RackMaintenanceRequestOutcome::Scheduled { request_id }, + false, + )) + }) + }) + .await; + + let (outcome, cleanup_credential) = match finalization { + Ok(Ok(result)) => result, + Ok(Err(error)) => { + let failed = fail_preparing_rack_maintenance_request( + db_pool, + request_id, + "failed to publish rack maintenance request", + ) + .await; + if failed && let Some(credential_manager) = credential_manager { + cleanup_rack_maintenance_access_token(credential_manager, rack_id, request_id) + .await; + } + return Err(error); + } + Err(error) => { + let failed = fail_preparing_rack_maintenance_request( + db_pool, + request_id, + "failed to commit rack maintenance request", + ) + .await; + if failed && let Some(credential_manager) = credential_manager { + cleanup_rack_maintenance_access_token(credential_manager, rack_id, request_id) + .await; + } + return Err(ComponentManagerError::Internal(error.to_string())); + } + }; + + if cleanup_credential && let Some(credential_manager) = credential_manager { + cleanup_rack_maintenance_access_token(credential_manager, rack_id, request_id).await; + } + + Ok(outcome) +} + +async fn ensure_rack_maintenance_access_token( + credential_manager: &dyn CredentialManager, + rack_id: &RackId, + request_id: uuid::Uuid, + token: String, +) -> Result<(), String> { + let key = rack_maintenance_access_token_key(rack_id, Some(request_id)); + let credentials = Credentials::UsernamePassword { + username: "access_token".into(), + password: token, + }; + match credential_manager + .create_credentials(&key, &credentials) + .await + { + Ok(()) => Ok(()), + Err(create_error) => match credential_manager.get_credentials(&key).await { + Ok(Some(existing)) if existing == credentials => { + tracing::debug!( + rack_id = %rack_id, + %request_id, + "Adopting access token staged by an earlier attempt" + ); + Ok(()) + } + Ok(Some(_)) => Err(format!( + "{create_error}; request-scoped credential already contains a different token" + )), + Ok(None) => Err(create_error.to_string()), + Err(read_error) => Err(format!( + "{create_error}; failed to check for an existing staged token: {read_error}" + )), + }, + } +} + +async fn fail_preparing_rack_maintenance_request( + db_pool: &PgPool, + request_id: uuid::Uuid, + error_message: &'static str, +) -> bool { + let result = db_pool + .with_txn(|txn| { + Box::pin(async move { + db::rack_maintenance_request::mark_preparing_failed( + txn.as_mut(), + request_id, + error_message, + ) + .await + }) + }) + .await; + match result { + Ok(Ok(updated)) => updated, + Err(error) | Ok(Err(error)) => { + tracing::warn!( + %request_id, + error = %error, + "Failed to mark rack maintenance request preparation as failed" + ); + false + } + } +} + +async fn cleanup_rack_maintenance_access_token( + credential_manager: &dyn CredentialManager, + rack_id: &RackId, + request_id: uuid::Uuid, +) { + if let Err(error) = credential_manager + .delete_credentials(&rack_maintenance_access_token_key( + rack_id, + Some(request_id), + )) + .await + { + tracing::warn!( + rack_id = %rack_id, + %request_id, + error = %error, + "failed to delete request-scoped rack maintenance access token", + ); + } +} + impl ComponentManager { pub async fn configure_switch_certificate( &self, @@ -213,6 +649,13 @@ pub async fn build_component_manager( #[cfg(test)] mod tests { + use carbide_secrets::credentials::{CredentialKey, CredentialReader}; + use carbide_secrets::test_support::credentials::TestCredentialManager; + use carbide_uuid::rack::RackId; + use db::ObjectColumnFilter; + use model::rack::{ + FirmwareUpgradeJob, MaintenanceActivity, MaintenanceScope, RackConfig, RackState, + }; use model::rack_type::{ RackCapabilitiesSet, RackCapabilityCompute, RackCapabilityPowerShelf, RackCapabilitySwitch, RackHardwareTopology, RackProductFamily, RackProfile, @@ -221,6 +664,277 @@ mod tests { use super::*; use crate::config::ComponentManagerConfig; + async fn create_rack_in_state(pool: &PgPool, state: RackState) -> RackId { + let rack_id = RackId::new(uuid::Uuid::new_v4().to_string()); + let mut txn = pool.begin().await.unwrap(); + let rack = db::rack::create(txn.as_mut(), &rack_id, None, &RackConfig::default(), None) + .await + .unwrap(); + if state != RackState::Created { + assert!( + db::rack::try_update_controller_state( + txn.as_mut(), + &rack_id, + rack.controller_state.version, + rack.controller_state.version.increment(), + &state, + ) + .await + .unwrap() + ); + } + txn.commit().await.unwrap(); + rack_id + } + + async fn load_rack(pool: &PgPool, rack_id: &RackId) -> model::rack::Rack { + let mut conn = pool.acquire().await.unwrap(); + db::rack::find_by( + conn.as_mut(), + ObjectColumnFilter::One(db::rack::IdColumn, rack_id), + ) + .await + .unwrap() + .pop() + .unwrap() + } + + fn nmx_scope() -> MaintenanceScope { + MaintenanceScope { + activities: vec![MaintenanceActivity::ConfigureNmxCluster], + ..Default::default() + } + } + + #[carbide_macros::sqlx_test] + async fn rack_maintenance_scheduler_is_atomic_and_idempotent(pool: PgPool) { + let ready_rack = create_rack_in_state(&pool, RackState::Ready).await; + let scope = nmx_scope(); + + let request_id = match request_rack_maintenance_via_state_controller( + &pool, + &ready_rack, + scope.clone(), + RackMaintenanceEligibility::RequireReady, + None, + ) + .await + .unwrap() + { + RackMaintenanceRequestOutcome::Scheduled { request_id } => request_id, + outcome => panic!("expected Scheduled, got {outcome:?}"), + }; + let rack = load_rack(&pool, &ready_rack).await; + assert_eq!(rack.config.maintenance_requested, Some(scope.clone())); + assert_eq!(rack.config.maintenance_request_id, Some(request_id)); + let mut conn = pool.acquire().await.unwrap(); + let request = db::rack_maintenance_request::find_by_id(conn.as_mut(), request_id) + .await + .unwrap() + .unwrap(); + assert_eq!(request.status, RackMaintenanceRequestStatus::Ready); + drop(conn); + + // Exact retries are idempotent, including after the rack state has + // moved on. A different request is busy and cannot replace it. + let rack = load_rack(&pool, &ready_rack).await; + let mut txn = pool.begin().await.unwrap(); + assert!( + db::rack::try_update_controller_state( + txn.as_mut(), + &ready_rack, + rack.controller_state.version, + rack.controller_state.version.increment(), + &RackState::Discovering, + ) + .await + .unwrap() + ); + txn.commit().await.unwrap(); + assert_eq!( + request_rack_maintenance_via_state_controller( + &pool, + &ready_rack, + scope.clone(), + RackMaintenanceEligibility::RequireReady, + None, + ) + .await + .unwrap(), + RackMaintenanceRequestOutcome::AlreadyPending { + request_id: Some(request_id), + }, + ); + + let unrelated_scope = MaintenanceScope { + activities: vec![MaintenanceActivity::PowerSequence], + ..Default::default() + }; + assert_eq!( + request_rack_maintenance_via_state_controller( + &pool, + &ready_rack, + unrelated_scope, + RackMaintenanceEligibility::RequireReady, + None, + ) + .await + .unwrap(), + RackMaintenanceRequestOutcome::Busy, + ); + assert_eq!( + load_rack(&pool, &ready_rack) + .await + .config + .maintenance_requested, + Some(scope), + ); + + // Automatic callers defer non-Ready racks; operator callers may + // explicitly opt into recovering a rack from Error. + let error_state = RackState::Error { + cause: "test".into(), + }; + let error_rack = create_rack_in_state(&pool, error_state.clone()).await; + assert_eq!( + request_rack_maintenance_via_state_controller( + &pool, + &error_rack, + nmx_scope(), + RackMaintenanceEligibility::RequireReady, + None, + ) + .await + .unwrap(), + RackMaintenanceRequestOutcome::Deferred { state: error_state }, + ); + assert!(matches!( + request_rack_maintenance_via_state_controller( + &pool, + &error_rack, + nmx_scope(), + RackMaintenanceEligibility::AllowError, + None, + ) + .await + .unwrap(), + RackMaintenanceRequestOutcome::Scheduled { .. }, + )); + + // Two different requests racing for an empty slot serialize on the + // rack row: one wins and the other observes Busy. + let race_rack = create_rack_in_state(&pool, RackState::Ready).await; + let first_scope = MaintenanceScope { + activities: vec![MaintenanceActivity::FirmwareUpgrade { + firmware_version: Some(r#"{"Id":"first"}"#.into()), + components: vec![], + force_update: false, + }], + ..Default::default() + }; + let second_scope = MaintenanceScope { + activities: vec![MaintenanceActivity::FirmwareUpgrade { + firmware_version: Some(r#"{"Id":"second"}"#.into()), + components: vec![], + force_update: false, + }], + ..Default::default() + }; + let credential_manager = TestCredentialManager::default(); + let (first, second) = tokio::join!( + request_rack_maintenance_via_state_controller( + &pool, + &race_rack, + first_scope, + RackMaintenanceEligibility::RequireReady, + Some(RackMaintenanceAccessToken { + credential_manager: &credential_manager, + token: "first-token".into(), + }), + ), + request_rack_maintenance_via_state_controller( + &pool, + &race_rack, + second_scope, + RackMaintenanceEligibility::RequireReady, + Some(RackMaintenanceAccessToken { + credential_manager: &credential_manager, + token: "second-token".into(), + }), + ), + ); + let first = first.unwrap(); + let second = second.unwrap(); + let (winning_token, request_id) = match (&first, &second) { + ( + RackMaintenanceRequestOutcome::Scheduled { request_id }, + RackMaintenanceRequestOutcome::Busy, + ) => ("first-token", *request_id), + ( + RackMaintenanceRequestOutcome::Busy, + RackMaintenanceRequestOutcome::Scheduled { request_id }, + ) => ("second-token", *request_id), + unexpected => panic!("expected one Scheduled and one Busy outcome, got {unexpected:?}"), + }; + let stored_credentials = credential_manager + .get_credentials(&CredentialKey::RackMaintenanceRequestAccessToken { + rack_id: race_rack.clone(), + request_id, + }) + .await + .unwrap() + .expect("the winning request should persist its token"); + assert_eq!( + stored_credentials, + Credentials::UsernamePassword { + username: "access_token".into(), + password: winning_token.into(), + } + ); + + // Firmware bookkeeping is cleared in the same transaction that + // accepts a firmware maintenance request. + let firmware_rack = create_rack_in_state(&pool, RackState::Ready).await; + let mut txn = pool.begin().await.unwrap(); + db::rack::update_firmware_upgrade_job( + txn.as_mut(), + &firmware_rack, + Some(&FirmwareUpgradeJob { + job_id: Some("stale-job".into()), + ..Default::default() + }), + ) + .await + .unwrap(); + txn.commit().await.unwrap(); + let firmware_scope = MaintenanceScope { + activities: vec![MaintenanceActivity::FirmwareUpgrade { + firmware_version: Some("{}".into()), + components: vec![], + force_update: false, + }], + ..Default::default() + }; + assert!(matches!( + request_rack_maintenance_via_state_controller( + &pool, + &firmware_rack, + firmware_scope, + RackMaintenanceEligibility::RequireReady, + None, + ) + .await + .unwrap(), + RackMaintenanceRequestOutcome::Scheduled { .. }, + )); + assert!( + load_rack(&pool, &firmware_rack) + .await + .firmware_upgrade_job + .is_none() + ); + } + fn rms_rack_profiles(profile: RackProfile) -> RackProfileConfig { RackProfileConfig { rack_profiles: [("NVL72".to_string(), profile)].into_iter().collect(), diff --git a/crates/rack-controller/Cargo.toml b/crates/rack-controller/Cargo.toml index 2e6b5344f1..e4f1d01366 100644 --- a/crates/rack-controller/Cargo.toml +++ b/crates/rack-controller/Cargo.toml @@ -44,6 +44,7 @@ serde = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } tracing = { workspace = true } +uuid = { workspace = true } tonic = { workspace = true } [dev-dependencies] diff --git a/crates/rack-controller/src/error_state.rs b/crates/rack-controller/src/error_state.rs index 11c6267112..c5985fd752 100644 --- a/crates/rack-controller/src/error_state.rs +++ b/crates/rack-controller/src/error_state.rs @@ -62,7 +62,14 @@ pub async fn handle_error( activities_desc, ); } - let txn = ctx.services.db_pool.begin().await?; + let mut txn = ctx.services.db_pool.begin().await?; + if let Some(request_id) = config.maintenance_request_id + && !db::rack_maintenance_request::mark_running(txn.as_mut(), request_id).await? + { + return Err(StateHandlerError::GenericError(eyre::eyre!( + "rack maintenance request {request_id} is not ready to run" + ))); + } return Ok(StateHandlerOutcome::transition(RackState::Maintenance { maintenance_state: first_maintenance_state(scope), }) diff --git a/crates/rack-controller/src/maintenance.rs b/crates/rack-controller/src/maintenance.rs index a66c624307..f35c8b00d1 100644 --- a/crates/rack-controller/src/maintenance.rs +++ b/crates/rack-controller/src/maintenance.rs @@ -169,8 +169,10 @@ async fn transition_to_rack_error( ) -> Result, StateHandlerError> { let cause = cause.into(); tracing::warn!(rack_id = %rack_id, %cause, "Rack firmware upgrade failed before polling started"); - let outcome = StateHandlerOutcome::transition(RackState::Error { cause }); - clear_maintenance_requested_on_error(rack_id, state, outcome, ctx).await + let outcome = StateHandlerOutcome::transition(RackState::Error { + cause: cause.clone(), + }); + clear_maintenance_requested_on_error(rack_id, state, &cause, outcome, ctx).await } async fn transition_to_rack_error_with_firmware_job( @@ -192,9 +194,13 @@ async fn transition_to_rack_error_with_firmware_job( ..Default::default() }; state.firmware_upgrade_job = Some(job.clone()); + let maintenance_request_id = state.config.maintenance_request_id.take(); state.config.maintenance_requested = None; let mut txn = ctx.services.db_pool.begin().await?; + if let Some(request_id) = maintenance_request_id { + mark_maintenance_request_failed(txn.as_mut(), request_id, &cause).await?; + } db_rack::update_firmware_upgrade_job(txn.as_mut(), rack_id, Some(&job)).await?; db_rack::update(txn.as_mut(), rack_id, &state.config).await?; @@ -207,18 +213,49 @@ async fn transition_to_rack_error_with_firmware_job( async fn clear_maintenance_requested_on_error( rack_id: &RackId, state: &mut Rack, + cause: &str, outcome: StateHandlerOutcome, ctx: &mut StateHandlerContext<'_, RackStateHandlerContextObjects>, ) -> Result, StateHandlerError> { - if state.config.maintenance_requested.is_none() { + if state.config.maintenance_requested.is_none() && state.config.maintenance_request_id.is_none() + { return Ok(outcome); } + let maintenance_request_id = state.config.maintenance_request_id.take(); state.config.maintenance_requested = None; let mut txn = ctx.services.db_pool.begin().await?; + if let Some(request_id) = maintenance_request_id { + mark_maintenance_request_failed(txn.as_mut(), request_id, cause).await?; + } db_rack::update(txn.as_mut(), rack_id, &state.config).await?; Ok(outcome.with_txn(txn)) } +async fn mark_maintenance_request_failed( + txn: &mut sqlx::PgConnection, + request_id: uuid::Uuid, + cause: &str, +) -> Result<(), StateHandlerError> { + if !db::rack_maintenance_request::mark_failed(txn, request_id, cause).await? { + return Err(StateHandlerError::GenericError(eyre::eyre!( + "rack maintenance request {request_id} is not active" + ))); + } + Ok(()) +} + +async fn mark_maintenance_request_completed( + txn: &mut sqlx::PgConnection, + request_id: uuid::Uuid, +) -> Result<(), StateHandlerError> { + if !db::rack_maintenance_request::mark_completed(txn, request_id).await? { + return Err(StateHandlerError::GenericError(eyre::eyre!( + "rack maintenance request {request_id} is not running" + ))); + } + Ok(()) +} + fn nvos_update_requested(scope: &MaintenanceScope) -> bool { scope .activities @@ -265,8 +302,9 @@ fn requested_firmware_object_json_upgrade( async fn load_rack_maintenance_access_token( credential_manager: &dyn CredentialManager, rack_id: &RackId, + request_id: Option, ) -> Result { - let key = rack_maintenance_access_token_key(rack_id); + let key = rack_maintenance_access_token_key(rack_id, request_id); let credentials = credential_manager .get_credentials(&key) .await @@ -289,9 +327,10 @@ async fn load_rack_maintenance_access_token( async fn delete_rack_maintenance_access_token( credential_manager: &dyn CredentialManager, rack_id: &RackId, + request_id: Option, ) { if let Err(error) = credential_manager - .delete_credentials(&rack_maintenance_access_token_key(rack_id)) + .delete_credentials(&rack_maintenance_access_token_key(rack_id, request_id)) .await { tracing::warn!( @@ -1362,6 +1401,7 @@ pub async fn handle_maintenance( delete_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await; return transition_to_rack_error(id, state, "RMS client not configured", ctx) @@ -1370,6 +1410,7 @@ pub async fn handle_maintenance( let access_token = match load_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await { @@ -1404,6 +1445,7 @@ pub async fn handle_maintenance( delete_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await; } @@ -1421,6 +1463,7 @@ pub async fn handle_maintenance( delete_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await; return transition_to_rack_error( @@ -1462,6 +1505,7 @@ pub async fn handle_maintenance( delete_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await; } @@ -1518,6 +1562,7 @@ pub async fn handle_maintenance( delete_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await; } @@ -1537,6 +1582,7 @@ pub async fn handle_maintenance( delete_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await; } @@ -1639,23 +1685,30 @@ pub async fn handle_maintenance( if failed > 0 { let now = chrono::Utc::now(); + let cause = format!( + "firmware upgrade failed: {}/{} devices failed", + failed, total + ); job.status = Some("failed".into()); if job.completed_at.is_none() { job.completed_at = Some(now); } db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; state.firmware_upgrade_job = Some(job); - if state.config.maintenance_requested.is_some() { + if state.config.maintenance_requested.is_some() + || state.config.maintenance_request_id.is_some() + { + let maintenance_request_id = state.config.maintenance_request_id.take(); state.config.maintenance_requested = None; + if let Some(request_id) = maintenance_request_id { + mark_maintenance_request_failed(txn.as_mut(), request_id, &cause) + .await?; + } db_rack::update(txn.as_mut(), id, &state.config).await?; } - return Ok(StateHandlerOutcome::transition(RackState::Error { - cause: format!( - "firmware upgrade failed: {}/{} devices failed", - failed, total - ), - }) - .with_txn(txn)); + return Ok( + StateHandlerOutcome::transition(RackState::Error { cause }).with_txn(txn) + ); } let now = chrono::Utc::now(); @@ -1710,6 +1763,7 @@ pub async fn handle_maintenance( delete_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await; return transition_to_rack_error(id, state, "RMS client not configured", ctx) @@ -1718,6 +1772,7 @@ pub async fn handle_maintenance( let access_token = match load_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await { @@ -1749,6 +1804,7 @@ pub async fn handle_maintenance( delete_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await; let next = next_state_after_nvos(scope); @@ -1765,6 +1821,7 @@ pub async fn handle_maintenance( delete_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await; return transition_to_rack_error( @@ -1781,6 +1838,7 @@ pub async fn handle_maintenance( delete_rack_maintenance_access_token( ctx.services.credential_manager.as_ref(), id, + state.config.maintenance_request_id, ) .await; return transition_to_rack_error(id, state, error.to_string(), ctx).await; @@ -1830,8 +1888,12 @@ pub async fn handle_maintenance( switch_inventory.switches, ) .await; - delete_rack_maintenance_access_token(ctx.services.credential_manager.as_ref(), id) - .await; + delete_rack_maintenance_access_token( + ctx.services.credential_manager.as_ref(), + id, + state.config.maintenance_request_id, + ) + .await; let job = match submit_result { Ok(job) => job, @@ -1948,16 +2010,23 @@ pub async fn handle_maintenance( .count(); if failed > 0 { + let cause = format!("NVOS update failed: {}/{} switches failed", failed, total); db_rack::update_nvos_update_job(txn.as_mut(), id, Some(&job)).await?; state.nvos_update_job = Some(job); - if state.config.maintenance_requested.is_some() { + if state.config.maintenance_requested.is_some() + || state.config.maintenance_request_id.is_some() + { + let maintenance_request_id = state.config.maintenance_request_id.take(); state.config.maintenance_requested = None; + if let Some(request_id) = maintenance_request_id { + mark_maintenance_request_failed(txn.as_mut(), request_id, &cause) + .await?; + } db_rack::update(txn.as_mut(), id, &state.config).await?; } - return Ok(StateHandlerOutcome::transition(RackState::Error { - cause: format!("NVOS update failed: {}/{} switches failed", failed, total), - }) - .with_txn(txn)); + return Ok( + StateHandlerOutcome::transition(RackState::Error { cause }).with_txn(txn) + ); } if completed < total { @@ -2456,9 +2525,15 @@ pub async fn handle_maintenance( validating_state: RackValidationState::Pending, }); - if state.config.maintenance_requested.is_some() { + if state.config.maintenance_requested.is_some() + || state.config.maintenance_request_id.is_some() + { + let maintenance_request_id = state.config.maintenance_request_id.take(); state.config.maintenance_requested = None; let mut txn = ctx.services.db_pool.begin().await?; + if let Some(request_id) = maintenance_request_id { + mark_maintenance_request_completed(txn.as_mut(), request_id).await?; + } db_rack::update(txn.as_mut(), id, &state.config).await?; outcome = outcome.with_txn(txn); } diff --git a/crates/rack-controller/src/ready.rs b/crates/rack-controller/src/ready.rs index bb99015daa..684b28ecdc 100644 --- a/crates/rack-controller/src/ready.rs +++ b/crates/rack-controller/src/ready.rs @@ -62,7 +62,20 @@ pub async fn handle_ready( ); state.config.reprovision_requested = false; state.config.maintenance_requested = None; + let maintenance_request_id = state.config.maintenance_request_id.take(); let mut txn = ctx.services.db_pool.begin().await?; + if let Some(request_id) = maintenance_request_id + && !db::rack_maintenance_request::mark_cancelled( + txn.as_mut(), + request_id, + "rack reprovisioning superseded on-demand maintenance", + ) + .await? + { + return Err(StateHandlerError::GenericError(eyre::eyre!( + "rack maintenance request {request_id} could not be cancelled" + ))); + } db_rack::update(txn.as_mut(), id, &state.config).await?; return Ok(StateHandlerOutcome::transition(RackState::Maintenance { maintenance_state: RackMaintenanceState::FirmwareUpgrade { @@ -101,7 +114,14 @@ pub async fn handle_ready( } // Leave maintenance_requested set; the maintenance handler reads the // scope to decide which activities to run and clears it on Completed. - let txn = ctx.services.db_pool.begin().await?; + let mut txn = ctx.services.db_pool.begin().await?; + if let Some(request_id) = config.maintenance_request_id + && !db::rack_maintenance_request::mark_running(txn.as_mut(), request_id).await? + { + return Err(StateHandlerError::GenericError(eyre::eyre!( + "rack maintenance request {request_id} is not ready to run" + ))); + } return Ok(StateHandlerOutcome::transition(RackState::Maintenance { maintenance_state: first_maintenance_state(scope), }) diff --git a/crates/rack/Cargo.toml b/crates/rack/Cargo.toml index 95e460f8dd..0f2036c92c 100644 --- a/crates/rack/Cargo.toml +++ b/crates/rack/Cargo.toml @@ -46,10 +46,11 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sqlx = { workspace = true } thiserror = { workspace = true } -tracing = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } tonic = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } [dev-dependencies] carbide-test-support = { path = "../test-support" } diff --git a/crates/rack/src/firmware_object.rs b/crates/rack/src/firmware_object.rs index f24c0db611..88483e03b1 100644 --- a/crates/rack/src/firmware_object.rs +++ b/crates/rack/src/firmware_object.rs @@ -37,9 +37,18 @@ pub fn rms_access_token_or_noauth(access_token: Option<&str>) -> String { .to_string() } -pub fn rack_maintenance_access_token_key(rack_id: &RackId) -> CredentialKey { - CredentialKey::RackMaintenanceAccessToken { - rack_id: rack_id.clone(), +pub fn rack_maintenance_access_token_key( + rack_id: &RackId, + request_id: Option, +) -> CredentialKey { + match request_id { + Some(request_id) => CredentialKey::RackMaintenanceRequestAccessToken { + rack_id: rack_id.clone(), + request_id, + }, + None => CredentialKey::RackMaintenanceAccessToken { + rack_id: rack_id.clone(), + }, } } diff --git a/crates/secrets/Cargo.toml b/crates/secrets/Cargo.toml index b2348d9062..1e04c2c725 100644 --- a/crates/secrets/Cargo.toml +++ b/crates/secrets/Cargo.toml @@ -46,6 +46,7 @@ sha2 = { workspace = true } serde_json = { workspace = true } serde_yaml = { workspace = true } thiserror = { workspace = true } +uuid = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } vaultrs = { workspace = true } diff --git a/crates/secrets/src/credentials.rs b/crates/secrets/src/credentials.rs index f322955315..5f423fabae 100644 --- a/crates/secrets/src/credentials.rs +++ b/crates/secrets/src/credentials.rs @@ -428,6 +428,10 @@ pub enum CredentialKey { RackMaintenanceAccessToken { rack_id: RackId, }, + RackMaintenanceRequestAccessToken { + rack_id: RackId, + request_id: uuid::Uuid, + }, } /// The site-wide default credentials endpoint exploration requires before it @@ -572,7 +576,10 @@ impl CredentialKey { Self::MachineIdentityEncryptionKey { .. } => { CredentialPrefix::MachineIdentityEncryptionKey } - Self::RackMaintenanceAccessToken { .. } => CredentialPrefix::RackMaintenanceAccessToken, + Self::RackMaintenanceAccessToken { .. } + | Self::RackMaintenanceRequestAccessToken { .. } => { + CredentialPrefix::RackMaintenanceAccessToken + } } } @@ -684,6 +691,12 @@ impl CredentialKey { CredentialKey::RackMaintenanceAccessToken { rack_id } => { Cow::from(format!("racks/{rack_id}/maintenance/access-token")) } + CredentialKey::RackMaintenanceRequestAccessToken { + rack_id, + request_id, + } => Cow::from(format!( + "racks/{rack_id}/maintenance/requests/{request_id}/access-token" + )), } } } @@ -1185,7 +1198,20 @@ mod tests { Check { scenario: "rack maintenance access token", input: Row { - key: CredentialKey::RackMaintenanceAccessToken { rack_id }, + key: CredentialKey::RackMaintenanceAccessToken { + rack_id: rack_id.clone(), + }, + expected_prefix: "racks/", + }, + expect: PathChecks::all_hold(), + }, + Check { + scenario: "request-scoped rack maintenance access token", + input: Row { + key: CredentialKey::RackMaintenanceRequestAccessToken { + rack_id, + request_id: uuid::Uuid::nil(), + }, expected_prefix: "racks/", }, expect: PathChecks::all_hold(), @@ -1264,7 +1290,13 @@ mod tests { CredentialKey::MachineIdentityEncryptionKey { key_id: "k".to_string(), }, - CredentialKey::RackMaintenanceAccessToken { rack_id }, + CredentialKey::RackMaintenanceAccessToken { + rack_id: rack_id.clone(), + }, + CredentialKey::RackMaintenanceRequestAccessToken { + rack_id, + request_id: uuid::Uuid::nil(), + }, ]; for key in &keys {