Avoid machine validation result/heartbeat deadlocks

crates/api-core/src/handlers/machine_validation.rs

@@ -267,8 +267,15 @@ pub(crate) async fn persist_validation_result(
                 );
             }
         };
-    let machine_validation =
-        db::machine_validation::find_by_id(&mut txn, &validation_result.validation_id).await?;
+    // Acquire the parent-run lock before record_result() touches run-item rows.
+    // Heartbeats and stale-attempt reconciliation use the same parent-run ->
+    // run-item order. Successful results also serialize with the trigger that
+    // increments the parent run's completed count.
+    let machine_validation = db::machine_validation::lock_by_id_no_key_update(
+        &mut txn,
+        &validation_result.validation_id,
+    )
+    .await?;
     if !db::machine_validation::is_active(&machine_validation) {
         tracing::info!(
             validation_id = %validation_result.validation_id,

crates/api-core/src/machine_validation/mod.rs

@@ -422,6 +422,10 @@ async fn reconcile_stale_attempt(
         stale_attempt.attempt_id, stale_attempt.test_id, stale_attempt.validation_id
     );
 
+    // Keep the same parent-run -> run-item lock order used by result
+    // persistence and heartbeats.
+    db::machine_validation::lock_by_id_no_key_update(txn, &stale_attempt.validation_id).await?;
+
     let Some(validation_id) = db::machine_validation_execution::mark_attempt_stale_if_active(
         txn,
         &stale_attempt.attempt_id,

crates/api-db/src/machine_validation.rs

@@ -347,6 +347,36 @@ pub async fn find_by_id(
     )))
 }
 
+pub async fn lock_by_id_no_key_update(
+    txn: &mut PgConnection,
+    id: &MachineValidationId,
+) -> DatabaseResult<MachineValidation> {
+    // Raw SQL is required because FilterableQueryBuilder does not support row-lock clauses.
+    let query = "
+        SELECT
+            id,
+            machine_id,
+            name,
+            start_time,
+            end_time,
+            filter,
+            context,
+            state,
+            total,
+            completed,
+            duration_to_complete,
+            last_heartbeat_at
+        FROM machine_validation
+        WHERE id=$1
+        FOR NO KEY UPDATE";
+    sqlx::query_as::<_, MachineValidation>(query)
+        .bind(id)
+        .fetch_optional(txn)
+        .await
+        .map_err(|e| DatabaseError::query(query, e))?
+        .ok_or_else(|| DatabaseError::InvalidArgument(format!("Validation Id not found {id:?}")))
+}
+
 pub async fn find_all(txn: impl DbReader<'_>) -> DatabaseResult<Vec<MachineValidation>> {
     find_by(txn, ObjectColumnFilter::<IdColumn>::All).await
 }