From 62c96b6aae90eb33a801b7ea7cfdfa678bdba2ac Mon Sep 17 00:00:00 2001 From: Shahanur Islam Shagor Date: Sun, 6 Sep 2026 17:45:51 +0300 Subject: [PATCH 1/2] feat(openagent): harden restore recovery --- docs/OPENAGENT_ENGINEERING_PLAN.md | 2 +- .../008_openagent_restore_audit.sql | 17 ++ src-tauri/src/database.rs | 55 +++++ src-tauri/src/local_agent.rs | 218 +++++++++++++++--- src-tauri/src/openagent_runs.rs | 35 +++ src-tauri/src/portable_root.rs | 2 +- src/api.ts | 13 ++ 7 files changed, 311 insertions(+), 31 deletions(-) create mode 100644 src-tauri/migrations/008_openagent_restore_audit.sql diff --git a/docs/OPENAGENT_ENGINEERING_PLAN.md b/docs/OPENAGENT_ENGINEERING_PLAN.md index dda7fd1..cc074e6 100644 --- a/docs/OPENAGENT_ENGINEERING_PLAN.md +++ b/docs/OPENAGENT_ENGINEERING_PLAN.md @@ -61,4 +61,4 @@ The initial tool set covers directory listing, bounded file reads, text search, Stage 1 is complete. The bounded agent loop is connected to installed Nemotron 3.5 Lightning packages, uses the selected model ID, retains compatible lower-memory fallbacks, and runs inside the attached-root file boundary. -Stage 2 is in progress. OpenAgent now persists run state, ordered tool steps, bounded results, validation state, and before/after checkpoints. File-tool checkpoints include bounded file contents plus SHA-256 evidence and block destructive mutations that exceed the entry or byte limits. Completed mutations can be restored only after the matching after-state passes conflict, digest, symlink, and attached-root checks; active runs cannot be restored. Startup recovery marks abandoned runs as interrupted, and typed desktop APIs expose run history, checkpoint IDs, step details, and restore results. Automatic interrupted-run replay, terminal-command snapshots, token/runtime metrics, and the visible timeline UI are still pending and must not be represented as complete. The OS-level process sandbox remains Stage 3. +Stage 2 is in progress. OpenAgent now persists run state, ordered tool steps, bounded results, validation state, before/after checkpoints, and restore audit events. File-tool checkpoints include bounded file contents plus SHA-256 evidence and block destructive mutations that exceed the entry or byte limits. Completed mutations can be restored only after the matching after-state passes conflict, digest, symlink, and attached-root checks; active runs cannot be restored. Restore payloads are verified before mutation, failed restores attempt a compensating rollback to the captured after-state, and restored runs return to a validation-required state. Startup recovery marks abandoned runs as interrupted, blocks their active steps, and flags unfinished restore events for operator verification. Typed desktop APIs expose run history, checkpoint IDs, step details, restore results, and restore-event status. When terminal access is disabled, a mutated run reports that required validation was not run instead of presenting an unqualified success. True filesystem transactions, handle-based protection against every symlink race, automatic interrupted-run replay, terminal-command snapshots, token/runtime metrics, and the visible timeline UI are still pending and must not be represented as complete. The OS-level process sandbox remains Stage 3. diff --git a/src-tauri/migrations/008_openagent_restore_audit.sql b/src-tauri/migrations/008_openagent_restore_audit.sql new file mode 100644 index 0000000..182e45d --- /dev/null +++ b/src-tauri/migrations/008_openagent_restore_audit.sql @@ -0,0 +1,17 @@ +CREATE TABLE openagent_restore_events ( + id TEXT PRIMARY KEY, + checkpoint_id TEXT NOT NULL, + run_id TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('running', 'completed', 'rolled_back', 'rollback_failed')), + restored_files INTEGER NOT NULL DEFAULT 0, + restored_directories INTEGER NOT NULL DEFAULT 0, + removed_paths INTEGER NOT NULL DEFAULT 0, + error TEXT, + started_at TEXT NOT NULL, + completed_at TEXT, + FOREIGN KEY(checkpoint_id) REFERENCES openagent_checkpoints(id) ON DELETE CASCADE, + FOREIGN KEY(run_id) REFERENCES openagent_runs(id) ON DELETE CASCADE +); + +CREATE INDEX idx_openagent_restore_events_run_started + ON openagent_restore_events(run_id, started_at DESC); diff --git a/src-tauri/src/database.rs b/src-tauri/src/database.rs index e58f01e..eb3dab2 100644 --- a/src-tauri/src/database.rs +++ b/src-tauri/src/database.rs @@ -61,6 +61,11 @@ const MIGRATIONS: &[Migration] = &[ name: "007_openagent_durable_runs", sql: include_str!("../migrations/007_openagent_durable_runs.sql"), }, + Migration { + number: 8, + name: "008_openagent_restore_audit", + sql: include_str!("../migrations/008_openagent_restore_audit.sql"), + }, ]; pub struct Database { @@ -82,6 +87,8 @@ impl Database { database.migrate()?; database.recover_interrupted_chat_messages()?; database.recover_interrupted_openagent_runs()?; + database.recover_interrupted_openagent_steps()?; + database.recover_interrupted_restore_events()?; database.ensure_local_profile()?; Ok(database) } @@ -183,6 +190,34 @@ impl Database { .map_err(AppError::from) } + fn recover_interrupted_openagent_steps(&self) -> Result { + let now = Utc::now().to_rfc3339(); + self.connection + .execute( + "UPDATE openagent_steps + SET status = 'blocked', error = 'Application exited while this step was active', + completed_at = ?1 + WHERE status = 'running' + AND run_id IN (SELECT id FROM openagent_runs WHERE status = 'interrupted')", + params![now], + ) + .map_err(AppError::from) + } + + fn recover_interrupted_restore_events(&self) -> Result { + let now = Utc::now().to_rfc3339(); + self.connection + .execute( + "UPDATE openagent_restore_events + SET status = 'rollback_failed', + error = 'Application exited during restore; workspace verification required', + completed_at = ?1 + WHERE status = 'running'", + params![now], + ) + .map_err(AppError::from) + } + /// Records the schema version this build understands. Kept independent /// from `app_version` (Cargo package version) — application releases and /// database schema revisions evolve on different timelines. @@ -406,6 +441,7 @@ mod tests { let message_id = Uuid::new_v4().to_string(); let model_id = Uuid::new_v4().to_string(); let run_id = Uuid::new_v4().to_string(); + let step_id = Uuid::new_v4().to_string(); let now = Utc::now().to_rfc3339(); { @@ -447,6 +483,12 @@ mod tests { VALUES (?1, ?2, ?3, ?4, ?5, 'test', 'running', 28, ?6, ?6)", params![run_id, conversation_id, project_id, message_id, model_id, now], ).unwrap(); + connection.execute( + "INSERT INTO openagent_steps + (id, run_id, step_index, tool, action_json, status, started_at) + VALUES (?1, ?2, 1, 'write_file', '{}', 'running', ?3)", + params![step_id, run_id, now], + ).unwrap(); } let database = Database::open(path).unwrap(); @@ -460,6 +502,19 @@ mod tests { .unwrap(); assert_eq!(status, "interrupted"); assert!(completed_at.is_some()); + + let (step_status, step_error, step_completed_at): (String, Option, Option) = + database + .connection() + .query_row( + "SELECT status, error, completed_at FROM openagent_steps WHERE id = ?1", + params![step_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(step_status, "blocked"); + assert!(step_error.unwrap().contains("exited")); + assert!(step_completed_at.is_some()); } #[test] diff --git a/src-tauri/src/local_agent.rs b/src-tauri/src/local_agent.rs index fa46a66..5529592 100644 --- a/src-tauri/src/local_agent.rs +++ b/src-tauri/src/local_agent.rs @@ -6,12 +6,14 @@ use std::{ }; use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; +use chrono::Utc; use rusqlite::{params, OptionalExtension}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; use tauri::{AppHandle, Emitter, State}; use tokio::process::Command; +use uuid::Uuid; use crate::{ app_error::AppError, @@ -201,20 +203,28 @@ pub fn restore_openagent_checkpoint( checkpoint_id: String, state: State, ) -> Result { - let (project_id, run_status, before_json, after_json) = { + let (project_id, run_id, run_status, before_json, after_json) = { let db = state .database .lock() .map_err(|_| AppError::internal("database lock poisoned"))?; - let before: (String, String, String, String) = db + let before: (String, String, String, String, String) = db .connection() .query_row( - "SELECT r.project_id, r.status, c.step_id, c.workspace_snapshot_json + "SELECT r.project_id, r.id, r.status, c.step_id, c.workspace_snapshot_json FROM openagent_checkpoints c JOIN openagent_runs r ON r.id = c.run_id WHERE c.id = ?1 AND c.kind = 'before_mutation'", params![checkpoint_id], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, ) .optional()? .ok_or_else(|| AppError::internal("restorable OpenAgent checkpoint not found"))?; @@ -224,12 +234,12 @@ pub fn restore_openagent_checkpoint( "SELECT workspace_snapshot_json FROM openagent_checkpoints WHERE step_id = ?1 AND kind = 'after_mutation' ORDER BY created_at DESC LIMIT 1", - params![before.2], + params![before.3], |row| row.get(0), ) .optional()? .ok_or_else(|| AppError::internal("checkpoint has no completed mutation snapshot"))?; - (before.0, before.1, before.3, after_json) + (before.0, before.1, before.2, before.4, after_json) }; if run_status == "running" { return Err(AppError::internal( @@ -251,8 +261,53 @@ pub fn restore_openagent_checkpoint( load_workspace_config(&db, &project_id)? }; preflight_checkpoint_restore(&workspace, &after.entries)?; - let result = apply_checkpoint_restore(&checkpoint_id, &workspace, &before.entries)?; - Ok(result) + validate_checkpoint_payloads(&before.entries)?; + validate_checkpoint_payloads(&after.entries)?; + let event_id = start_restore_event(&state, &checkpoint_id, &run_id)?; + match apply_checkpoint_restore(&checkpoint_id, &workspace, &before.entries) { + Ok(result) => { + finish_restore_event(&state, &event_id, "completed", &result, None)?; + mark_run_unvalidated_after_restore(&state, &run_id)?; + Ok(result) + } + Err(restore_error) => { + let rollback = apply_checkpoint_restore(&checkpoint_id, &workspace, &after.entries); + let empty = CheckpointRestoreResult { + checkpoint_id: checkpoint_id.clone(), + restored_files: 0, + restored_directories: 0, + removed_paths: 0, + validation_required: true, + }; + match rollback { + Ok(_) => { + finish_restore_event( + &state, + &event_id, + "rolled_back", + &empty, + Some(&restore_error.to_string()), + )?; + Err(AppError::internal(format!( + "checkpoint restore failed and was rolled back: {restore_error}" + ))) + } + Err(rollback_error) => { + let message = format!( + "restore failed: {restore_error}; rollback also failed: {rollback_error}" + ); + finish_restore_event( + &state, + &event_id, + "rollback_failed", + &empty, + Some(&message), + )?; + Err(AppError::internal(message)) + } + } + } + } } async fn run_agent_message( @@ -468,6 +523,10 @@ async fn run_agent_message( one_line(reason, 320) )); } + } else if validation_required && !agent_context.workspace.full_pc_access { + completion.push_str( + "\n\nValidation: required but not run because Full PC + Terminal access is disabled. Workspace changes are unverified.", + ); } emit_agent_chunk( app, @@ -1001,6 +1060,27 @@ fn preflight_checkpoint_restore( Ok(()) } +fn validate_checkpoint_payloads(entries: &[CheckpointEntry]) -> Result<(), AppError> { + for entry in entries { + if entry.kind != "file" { + continue; + } + let encoded = entry.content_base64.as_deref().ok_or_else(|| { + AppError::internal("checkpoint file is missing its content payload") + })?; + let content = BASE64 + .decode(encoded) + .map_err(|_| AppError::internal("checkpoint file payload is invalid"))?; + let digest = format!("{:x}", Sha256::digest(&content)); + if entry.sha256.as_deref() != Some(digest.as_str()) { + return Err(AppError::internal( + "checkpoint file digest verification failed", + )); + } + } + Ok(()) +} + fn apply_checkpoint_restore( checkpoint_id: &str, config: &AgentWorkspaceConfig, @@ -1010,33 +1090,19 @@ fn apply_checkpoint_restore( .iter() .map(|entry| Ok((checkpoint_path(config, &entry.path)?, entry))) .collect::, AppError>>()?; - for (_, entry) in &paths { - if entry.kind == "file" { - let encoded = entry.content_base64.as_deref().ok_or_else(|| { - AppError::internal("checkpoint file is missing its content payload") - })?; - let content = BASE64 - .decode(encoded) - .map_err(|_| AppError::internal("checkpoint file payload is invalid"))?; - let digest = format!("{:x}", Sha256::digest(&content)); - if entry.sha256.as_deref() != Some(digest.as_str()) { - return Err(AppError::internal( - "checkpoint file digest verification failed", - )); - } - } - } + validate_checkpoint_payloads(entries)?; paths.sort_by_key(|(path, _)| std::cmp::Reverse(path.components().count())); let mut removed_paths = 0; - for (path, entry) in &paths { + for (_, entry) in &paths { + let path = checkpoint_path(config, &entry.path)?; if entry.kind == "missing" && path.exists() { - if fs::symlink_metadata(path)?.file_type().is_symlink() { + if fs::symlink_metadata(&path)?.file_type().is_symlink() { return Err(AppError::internal("refusing to restore over a symlink")); } if path.is_dir() { - fs::remove_dir_all(path)?; + fs::remove_dir_all(&path)?; } else { - fs::remove_file(path)?; + fs::remove_file(&path)?; } removed_paths += 1; } @@ -1044,7 +1110,8 @@ fn apply_checkpoint_restore( paths.sort_by_key(|(path, _)| path.components().count()); let mut restored_directories = 0; let mut restored_files = 0; - for (path, entry) in paths { + for (_, entry) in paths { + let path = checkpoint_path(config, &entry.path)?; match entry.kind.as_str() { "directory" => { fs::create_dir_all(&path)?; @@ -1080,6 +1147,71 @@ fn apply_checkpoint_restore( }) } +fn start_restore_event( + state: &State<'_, AppState>, + checkpoint_id: &str, + run_id: &str, +) -> Result { + let id = Uuid::new_v4().to_string(); + let db = state + .database + .lock() + .map_err(|_| AppError::internal("database lock poisoned"))?; + db.connection().execute( + "INSERT INTO openagent_restore_events + (id, checkpoint_id, run_id, status, started_at) + VALUES (?1, ?2, ?3, 'running', ?4)", + params![id, checkpoint_id, run_id, Utc::now().to_rfc3339()], + )?; + Ok(id) +} + +fn finish_restore_event( + state: &State<'_, AppState>, + event_id: &str, + status: &str, + result: &CheckpointRestoreResult, + error: Option<&str>, +) -> Result<(), AppError> { + let db = state + .database + .lock() + .map_err(|_| AppError::internal("database lock poisoned"))?; + db.connection().execute( + "UPDATE openagent_restore_events + SET status = ?1, restored_files = ?2, restored_directories = ?3, + removed_paths = ?4, error = ?5, completed_at = ?6 + WHERE id = ?7", + params![ + status, + result.restored_files as i64, + result.restored_directories as i64, + result.removed_paths as i64, + error, + Utc::now().to_rfc3339(), + event_id + ], + )?; + Ok(()) +} + +fn mark_run_unvalidated_after_restore( + state: &State<'_, AppState>, + run_id: &str, +) -> Result<(), AppError> { + let db = state + .database + .lock() + .map_err(|_| AppError::internal("database lock poisoned"))?; + db.connection().execute( + "UPDATE openagent_runs + SET validation_status = 'required', validation_command = NULL, updated_at = ?1 + WHERE id = ?2", + params![Utc::now().to_rfc3339(), run_id], + )?; + Ok(()) +} + fn finish_durable_run( state: &State<'_, AppState>, run_id: &str, @@ -2661,6 +2793,34 @@ mod tests { assert_eq!(fs::read(&file).unwrap(), b"user edit"); } + #[test] + fn checkpoint_restore_rejects_corrupt_payload_before_mutation() { + let temp = tempfile::tempdir().unwrap(); + let file = temp.path().join("src.txt"); + fs::write(&file, b"current").unwrap(); + let config = AgentWorkspaceConfig { + full_pc_access: false, + roots: vec![AgentWorkspaceRoot { + id: "root".to_string(), + path: temp.path().display().to_string(), + created_at: "now".to_string(), + }], + }; + let mut entries = decoded_checkpoint_entries( + capture_checkpoint_entries( + &config, + "write_file", + &json!({"rootId": "root", "path": "src.txt", "content": "next"}), + ) + .unwrap(), + ); + entries[0].content_base64 = Some(BASE64.encode(b"tampered")); + + let error = apply_checkpoint_restore("checkpoint", &config, &entries).unwrap_err(); + assert!(error.to_string().contains("digest verification failed")); + assert_eq!(fs::read(&file).unwrap(), b"current"); + } + #[test] fn recognizes_common_validation_commands() { assert!(is_validation_command("npm run lint")); diff --git a/src-tauri/src/openagent_runs.rs b/src-tauri/src/openagent_runs.rs index 49367c4..576cf0a 100644 --- a/src-tauri/src/openagent_runs.rs +++ b/src-tauri/src/openagent_runs.rs @@ -50,6 +50,21 @@ pub struct OpenAgentRunDetails { pub run: OpenAgentRun, pub steps: Vec, pub checkpoints: Vec, + pub restore_events: Vec, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OpenAgentRestoreEvent { + pub id: String, + pub checkpoint_id: String, + pub status: String, + pub restored_files: i64, + pub restored_directories: i64, + pub removed_paths: i64, + pub error: Option, + pub started_at: String, + pub completed_at: Option, } #[derive(Debug, Clone, Serialize)] @@ -241,10 +256,30 @@ impl<'a> OpenAgentRunRepository<'a> { }) })?; let checkpoints = rows.collect::, _>>()?; + let mut statement = self.database.connection().prepare( + "SELECT id, checkpoint_id, status, restored_files, restored_directories, + removed_paths, error, started_at, completed_at + FROM openagent_restore_events WHERE run_id = ?1 ORDER BY started_at ASC", + )?; + let rows = statement.query_map(params![run_id], |row| { + Ok(OpenAgentRestoreEvent { + id: row.get(0)?, + checkpoint_id: row.get(1)?, + status: row.get(2)?, + restored_files: row.get(3)?, + restored_directories: row.get(4)?, + removed_paths: row.get(5)?, + error: row.get(6)?, + started_at: row.get(7)?, + completed_at: row.get(8)?, + }) + })?; + let restore_events = rows.collect::, _>>()?; Ok(Some(OpenAgentRunDetails { run, steps, checkpoints, + restore_events, })) } } diff --git a/src-tauri/src/portable_root.rs b/src-tauri/src/portable_root.rs index 6bfae61..c0a70b2 100644 --- a/src-tauri/src/portable_root.rs +++ b/src-tauri/src/portable_root.rs @@ -9,7 +9,7 @@ use crate::app_error::AppError; const MARKER_FILE: &str = "openmindai.marker"; const INSTALL_CONFIG_FILE: &str = "install.json"; -pub const CURRENT_SCHEMA_VERSION: u32 = 7; +pub const CURRENT_SCHEMA_VERSION: u32 = 8; pub const DEFAULT_PROFILE_NAME: &str = "openmindai"; /// Rough initial footprint for a first-run install: llama.cpp runtime + /// Qwen3 4B Q4_K_M model + database + safety margin. Used only to preview diff --git a/src/api.ts b/src/api.ts index 7f24831..606bd8f 100644 --- a/src/api.ts +++ b/src/api.ts @@ -73,6 +73,19 @@ export type OpenAgentRunDetails = { run: OpenAgentRun; steps: OpenAgentStep[]; checkpoints: OpenAgentCheckpoint[]; + restoreEvents: OpenAgentRestoreEvent[]; +}; + +export type OpenAgentRestoreEvent = { + id: string; + checkpointId: string; + status: "running" | "completed" | "rolled_back" | "rollback_failed"; + restoredFiles: number; + restoredDirectories: number; + removedPaths: number; + error: string | null; + startedAt: string; + completedAt: string | null; }; export type OpenAgentCheckpoint = { From 288cd8c302ffe5d503204644128d0d813724367a Mon Sep 17 00:00:00 2001 From: Shahanur Islam Shagor Date: Sun, 6 Sep 2026 17:48:19 +0300 Subject: [PATCH 2/2] style(openagent): apply Rust formatting --- src-tauri/src/database.rs | 10 ++++++---- src-tauri/src/local_agent.rs | 7 ++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src-tauri/src/database.rs b/src-tauri/src/database.rs index eb3dab2..320c1c4 100644 --- a/src-tauri/src/database.rs +++ b/src-tauri/src/database.rs @@ -483,12 +483,14 @@ mod tests { VALUES (?1, ?2, ?3, ?4, ?5, 'test', 'running', 28, ?6, ?6)", params![run_id, conversation_id, project_id, message_id, model_id, now], ).unwrap(); - connection.execute( - "INSERT INTO openagent_steps + connection + .execute( + "INSERT INTO openagent_steps (id, run_id, step_index, tool, action_json, status, started_at) VALUES (?1, ?2, 1, 'write_file', '{}', 'running', ?3)", - params![step_id, run_id, now], - ).unwrap(); + params![step_id, run_id, now], + ) + .unwrap(); } let database = Database::open(path).unwrap(); diff --git a/src-tauri/src/local_agent.rs b/src-tauri/src/local_agent.rs index 5529592..622da04 100644 --- a/src-tauri/src/local_agent.rs +++ b/src-tauri/src/local_agent.rs @@ -1065,9 +1065,10 @@ fn validate_checkpoint_payloads(entries: &[CheckpointEntry]) -> Result<(), AppEr if entry.kind != "file" { continue; } - let encoded = entry.content_base64.as_deref().ok_or_else(|| { - AppError::internal("checkpoint file is missing its content payload") - })?; + let encoded = entry + .content_base64 + .as_deref() + .ok_or_else(|| AppError::internal("checkpoint file is missing its content payload"))?; let content = BASE64 .decode(encoded) .map_err(|_| AppError::internal("checkpoint file payload is invalid"))?;