diff --git a/clients/openframe-client/Cargo.lock b/clients/openframe-client/Cargo.lock index 5f47e5487..c78a1dee5 100644 --- a/clients/openframe-client/Cargo.lock +++ b/clients/openframe-client/Cargo.lock @@ -1046,13 +1046,13 @@ dependencies = [ [[package]] name = "hostname" -version = "0.3.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" dependencies = [ + "cfg-if", "libc", - "match_cfg", - "winapi", + "windows-link 0.2.1", ] [[package]] @@ -1401,12 +1401,6 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - [[package]] name = "matchers" version = "0.2.0" @@ -1602,6 +1596,7 @@ dependencies = [ "log", "nix", "plist", + "redb", "regex", "reqwest", "rustls-pemfile 1.0.4", @@ -2014,6 +2009,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2392,14 +2396,15 @@ dependencies = [ [[package]] name = "service-manager" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cae942acfe9cecd4450998408f52e1c1ee083145226b7b803bd0d82e1c86912" +checksum = "b9ff6975a4ea07dda326cf122fcc1b5cf6266daad42d1442a666a99fe50f0de9" dependencies = [ "cfg-if", "dirs", "encoding-utils", "encoding_rs", + "log", "plist", "which", "xml-rs", @@ -3513,13 +3518,13 @@ dependencies = [ [[package]] name = "windows-service" -version = "0.6.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd9db37ecb5b13762d95468a2fc6009d4b2c62801243223aabd44fca13ad13c8" +checksum = "857224b3b211c6f3616921f081ee54721ee3ad2ace2fac6a6337e032f7b4dcf2" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.4", "widestring", - "windows-sys 0.45.0", + "windows-sys 0.61.2", ] [[package]] @@ -3540,15 +3545,6 @@ dependencies = [ "windows-link 0.2.1", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -3594,21 +3590,6 @@ dependencies = [ "windows-link 0.2.1", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -3657,12 +3638,6 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3681,12 +3656,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3705,12 +3674,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3741,12 +3704,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3765,12 +3722,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3789,12 +3740,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3813,12 +3758,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/clients/openframe-client/Cargo.toml b/clients/openframe-client/Cargo.toml index 390136f19..465949b60 100644 --- a/clients/openframe-client/Cargo.toml +++ b/clients/openframe-client/Cargo.toml @@ -110,20 +110,21 @@ tracing-appender = "0.2" flate2 = "1.0" indexmap = "2.0" tar = "0.4" -service-manager = "0.8" +service-manager = "0.11" plist = "1.7.1" # System information collection dependencies -hostname = "0.3" +hostname = "0.4" bytes = "1.10.1" async-nats = { git = "https://github.com/flamingo-stack/nats.rs.git", rev = "3cfbf0f393a964be1255b382065079a82f316069", features = ["websockets"] } rustls-pemfile = "1.0" +redb = "2" regex = "1.11.1" [target.'cfg(windows)'.dependencies] winreg = "0.52" winapi = { version = "0.3", features = ["winuser", "shellapi", "securitybaseapi", "errhandlingapi", "winerror", "wincon", "consoleapi", "processenv", "winbase"] } is_elevated = "0.1" -windows-service = "0.6" +windows-service = "0.8" windows = { version = "0.52", features = ["Win32_Foundation", "Win32_System_Threading", "Win32_System_RemoteDesktop", "Win32_System_JobObjects", "Win32_UI_WindowsAndMessaging", "Win32_System_RestartManager", "Win32_Security", "Win32_Security_Authorization", "Win32_System_Registry", "Win32_System_Environment"] } [target.'cfg(unix)'.dependencies] diff --git a/clients/openframe-client/src/clients/auth_client.rs b/clients/openframe-client/src/clients/auth_client.rs index 9df83780e..1c31b528f 100644 --- a/clients/openframe-client/src/clients/auth_client.rs +++ b/clients/openframe-client/src/clients/auth_client.rs @@ -4,20 +4,28 @@ use reqwest::{ Client, }; use std::collections::HashMap; +use std::sync::Arc; use crate::models::AgentTokenResponse; +use crate::services::deactivation_service::DeactivationService; #[derive(Clone)] pub struct AuthClient { http_client: Client, base_url: String, + deactivation: Arc, } impl AuthClient { - pub fn new(base_url: String, http_client: Client) -> Self { + pub fn new( + base_url: String, + http_client: Client, + deactivation: Arc, + ) -> Self { Self { http_client, base_url, + deactivation, } } @@ -49,6 +57,7 @@ impl AuthClient { .context("Failed to send token request")?; let status = response.status(); + self.deactivation.on_gateway_status(status).await; if !status.is_success() { return Err(anyhow::anyhow!( @@ -92,6 +101,7 @@ impl AuthClient { .context("Failed to send refresh token request")?; let status = response.status(); + self.deactivation.on_gateway_status(status).await; if !status.is_success() { return Err(anyhow::anyhow!( diff --git a/clients/openframe-client/src/clients/registration_client.rs b/clients/openframe-client/src/clients/registration_client.rs index 7a98a0e55..a6a2413cb 100644 --- a/clients/openframe-client/src/clients/registration_client.rs +++ b/clients/openframe-client/src/clients/registration_client.rs @@ -118,38 +118,5 @@ fn is_client_secret_error(status: StatusCode, body: &str) -> bool { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn detects_client_secret_invalid() { - let body = r#"{"code":"CLIENT_SECRET_INVALID","message":"Invalid client secret"}"#; - assert!(is_client_secret_error(StatusCode::UNAUTHORIZED, body)); - } - - #[test] - fn detects_client_secret_empty() { - let body = r#"{"code":"CLIENT_SECRET_EMPTY","message":"Client secret is empty"}"#; - assert!(is_client_secret_error(StatusCode::UNAUTHORIZED, body)); - } - - #[test] - fn ignores_other_401_error_codes() { - let body = r#"{"code":"INITIAL_KEY_INVALID","message":"..."}"#; - assert!(!is_client_secret_error(StatusCode::UNAUTHORIZED, body)); - } - - #[test] - fn ignores_client_secret_error_on_non_401() { - let body = r#"{"code":"CLIENT_SECRET_INVALID"}"#; - assert!(!is_client_secret_error(StatusCode::BAD_REQUEST, body)); - } - - #[test] - fn handles_non_json_body() { - assert!(!is_client_secret_error( - StatusCode::UNAUTHORIZED, - "gateway timeout" - )); - } -} +#[path = "registration_client_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/clients/registration_client_tests.rs b/clients/openframe-client/src/clients/registration_client_tests.rs new file mode 100644 index 000000000..ab6706817 --- /dev/null +++ b/clients/openframe-client/src/clients/registration_client_tests.rs @@ -0,0 +1,33 @@ +use super::*; + +#[test] +fn detects_client_secret_invalid() { + let body = r#"{"code":"CLIENT_SECRET_INVALID","message":"Invalid client secret"}"#; + assert!(is_client_secret_error(StatusCode::UNAUTHORIZED, body)); +} + +#[test] +fn detects_client_secret_empty() { + let body = r#"{"code":"CLIENT_SECRET_EMPTY","message":"Client secret is empty"}"#; + assert!(is_client_secret_error(StatusCode::UNAUTHORIZED, body)); +} + +#[test] +fn ignores_other_401_error_codes() { + let body = r#"{"code":"INITIAL_KEY_INVALID","message":"..."}"#; + assert!(!is_client_secret_error(StatusCode::UNAUTHORIZED, body)); +} + +#[test] +fn ignores_client_secret_error_on_non_401() { + let body = r#"{"code":"CLIENT_SECRET_INVALID"}"#; + assert!(!is_client_secret_error(StatusCode::BAD_REQUEST, body)); +} + +#[test] +fn handles_non_json_body() { + assert!(!is_client_secret_error( + StatusCode::UNAUTHORIZED, + "gateway timeout" + )); +} diff --git a/clients/openframe-client/src/config/mod.rs b/clients/openframe-client/src/config/mod.rs index 8a803b44a..db6bedfec 100644 --- a/clients/openframe-client/src/config/mod.rs +++ b/clients/openframe-client/src/config/mod.rs @@ -17,7 +17,7 @@ pub mod service_stop { pub const FORCE_KILL_TIMEOUT_SECS: u64 = 3; /// Force-kill attempts per process. pub const MAX_KILL_RETRIES: u32 = 3; - /// Polls awaiting a Windows service to report STOPPED (~10s). + /// Polls awaiting a Windows service state change (~10s; up to ~210s if SCM queries keep timing out non-consecutively). pub const SERVICE_STOP_MAX_ATTEMPTS: u32 = 20; /// Force-kill rounds for a stuck service (margin over Windows' 3 auto-restarts). pub const SERVICE_FORCE_KILL_MAX_ATTEMPTS: u32 = 6; @@ -25,6 +25,16 @@ pub mod service_stop { pub const SERVICE_STOP_CALL_TIMEOUT_SECS: u64 = 10; /// Start attempts for a service before giving up. pub const SERVICE_START_MAX_ATTEMPTS: u32 = 3; + /// Cap on any blocking SCM query (status/config/delete) so a wedged SCM can't hang a task. + pub const SCM_QUERY_TIMEOUT_SECS: u64 = 10; + /// Cap on a blocking SCM `start()` call (StartService can stall behind a busy service). + pub const SERVICE_START_CALL_TIMEOUT_SECS: u64 = 30; + /// Consecutive SCM query failures before a wait loop stops polling and escalates. + pub const SCM_QUERY_MAX_CONSECUTIVE_FAILURES: u32 = 2; + /// Overall cap on a guarded tool restart; guarantees the updating flag and tool lock release. + pub const TOOL_RESTART_TIMEOUT_SECS: u64 = 600; + /// In-flight SCM call permits; bounds blocking threads a wedged SCM can park. + pub const SCM_MAX_IN_FLIGHT: usize = 4; } // Legacy config structs, currently unused (candidates for removal). diff --git a/clients/openframe-client/src/config/update_config.rs b/clients/openframe-client/src/config/update_config.rs index d3902a648..dec1653e1 100644 --- a/clients/openframe-client/src/config/update_config.rs +++ b/clients/openframe-client/src/config/update_config.rs @@ -16,8 +16,11 @@ pub const CONSUMER_CYCLE_PAUSE_MS: u64 = 30000; // 30 seconds pause between retr // Reconnection pub const RECONNECTION_DELAY_MS: u64 = 5000; // 5 seconds -// Execution concurrency -pub const EXECUTION_MIN_CONCURRENCY: usize = 4; +// Durable result outbox +pub const OUTBOX_MAX_ENTRIES: usize = 500; +pub const OUTBOX_FLUSH_INTERVAL_SECS: u64 = 30; +pub const FLUSH_PUBLISH_TIMEOUT_SECS: u64 = 30; +pub const OUTBOX_MAX_PAYLOAD_BYTES: usize = 1024 * 1024 - 64 * 1024; // Last-known-good update ratchet /// How long the updater waits for the new binary's boot marker. diff --git a/clients/openframe-client/src/doctor/checks.rs b/clients/openframe-client/src/doctor/checks.rs index f5c488460..6ee1b4ae9 100644 --- a/clients/openframe-client/src/doctor/checks.rs +++ b/clients/openframe-client/src/doctor/checks.rs @@ -302,6 +302,52 @@ pub async fn check_websocket_upgrade(server_url: &str) -> CheckResult { } } +// Detect the Edge WebView2 Runtime (required by the Tauri chat window) via its EdgeUpdate registry version. +#[cfg(windows)] +pub fn check_webview2_runtime() -> Option { + use winreg::enums::{HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE, KEY_READ}; + use winreg::RegKey; + + const CLIENT: &str = + r"SOFTWARE\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}"; + const CLIENT_WOW: &str = + r"SOFTWARE\WOW6432Node\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}"; + + let candidates = [ + (HKEY_LOCAL_MACHINE, CLIENT_WOW), + (HKEY_LOCAL_MACHINE, CLIENT), + (HKEY_CURRENT_USER, CLIENT), + (HKEY_CURRENT_USER, CLIENT_WOW), + ]; + + let version = candidates.iter().find_map(|(hive, path)| { + RegKey::predef(*hive) + .open_subkey_with_flags(path, KEY_READ) + .ok() + .and_then(|k| k.get_value::("pv").ok()) + .filter(|v| !v.trim().is_empty() && v.trim() != "0.0.0.0") + }); + + Some(match version { + Some(v) => CheckResult::pass( + CheckCategory::Runtime, + &format!("Runtime: WebView2 Runtime {} installed", v.trim()), + ), + None => CheckResult::warn( + CheckCategory::Runtime, + "Runtime: WebView2 Runtime not installed", + "The OpenFrame chat window requires the Microsoft Edge WebView2 Runtime and will not launch without it. \ + Install the Evergreen WebView2 Runtime from \ + https://developer.microsoft.com/microsoft-edge/webview2/.", + ), + }) +} + +#[cfg(not(windows))] +pub fn check_webview2_runtime() -> Option { + None +} + pub fn check_proxy_env() -> Option { let mut detected = Vec::new(); for var in ["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"] { diff --git a/clients/openframe-client/src/doctor/mod.rs b/clients/openframe-client/src/doctor/mod.rs index c7bc820e5..40f5c629f 100644 --- a/clients/openframe-client/src/doctor/mod.rs +++ b/clients/openframe-client/src/doctor/mod.rs @@ -9,6 +9,7 @@ use checks::*; pub enum CheckCategory { Command, Admin, + Runtime, Disk, Network, } @@ -128,6 +129,10 @@ pub async fn run_preinstall(params: &InstallConfigParams) -> DoctorReport { }; } + if let Some(webview2) = check_webview2_runtime() { + results.push(webview2); + } + let dir_manager = DirectoryManager::new(); let disk_targets: Vec<(&std::path::Path, &str)> = vec![ ( @@ -179,6 +184,10 @@ pub async fn run_healthcheck() -> DoctorReport { }; } + if let Some(webview2) = check_webview2_runtime() { + results.push(webview2); + } + let dir_manager = DirectoryManager::new(); let config_path = dir_manager.secured_dir().join("initial_config.json"); diff --git a/clients/openframe-client/src/executor/env.rs b/clients/openframe-client/src/executor/env.rs index 62b52118c..cd0af1e4a 100644 --- a/clients/openframe-client/src/executor/env.rs +++ b/clients/openframe-client/src/executor/env.rs @@ -18,21 +18,5 @@ pub(crate) fn apply_env_vars(cmd: &mut Command, env_vars: &[String]) { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn splits_simple() { - assert_eq!(split_env("KEY=value"), Some(("KEY", "value"))); - } - - #[test] - fn keeps_equals_in_value() { - assert_eq!(split_env("K=a=b"), Some(("K", "a=b"))); - } - - #[test] - fn rejects_without_equals() { - assert_eq!(split_env("NOEQ"), None); - } -} +#[path = "env_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/executor/env_tests.rs b/clients/openframe-client/src/executor/env_tests.rs new file mode 100644 index 000000000..ab7785ffd --- /dev/null +++ b/clients/openframe-client/src/executor/env_tests.rs @@ -0,0 +1,16 @@ +use super::*; + +#[test] +fn splits_simple() { + assert_eq!(split_env("KEY=value"), Some(("KEY", "value"))); +} + +#[test] +fn keeps_equals_in_value() { + assert_eq!(split_env("K=a=b"), Some(("K", "a=b"))); +} + +#[test] +fn rejects_without_equals() { + assert_eq!(split_env("NOEQ"), None); +} diff --git a/clients/openframe-client/src/executor/output.rs b/clients/openframe-client/src/executor/output.rs index 2207b5eae..e51352f5e 100644 --- a/clients/openframe-client/src/executor/output.rs +++ b/clients/openframe-client/src/executor/output.rs @@ -1,7 +1,34 @@ use tokio::io::{AsyncRead, AsyncReadExt}; +use tokio::task::JoinHandle; +use tokio::time::{timeout, Duration}; pub(crate) const MAX_OUTPUT_SIZE: usize = 10 * 1024 * 1024; +pub(crate) async fn join_reads( + mut stdout_task: JoinHandle>, + mut stderr_task: JoinHandle>, + grace: Duration, + kill: impl FnOnce(), +) -> (Vec, Vec) { + let pair = async { + let out = (&mut stdout_task).await.unwrap_or_default(); + let err = (&mut stderr_task).await.unwrap_or_default(); + (out, err) + }; + tokio::pin!(pair); + + match timeout(grace, &mut pair).await { + Ok(result) => result, + Err(_) => { + tracing::warn!( + "output streams still open after exit (backgrounded child?), killing process tree" + ); + kill(); + timeout(grace, &mut pair).await.unwrap_or_default() + } + } +} + pub(crate) async fn read_capped(reader: Option) -> Vec where R: AsyncRead + Unpin, @@ -63,25 +90,5 @@ pub(crate) fn clean_string(bytes: &[u8]) -> String { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn strips_nulls() { - assert_eq!(clean_string(b"a\x00b\x00c"), "abc"); - } - - #[test] - fn drops_invalid_keeps_replacement_char() { - let mut bytes = b"ok".to_vec(); - bytes.push(0xFF); - bytes.extend_from_slice(b"end"); - assert_eq!(clean_string(&bytes), "okend"); - assert_eq!(clean_string("a\u{FFFD}b".as_bytes()), "a\u{FFFD}b"); - } - - #[test] - fn passes_valid_utf8_through() { - assert_eq!(clean_string("héllo\nworld".as_bytes()), "héllo\nworld"); - } -} +#[path = "output_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/executor/output_tests.rs b/clients/openframe-client/src/executor/output_tests.rs new file mode 100644 index 000000000..47f1a238b --- /dev/null +++ b/clients/openframe-client/src/executor/output_tests.rs @@ -0,0 +1,20 @@ +use super::*; + +#[test] +fn strips_nulls() { + assert_eq!(clean_string(b"a\x00b\x00c"), "abc"); +} + +#[test] +fn drops_invalid_keeps_replacement_char() { + let mut bytes = b"ok".to_vec(); + bytes.push(0xFF); + bytes.extend_from_slice(b"end"); + assert_eq!(clean_string(&bytes), "okend"); + assert_eq!(clean_string("a\u{FFFD}b".as_bytes()), "a\u{FFFD}b"); +} + +#[test] +fn passes_valid_utf8_through() { + assert_eq!(clean_string("héllo\nworld".as_bytes()), "héllo\nworld"); +} diff --git a/clients/openframe-client/src/executor/tempfile.rs b/clients/openframe-client/src/executor/tempfile.rs index 11d8deca9..f34088ee7 100644 --- a/clients/openframe-client/src/executor/tempfile.rs +++ b/clients/openframe-client/src/executor/tempfile.rs @@ -17,22 +17,5 @@ pub(crate) fn temp_script_name(ext: &str) -> String { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn guard_removes_file_on_drop() { - let path = std::env::temp_dir().join(temp_script_name("sh")); - std::fs::write(&path, b"x").unwrap(); - assert!(path.exists()); - { - let _guard = TempFileGuard { path: path.clone() }; - } - assert!(!path.exists()); - } - - #[test] - fn names_are_unique() { - assert_ne!(temp_script_name("ps1"), temp_script_name("ps1")); - } -} +#[path = "tempfile_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/executor/tempfile_tests.rs b/clients/openframe-client/src/executor/tempfile_tests.rs new file mode 100644 index 000000000..084aa16f6 --- /dev/null +++ b/clients/openframe-client/src/executor/tempfile_tests.rs @@ -0,0 +1,17 @@ +use super::*; + +#[test] +fn guard_removes_file_on_drop() { + let path = std::env::temp_dir().join(temp_script_name("sh")); + std::fs::write(&path, b"x").unwrap(); + assert!(path.exists()); + { + let _guard = TempFileGuard { path: path.clone() }; + } + assert!(!path.exists()); +} + +#[test] +fn names_are_unique() { + assert_ne!(temp_script_name("ps1"), temp_script_name("ps1")); +} diff --git a/clients/openframe-client/src/executor/unix/mod.rs b/clients/openframe-client/src/executor/unix/mod.rs index 4b199369e..94f581b35 100644 --- a/clients/openframe-client/src/executor/unix/mod.rs +++ b/clients/openframe-client/src/executor/unix/mod.rs @@ -79,99 +79,5 @@ fn build_script_command(script_path: &Path, args: &[String]) -> tokio::process:: } #[cfg(test)] -mod tests { - use super::*; - - async fn run(code: &str, args: &[String], env: &[String]) -> ExecResult { - execute_script(ScriptParams { - code, - shell: "/bin/sh", - args, - timeout_secs: 30, - privilege: crate::executor::Privilege::Agent, - env_vars: env, - }) - .await - } - - #[tokio::test] - async fn happy_path() { - let r = run("#!/bin/sh\necho hi\n", &[], &[]).await; - assert_eq!(r.stdout, "hi\n"); - assert_eq!(r.retcode, 0); - assert!(!r.timed_out); - } - - #[tokio::test] - async fn script_exit_code() { - let r = run("#!/bin/sh\nexit 42\n", &[], &[]).await; - assert_eq!(r.retcode, 42); - } - - #[tokio::test] - async fn stderr_captured() { - let r = run("#!/bin/sh\necho err 1>&2\n", &[], &[]).await; - assert_eq!(r.stdout, ""); - assert_eq!(r.stderr, "err\n"); - } - - #[tokio::test] - async fn raw_no_trailing_newline() { - let r = run("#!/bin/sh\nprintf hi\n", &[], &[]).await; - assert_eq!(r.stdout, "hi"); - } - - #[tokio::test] - async fn raw_preserves_crlf_in_output() { - let r = run("#!/bin/sh\nprintf 'a\\r\\nb'\n", &[], &[]).await; - assert_eq!(r.stdout, "a\r\nb"); - } - - #[tokio::test] - async fn passes_args() { - let args = vec!["x".to_string(), "y".to_string()]; - let r = run("#!/bin/sh\necho \"$1 $2\"\n", &args, &[]).await; - assert_eq!(r.stdout, "x y\n"); - } - - #[tokio::test] - async fn applies_env_vars() { - let env = vec!["FOO=bar".to_string()]; - let r = run("#!/bin/sh\nprintf '%s' \"$FOO\"\n", &[], &env).await; - assert_eq!(r.stdout, "bar"); - } - - #[tokio::test] - async fn env_value_with_equals() { - let env = vec!["K=a=b".to_string()]; - let r = run("#!/bin/sh\nprintf '%s' \"$K\"\n", &[], &env).await; - assert_eq!(r.stdout, "a=b"); - } - - #[tokio::test] - async fn crlf_in_code_normalized() { - let r = run("#!/bin/sh\r\necho hi\r\n", &[], &[]).await; - assert_eq!(r.stdout, "hi\n"); - assert_eq!(r.retcode, 0); - } - - #[tokio::test] - async fn no_shebang_runs_via_sh() { - let r = run("echo hi\n", &[], &[]).await; - assert_eq!(r.stdout, "hi\n"); - assert_eq!(r.retcode, 0); - } - - #[tokio::test] - async fn bad_shebang_interpreter_fails() { - let r = run("#!/nonexistent/ofcmd-bad-interp\necho hi\n", &[], &[]).await; - assert_eq!(r.retcode, 85); - } - - #[tokio::test] - async fn empty_code_runs_empty() { - let r = run("", &[], &[]).await; - assert_eq!(r.retcode, 0); - assert_eq!(r.stdout, ""); - } -} +#[path = "mod_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/executor/unix/mod_tests.rs b/clients/openframe-client/src/executor/unix/mod_tests.rs new file mode 100644 index 000000000..f651d4b84 --- /dev/null +++ b/clients/openframe-client/src/executor/unix/mod_tests.rs @@ -0,0 +1,94 @@ +use super::*; + +async fn run(code: &str, args: &[String], env: &[String]) -> ExecResult { + execute_script(ScriptParams { + code, + shell: "/bin/sh", + args, + timeout_secs: 30, + privilege: crate::executor::Privilege::Agent, + env_vars: env, + }) + .await +} + +#[tokio::test] +async fn happy_path() { + let r = run("#!/bin/sh\necho hi\n", &[], &[]).await; + assert_eq!(r.stdout, "hi\n"); + assert_eq!(r.retcode, 0); + assert!(!r.timed_out); +} + +#[tokio::test] +async fn script_exit_code() { + let r = run("#!/bin/sh\nexit 42\n", &[], &[]).await; + assert_eq!(r.retcode, 42); +} + +#[tokio::test] +async fn stderr_captured() { + let r = run("#!/bin/sh\necho err 1>&2\n", &[], &[]).await; + assert_eq!(r.stdout, ""); + assert_eq!(r.stderr, "err\n"); +} + +#[tokio::test] +async fn raw_no_trailing_newline() { + let r = run("#!/bin/sh\nprintf hi\n", &[], &[]).await; + assert_eq!(r.stdout, "hi"); +} + +#[tokio::test] +async fn raw_preserves_crlf_in_output() { + let r = run("#!/bin/sh\nprintf 'a\\r\\nb'\n", &[], &[]).await; + assert_eq!(r.stdout, "a\r\nb"); +} + +#[tokio::test] +async fn passes_args() { + let args = vec!["x".to_string(), "y".to_string()]; + let r = run("#!/bin/sh\necho \"$1 $2\"\n", &args, &[]).await; + assert_eq!(r.stdout, "x y\n"); +} + +#[tokio::test] +async fn applies_env_vars() { + let env = vec!["FOO=bar".to_string()]; + let r = run("#!/bin/sh\nprintf '%s' \"$FOO\"\n", &[], &env).await; + assert_eq!(r.stdout, "bar"); +} + +#[tokio::test] +async fn env_value_with_equals() { + let env = vec!["K=a=b".to_string()]; + let r = run("#!/bin/sh\nprintf '%s' \"$K\"\n", &[], &env).await; + assert_eq!(r.stdout, "a=b"); +} + +#[tokio::test] +async fn crlf_in_code_normalized() { + let r = run("#!/bin/sh\r\necho hi\r\n", &[], &[]).await; + assert_eq!(r.stdout, "hi\n"); + assert_eq!(r.retcode, 0); +} + +#[tokio::test] +async fn no_shebang_runs_via_sh() { + let r = run("echo hi\n", &[], &[]).await; + assert_eq!(r.stdout, "hi\n"); + assert_eq!(r.retcode, 0); +} + +#[tokio::test] +async fn bad_shebang_interpreter_fails() { + let r = run("#!/nonexistent/ofcmd-bad-interp\necho hi\n", &[], &[]).await; + assert_eq!(r.retcode, 85); +} + +#[tokio::test] +async fn empty_code_runs_empty() { + let r = run("", &[], &[]).await; + assert_eq!(r.retcode, 0); + assert_eq!(r.stdout, ""); +} diff --git a/clients/openframe-client/src/executor/unix/process.rs b/clients/openframe-client/src/executor/unix/process.rs index ef60d1c9d..2c757b463 100644 --- a/clients/openframe-client/src/executor/unix/process.rs +++ b/clients/openframe-client/src/executor/unix/process.rs @@ -2,10 +2,9 @@ use std::os::unix::process::ExitStatusExt; use std::process::Stdio; use tokio::process::Command; -use tokio::task::JoinHandle; use tokio::time::{timeout, Duration}; -use crate::executor::output::{clean_string, read_capped}; +use crate::executor::output::{clean_string, join_reads, read_capped}; use crate::executor::ExecResult; const READ_GRACE: Duration = Duration::from_secs(5); @@ -32,7 +31,10 @@ pub(crate) async fn execute_with_timeout(mut cmd: Command, timeout_secs: u32) -> match timeout(Duration::from_secs(timeout_secs as u64), child.wait()).await { Ok(Ok(status)) => { - let (out, err) = join_reads(stdout_task, stderr_task, pid).await; + let (out, err) = join_reads(stdout_task, stderr_task, READ_GRACE, || { + kill_process_tree(pid) + }) + .await; let retcode = status .code() .unwrap_or_else(|| status.signal().map(|s| 128 + s).unwrap_or(1)); @@ -44,7 +46,10 @@ pub(crate) async fn execute_with_timeout(mut cmd: Command, timeout_secs: u32) -> } } Ok(Err(e)) => { - let (out, err) = join_reads(stdout_task, stderr_task, pid).await; + let (out, err) = join_reads(stdout_task, stderr_task, READ_GRACE, || { + kill_process_tree(pid) + }) + .await; ExecResult { stdout: clean_string(&out), stderr: format!("{}\n{}", clean_string(&err), e), @@ -54,7 +59,10 @@ pub(crate) async fn execute_with_timeout(mut cmd: Command, timeout_secs: u32) -> } Err(_) => { kill_process_tree(pid); - let (out, err) = join_reads(stdout_task, stderr_task, pid).await; + let (out, err) = join_reads(stdout_task, stderr_task, READ_GRACE, || { + kill_process_tree(pid) + }) + .await; let _ = timeout(READ_GRACE, child.wait()).await; ExecResult { stdout: clean_string(&out), @@ -96,31 +104,6 @@ async fn spawn_with_retry( Err(last_err.unwrap()) } -async fn join_reads( - mut stdout_task: JoinHandle>, - mut stderr_task: JoinHandle>, - pid: u32, -) -> (Vec, Vec) { - let pair = async { - let out = (&mut stdout_task).await.unwrap_or_default(); - let err = (&mut stderr_task).await.unwrap_or_default(); - (out, err) - }; - tokio::pin!(pair); - - match timeout(READ_GRACE, &mut pair).await { - Ok(result) => result, - Err(_) => { - tracing::warn!( - pid, - "output streams still open after exit (backgrounded child?), killing process group" - ); - kill_process_tree(pid); - timeout(READ_GRACE, &mut pair).await.unwrap_or_default() - } - } -} - fn kill_process_tree(pid: u32) { if pid == 0 { tracing::error!("refusing to kill process group 0 (would target the agent itself)"); @@ -132,77 +115,5 @@ fn kill_process_tree(pid: u32) { } #[cfg(test)] -mod tests { - use super::*; - use crate::executor::unix::run_as_user::{configure_preexec, RunAs}; - - fn sh(script: &str) -> Command { - let mut cmd = Command::new("/bin/sh"); - cmd.arg("-c").arg(script); - configure_preexec(&mut cmd, &RunAs::Current).unwrap(); - cmd - } - - #[tokio::test] - async fn timeout_yields_98() { - let r = execute_with_timeout(sh("sleep 5"), 1).await; - assert_eq!(r.retcode, 98); - assert!(r.timed_out); - assert!(r.stderr.contains("timed out")); - } - - #[tokio::test] - async fn instant_exit_zero() { - let r = execute_with_timeout(sh("exit 0"), 30).await; - assert_eq!(r.retcode, 0); - assert!(!r.timed_out); - } - - #[tokio::test] - async fn large_timeout_ok() { - let r = execute_with_timeout(sh("echo hi"), 999_999).await; - assert_eq!(r.retcode, 0); - assert_eq!(r.stdout, "hi\n"); - } - - #[tokio::test] - async fn output_capped_and_no_hang() { - let r = execute_with_timeout(sh("yes | head -c 11534336"), 30).await; - assert_eq!(r.stdout.len(), crate::executor::output::MAX_OUTPUT_SIZE); - assert!(!r.timed_out); - } - - #[tokio::test] - async fn backgrounded_child_does_not_hang() { - let started = std::time::Instant::now(); - let r = execute_with_timeout(sh("sleep 30 & exit 0"), 30).await; - assert_eq!(r.retcode, 0); - assert!(!r.timed_out); - assert!(started.elapsed() < Duration::from_secs(20)); - } - - #[tokio::test] - async fn signal_kill_yields_137() { - let r = execute_with_timeout(sh("kill -9 $$"), 30).await; - assert_eq!(r.retcode, 137); - assert!(!r.timed_out); - } - - #[tokio::test] - async fn signal_term_yields_143() { - let r = execute_with_timeout(sh("kill -15 $$"), 30).await; - assert_eq!(r.retcode, 143); - } - - #[tokio::test] - async fn spawn_with_retry_returns_non_etxtbsy_error() { - let mut cmd = Command::new("/nonexistent/ofcmd-no-such-binary"); - let result = spawn_with_retry(&mut cmd, 5).await; - assert!(result.is_err()); - } - - #[test] - fn kill_process_group_zero_is_refused() { - kill_process_tree(0); - } -} +#[path = "process_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/executor/unix/process_tests.rs b/clients/openframe-client/src/executor/unix/process_tests.rs new file mode 100644 index 000000000..55302c707 --- /dev/null +++ b/clients/openframe-client/src/executor/unix/process_tests.rs @@ -0,0 +1,72 @@ +use super::*; +use crate::executor::unix::run_as_user::{configure_preexec, RunAs}; + +fn sh(script: &str) -> Command { + let mut cmd = Command::new("/bin/sh"); + cmd.arg("-c").arg(script); + configure_preexec(&mut cmd, &RunAs::Current).unwrap(); + cmd +} + +#[tokio::test] +async fn timeout_yields_98() { + let r = execute_with_timeout(sh("sleep 5"), 1).await; + assert_eq!(r.retcode, 98); + assert!(r.timed_out); + assert!(r.stderr.contains("timed out")); +} + +#[tokio::test] +async fn instant_exit_zero() { + let r = execute_with_timeout(sh("exit 0"), 30).await; + assert_eq!(r.retcode, 0); + assert!(!r.timed_out); +} + +#[tokio::test] +async fn large_timeout_ok() { + let r = execute_with_timeout(sh("echo hi"), 999_999).await; + assert_eq!(r.retcode, 0); + assert_eq!(r.stdout, "hi\n"); +} + +#[tokio::test] +async fn output_capped_and_no_hang() { + let r = execute_with_timeout(sh("yes | head -c 11534336"), 30).await; + assert_eq!(r.stdout.len(), crate::executor::output::MAX_OUTPUT_SIZE); + assert!(!r.timed_out); +} + +#[tokio::test] +async fn backgrounded_child_does_not_hang() { + let started = std::time::Instant::now(); + let r = execute_with_timeout(sh("sleep 30 & exit 0"), 30).await; + assert_eq!(r.retcode, 0); + assert!(!r.timed_out); + assert!(started.elapsed() < Duration::from_secs(20)); +} + +#[tokio::test] +async fn signal_kill_yields_137() { + let r = execute_with_timeout(sh("kill -9 $$"), 30).await; + assert_eq!(r.retcode, 137); + assert!(!r.timed_out); +} + +#[tokio::test] +async fn signal_term_yields_143() { + let r = execute_with_timeout(sh("kill -15 $$"), 30).await; + assert_eq!(r.retcode, 143); +} + +#[tokio::test] +async fn spawn_with_retry_returns_non_etxtbsy_error() { + let mut cmd = Command::new("/nonexistent/ofcmd-no-such-binary"); + let result = spawn_with_retry(&mut cmd, 5).await; + assert!(result.is_err()); +} + +#[test] +fn kill_process_group_zero_is_refused() { + kill_process_tree(0); +} diff --git a/clients/openframe-client/src/executor/unix/run_as_user.rs b/clients/openframe-client/src/executor/unix/run_as_user.rs index dd5f6d349..315c41266 100644 --- a/clients/openframe-client/src/executor/unix/run_as_user.rs +++ b/clients/openframe-client/src/executor/unix/run_as_user.rs @@ -130,54 +130,5 @@ pub(crate) fn configure_preexec(cmd: &mut Command, run_as: &RunAs) -> Result<()> } #[cfg(test)] -mod tests { - use super::*; - use crate::executor::{execute_script, Privilege, ScriptParams}; - - fn params<'a>(code: &'a str, privilege: Privilege) -> ScriptParams<'a> { - ScriptParams { - code, - shell: "/bin/sh", - args: &[], - timeout_secs: 30, - privilege, - env_vars: &[], - } - } - - #[tokio::test] - async fn agent_privilege_is_current() { - assert!(matches!( - resolve_run_as(Privilege::Agent).await.unwrap(), - RunAs::Current - )); - } - - #[tokio::test] - async fn agent_privilege_runs_as_agent() { - let r = execute_script(params("#!/bin/sh\necho hi\n", Privilege::Agent)).await; - assert_eq!(r.stdout, "hi\n"); - assert_eq!(r.retcode, 0); - assert!(!r.timed_out); - } - - #[tokio::test] - async fn user_privilege_runs_or_hard_fails() { - let r = execute_script(params("#!/bin/sh\nid -u\n", Privilege::User)).await; - assert!( - r.retcode == 0 || r.retcode == 85, - "retcode was {}", - r.retcode - ); - assert!(!r.timed_out); - } - - #[tokio::test] - #[ignore = "requires root + an active interactive session"] - async fn user_privilege_drops_privilege() { - let r = execute_script(params("#!/bin/sh\nid -u\n", Privilege::User)).await; - assert_eq!(r.retcode, 0); - let uid: u32 = r.stdout.trim().parse().unwrap(); - assert_ne!(uid, 0); - } -} +#[path = "run_as_user_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/executor/unix/run_as_user_tests.rs b/clients/openframe-client/src/executor/unix/run_as_user_tests.rs new file mode 100644 index 000000000..d75a84ae7 --- /dev/null +++ b/clients/openframe-client/src/executor/unix/run_as_user_tests.rs @@ -0,0 +1,49 @@ +use super::*; +use crate::executor::{execute_script, Privilege, ScriptParams}; + +fn params<'a>(code: &'a str, privilege: Privilege) -> ScriptParams<'a> { + ScriptParams { + code, + shell: "/bin/sh", + args: &[], + timeout_secs: 30, + privilege, + env_vars: &[], + } +} + +#[tokio::test] +async fn agent_privilege_is_current() { + assert!(matches!( + resolve_run_as(Privilege::Agent).await.unwrap(), + RunAs::Current + )); +} + +#[tokio::test] +async fn agent_privilege_runs_as_agent() { + let r = execute_script(params("#!/bin/sh\necho hi\n", Privilege::Agent)).await; + assert_eq!(r.stdout, "hi\n"); + assert_eq!(r.retcode, 0); + assert!(!r.timed_out); +} + +#[tokio::test] +async fn user_privilege_runs_or_hard_fails() { + let r = execute_script(params("#!/bin/sh\nid -u\n", Privilege::User)).await; + assert!( + r.retcode == 0 || r.retcode == 85, + "retcode was {}", + r.retcode + ); + assert!(!r.timed_out); +} + +#[tokio::test] +#[ignore = "requires root + an active interactive session"] +async fn user_privilege_drops_privilege() { + let r = execute_script(params("#!/bin/sh\nid -u\n", Privilege::User)).await; + assert_eq!(r.retcode, 0); + let uid: u32 = r.stdout.trim().parse().unwrap(); + assert_ne!(uid, 0); +} diff --git a/clients/openframe-client/src/executor/windows/mod.rs b/clients/openframe-client/src/executor/windows/mod.rs index 3cb4281f0..0a68beae8 100644 --- a/clients/openframe-client/src/executor/windows/mod.rs +++ b/clients/openframe-client/src/executor/windows/mod.rs @@ -140,268 +140,5 @@ fn create_temp_script(code: &str, ext: &str) -> Result { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn resolves_powershell() { - let i = resolve_interpreter("powershell").unwrap(); - assert!( - i.exe.to_lowercase().ends_with("powershell.exe"), - "exe was {:?}", - i.exe - ); - assert_eq!(i.ext, "ps1"); - assert_eq!(i.flags.last(), Some(&"-File")); - } - - #[test] - fn resolves_cmd() { - let i = resolve_interpreter("cmd").unwrap(); - assert!( - i.exe.to_lowercase().ends_with("cmd.exe"), - "exe was {:?}", - i.exe - ); - assert_eq!(i.ext, "bat"); - } - - #[test] - fn resolves_exe_to_absolute_system32() { - let ps = resolve_interpreter("powershell").unwrap().exe; - assert!( - std::path::Path::new(&ps).is_absolute(), - "expected absolute powershell path, got {ps:?}" - ); - } - - #[test] - fn rejects_unknown_shell() { - assert!(resolve_interpreter("bash").is_none()); - assert!(resolve_interpreter("").is_none()); - } - - #[tokio::test] - async fn unknown_shell_is_85() { - let r = execute_script(ScriptParams { - code: "echo hi", - shell: "bash", - args: &[], - timeout_secs: 30, - privilege: crate::executor::Privilege::Agent, - env_vars: &[], - }) - .await; - assert_eq!(r.retcode, 85); - assert!(r.stderr.contains("unsupported shell")); - } - - async fn run( - shell: &str, - code: &str, - args: &[String], - env: &[String], - timeout: u32, - ) -> ExecResult { - execute_script(ScriptParams { - code, - shell, - args, - timeout_secs: timeout, - privilege: crate::executor::Privilege::Agent, - env_vars: env, - }) - .await - } - - #[tokio::test] - async fn powershell_happy_path() { - let r = run("powershell", "Write-Output hi", &[], &[], 30).await; - assert_eq!(r.retcode, 0, "stderr: {}", r.stderr); - assert_eq!(r.stdout.trim_end(), "hi"); - assert!(!r.timed_out); - } - - #[tokio::test] - async fn cmd_happy_path() { - let r = run("cmd", "@echo hi", &[], &[], 30).await; - assert_eq!(r.retcode, 0, "stderr: {}", r.stderr); - assert!(r.stdout.contains("hi"), "stdout: {:?}", r.stdout); - assert!(!r.timed_out); - } - - #[tokio::test] - async fn cmd_exit_code() { - let r = run("cmd", "@exit /b 42", &[], &[], 30).await; - assert_eq!(r.retcode, 42); - } - - #[tokio::test] - async fn powershell_exit_code() { - let r = run("powershell", "exit 42", &[], &[], 30).await; - assert_eq!( - r.retcode, 42, - "stdout: {:?} stderr: {:?}", - r.stdout, r.stderr - ); - } - - #[tokio::test] - async fn powershell_stderr_captured() { - let r = run( - "powershell", - "[Console]::Error.WriteLine('boom')", - &[], - &[], - 30, - ) - .await; - assert!(r.stderr.contains("boom"), "stderr: {:?}", r.stderr); - } - - #[tokio::test] - async fn powershell_timeout_is_98() { - let r = run("powershell", "Start-Sleep -Seconds 60", &[], &[], 2).await; - assert_eq!(r.retcode, 98); - assert!(r.timed_out); - } - - #[tokio::test] - async fn powershell_passes_args() { - let args = vec!["alpha".to_string(), "beta".to_string()]; - let r = run( - "powershell", - "Write-Output \"$($args[0]) $($args[1])\"", - &args, - &[], - 30, - ) - .await; - assert_eq!(r.stdout.trim_end(), "alpha beta", "stderr: {}", r.stderr); - } - - #[tokio::test] - async fn powershell_applies_env_vars() { - let env = vec!["FOO=bar".to_string()]; - let r = run("powershell", "Write-Output $env:FOO", &[], &env, 30).await; - assert_eq!(r.stdout.trim_end(), "bar", "stderr: {}", r.stderr); - } - - #[tokio::test] - async fn cmd_applies_env_vars() { - let env = vec!["FOO=bar".to_string()]; - let r = run("cmd", "@echo %FOO%", &[], &env, 30).await; - assert_eq!(r.stdout.trim_end(), "bar", "stderr: {}", r.stderr); - } - - #[tokio::test] - async fn env_value_with_equals() { - let env = vec!["K=a=b".to_string()]; - let r = run("powershell", "Write-Output $env:K", &[], &env, 30).await; - assert_eq!(r.stdout.trim_end(), "a=b", "stderr: {}", r.stderr); - } - - #[tokio::test] - async fn empty_code_runs_ok() { - let r = run("powershell", "", &[], &[], 30).await; - assert_eq!(r.retcode, 0, "stderr: {}", r.stderr); - } - - #[tokio::test] - #[ignore = "spawns real child processes; run explicitly with --ignored"] - async fn timeout_tree_kills_grandchild() { - let marker = std::env::temp_dir().join(format!( - "ofcmd_heartbeat_{}.txt", - uuid::Uuid::new_v4().simple() - )); - let marker_ps = marker.to_string_lossy().replace('\'', "''"); - let code = format!( - "$gc = '$m = ''{m}''; while ($true) {{ Set-Content -Path $m -Value ([DateTime]::UtcNow.Ticks); Start-Sleep -Milliseconds 200 }}'\n\ - Start-Process powershell -WindowStyle Hidden -ArgumentList '-NoProfile','-Command',$gc\n\ - Start-Sleep -Seconds 120\n", - m = marker_ps - ); - - let r = run("powershell", &code, &[], &[], 3).await; - assert_eq!(r.retcode, 98, "expected timeout; stderr: {}", r.stderr); - assert!(r.timed_out); - - let mut waited = 0; - while !marker.exists() && waited < 5000 { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - waited += 100; - } - assert!(marker.exists(), "grandchild never wrote heartbeat marker"); - - let first = std::fs::read_to_string(&marker).unwrap_or_default(); - tokio::time::sleep(std::time::Duration::from_millis(1500)).await; - let second = std::fs::read_to_string(&marker).unwrap_or_default(); - let _ = std::fs::remove_file(&marker); - - assert_eq!( - first, second, - "heartbeat still advancing after timeout — grandchild escaped the Job Object tree-kill" - ); - } - - #[tokio::test] - #[ignore = "context-dependent: asserts 85 only when agent is NOT SYSTEM"] - async fn run_as_without_privilege_hard_fails_85() { - let r = execute_script(ScriptParams { - code: "Write-Output whoami", - shell: "powershell", - args: &[], - timeout_secs: 30, - privilege: crate::executor::Privilege::User, - env_vars: &[], - }) - .await; - assert_eq!( - r.retcode, 85, - "expected hard-fail 85 (no silent fallback); got {} stdout={:?} stderr={:?}", - r.retcode, r.stdout, r.stderr - ); - assert!( - r.stdout.is_empty(), - "must not have run anything: {:?}", - r.stdout - ); - } - - #[tokio::test] - #[ignore = "requires agent running as SYSTEM with an active interactive session"] - async fn run_as_user_impersonates_interactive_user() { - let r = execute_script(ScriptParams { - code: "whoami", - shell: "cmd", - args: &[], - timeout_secs: 30, - privilege: crate::executor::Privilege::User, - env_vars: &[], - }) - .await; - assert_eq!(r.retcode, 0, "stderr: {}", r.stderr); - let who = r.stdout.trim().to_lowercase(); - assert!( - !who.ends_with("\\system") && who != "nt authority\\system", - "expected the interactive user, not SYSTEM; got {who:?}" - ); - } - - #[tokio::test] - #[ignore = "requires agent running as SYSTEM with an active interactive session"] - async fn run_as_user_sees_env_vars() { - let env = vec!["OF_RUNAS_PROBE=present".to_string()]; - let r = execute_script(ScriptParams { - code: "Write-Output $env:OF_RUNAS_PROBE", - shell: "powershell", - args: &[], - timeout_secs: 30, - privilege: crate::executor::Privilege::User, - env_vars: &env, - }) - .await; - assert_eq!(r.retcode, 0, "stderr: {}", r.stderr); - assert_eq!(r.stdout.trim_end(), "present", "stderr: {}", r.stderr); - } -} +#[path = "mod_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/executor/windows/mod_tests.rs b/clients/openframe-client/src/executor/windows/mod_tests.rs new file mode 100644 index 000000000..fe1b92ec8 --- /dev/null +++ b/clients/openframe-client/src/executor/windows/mod_tests.rs @@ -0,0 +1,257 @@ +use super::*; + +#[test] +fn resolves_powershell() { + let i = resolve_interpreter("powershell").unwrap(); + assert!( + i.exe.to_lowercase().ends_with("powershell.exe"), + "exe was {:?}", + i.exe + ); + assert_eq!(i.ext, "ps1"); + assert_eq!(i.flags.last(), Some(&"-File")); +} + +#[test] +fn resolves_cmd() { + let i = resolve_interpreter("cmd").unwrap(); + assert!( + i.exe.to_lowercase().ends_with("cmd.exe"), + "exe was {:?}", + i.exe + ); + assert_eq!(i.ext, "bat"); +} + +#[test] +fn resolves_exe_to_absolute_system32() { + let ps = resolve_interpreter("powershell").unwrap().exe; + assert!( + std::path::Path::new(&ps).is_absolute(), + "expected absolute powershell path, got {ps:?}" + ); +} + +#[test] +fn rejects_unknown_shell() { + assert!(resolve_interpreter("bash").is_none()); + assert!(resolve_interpreter("").is_none()); +} + +#[tokio::test] +async fn unknown_shell_is_85() { + let r = execute_script(ScriptParams { + code: "echo hi", + shell: "bash", + args: &[], + timeout_secs: 30, + privilege: crate::executor::Privilege::Agent, + env_vars: &[], + }) + .await; + assert_eq!(r.retcode, 85); + assert!(r.stderr.contains("unsupported shell")); +} + +async fn run(shell: &str, code: &str, args: &[String], env: &[String], timeout: u32) -> ExecResult { + execute_script(ScriptParams { + code, + shell, + args, + timeout_secs: timeout, + privilege: crate::executor::Privilege::Agent, + env_vars: env, + }) + .await +} + +#[tokio::test] +async fn powershell_happy_path() { + let r = run("powershell", "Write-Output hi", &[], &[], 30).await; + assert_eq!(r.retcode, 0, "stderr: {}", r.stderr); + assert_eq!(r.stdout.trim_end(), "hi"); + assert!(!r.timed_out); +} + +#[tokio::test] +async fn cmd_happy_path() { + let r = run("cmd", "@echo hi", &[], &[], 30).await; + assert_eq!(r.retcode, 0, "stderr: {}", r.stderr); + assert!(r.stdout.contains("hi"), "stdout: {:?}", r.stdout); + assert!(!r.timed_out); +} + +#[tokio::test] +async fn cmd_exit_code() { + let r = run("cmd", "@exit /b 42", &[], &[], 30).await; + assert_eq!(r.retcode, 42); +} + +#[tokio::test] +async fn powershell_exit_code() { + let r = run("powershell", "exit 42", &[], &[], 30).await; + assert_eq!( + r.retcode, 42, + "stdout: {:?} stderr: {:?}", + r.stdout, r.stderr + ); +} + +#[tokio::test] +async fn powershell_stderr_captured() { + let r = run( + "powershell", + "[Console]::Error.WriteLine('boom')", + &[], + &[], + 30, + ) + .await; + assert!(r.stderr.contains("boom"), "stderr: {:?}", r.stderr); +} + +#[tokio::test] +async fn powershell_timeout_is_98() { + let r = run("powershell", "Start-Sleep -Seconds 60", &[], &[], 2).await; + assert_eq!(r.retcode, 98); + assert!(r.timed_out); +} + +#[tokio::test] +async fn powershell_passes_args() { + let args = vec!["alpha".to_string(), "beta".to_string()]; + let r = run( + "powershell", + "Write-Output \"$($args[0]) $($args[1])\"", + &args, + &[], + 30, + ) + .await; + assert_eq!(r.stdout.trim_end(), "alpha beta", "stderr: {}", r.stderr); +} + +#[tokio::test] +async fn powershell_applies_env_vars() { + let env = vec!["FOO=bar".to_string()]; + let r = run("powershell", "Write-Output $env:FOO", &[], &env, 30).await; + assert_eq!(r.stdout.trim_end(), "bar", "stderr: {}", r.stderr); +} + +#[tokio::test] +async fn cmd_applies_env_vars() { + let env = vec!["FOO=bar".to_string()]; + let r = run("cmd", "@echo %FOO%", &[], &env, 30).await; + assert_eq!(r.stdout.trim_end(), "bar", "stderr: {}", r.stderr); +} + +#[tokio::test] +async fn env_value_with_equals() { + let env = vec!["K=a=b".to_string()]; + let r = run("powershell", "Write-Output $env:K", &[], &env, 30).await; + assert_eq!(r.stdout.trim_end(), "a=b", "stderr: {}", r.stderr); +} + +#[tokio::test] +async fn empty_code_runs_ok() { + let r = run("powershell", "", &[], &[], 30).await; + assert_eq!(r.retcode, 0, "stderr: {}", r.stderr); +} + +#[tokio::test] +#[ignore = "spawns real child processes; run explicitly with --ignored"] +async fn timeout_tree_kills_grandchild() { + let marker = std::env::temp_dir().join(format!( + "ofcmd_heartbeat_{}.txt", + uuid::Uuid::new_v4().simple() + )); + let marker_ps = marker.to_string_lossy().replace('\'', "''"); + let code = format!( + "$gc = '$m = ''{m}''; while ($true) {{ Set-Content -Path $m -Value ([DateTime]::UtcNow.Ticks); Start-Sleep -Milliseconds 200 }}'\n\ + Start-Process powershell -WindowStyle Hidden -ArgumentList '-NoProfile','-Command',$gc\n\ + Start-Sleep -Seconds 120\n", + m = marker_ps + ); + + let r = run("powershell", &code, &[], &[], 3).await; + assert_eq!(r.retcode, 98, "expected timeout; stderr: {}", r.stderr); + assert!(r.timed_out); + + let mut waited = 0; + while !marker.exists() && waited < 5000 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + waited += 100; + } + assert!(marker.exists(), "grandchild never wrote heartbeat marker"); + + let first = std::fs::read_to_string(&marker).unwrap_or_default(); + tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + let second = std::fs::read_to_string(&marker).unwrap_or_default(); + let _ = std::fs::remove_file(&marker); + + assert_eq!( + first, second, + "heartbeat still advancing after timeout — grandchild escaped the Job Object tree-kill" + ); +} + +#[tokio::test] +#[ignore = "context-dependent: asserts 85 only when agent is NOT SYSTEM"] +async fn run_as_without_privilege_hard_fails_85() { + let r = execute_script(ScriptParams { + code: "Write-Output whoami", + shell: "powershell", + args: &[], + timeout_secs: 30, + privilege: crate::executor::Privilege::User, + env_vars: &[], + }) + .await; + assert_eq!( + r.retcode, 85, + "expected hard-fail 85 (no silent fallback); got {} stdout={:?} stderr={:?}", + r.retcode, r.stdout, r.stderr + ); + assert!( + r.stdout.is_empty(), + "must not have run anything: {:?}", + r.stdout + ); +} + +#[tokio::test] +#[ignore = "requires agent running as SYSTEM with an active interactive session"] +async fn run_as_user_impersonates_interactive_user() { + let r = execute_script(ScriptParams { + code: "whoami", + shell: "cmd", + args: &[], + timeout_secs: 30, + privilege: crate::executor::Privilege::User, + env_vars: &[], + }) + .await; + assert_eq!(r.retcode, 0, "stderr: {}", r.stderr); + let who = r.stdout.trim().to_lowercase(); + assert!( + !who.ends_with("\\system") && who != "nt authority\\system", + "expected the interactive user, not SYSTEM; got {who:?}" + ); +} + +#[tokio::test] +#[ignore = "requires agent running as SYSTEM with an active interactive session"] +async fn run_as_user_sees_env_vars() { + let env = vec!["OF_RUNAS_PROBE=present".to_string()]; + let r = execute_script(ScriptParams { + code: "Write-Output $env:OF_RUNAS_PROBE", + shell: "powershell", + args: &[], + timeout_secs: 30, + privilege: crate::executor::Privilege::User, + env_vars: &env, + }) + .await; + assert_eq!(r.retcode, 0, "stderr: {}", r.stderr); + assert_eq!(r.stdout.trim_end(), "present", "stderr: {}", r.stderr); +} diff --git a/clients/openframe-client/src/executor/windows/process.rs b/clients/openframe-client/src/executor/windows/process.rs index a531c1482..9e96c2153 100644 --- a/clients/openframe-client/src/executor/windows/process.rs +++ b/clients/openframe-client/src/executor/windows/process.rs @@ -7,10 +7,11 @@ use tokio::time::{timeout, Duration}; use super::job::JobHandle; use super::Interpreter; use crate::executor::env::apply_env_vars; -use crate::executor::output::{clean_string, read_capped}; +use crate::executor::output::{clean_string, join_reads, read_capped}; use crate::executor::{ExecResult, ScriptParams}; const CREATE_NO_WINDOW: u32 = 0x0800_0000; +const READ_GRACE: Duration = Duration::from_secs(5); pub(super) async fn run_normal( interpreter: &Interpreter, @@ -53,8 +54,8 @@ pub(super) async fn run_normal( .await { Ok(Ok(status)) => { - let out = stdout_task.await.unwrap_or_default(); - let err = stderr_task.await.unwrap_or_default(); + let (out, err) = + join_reads(stdout_task, stderr_task, READ_GRACE, || job.terminate()).await; ExecResult { stdout: clean_string(&out), stderr: clean_string(&err), @@ -63,8 +64,8 @@ pub(super) async fn run_normal( } } Ok(Err(e)) => { - let out = stdout_task.await.unwrap_or_default(); - let err = stderr_task.await.unwrap_or_default(); + let (out, err) = + join_reads(stdout_task, stderr_task, READ_GRACE, || job.terminate()).await; ExecResult { stdout: clean_string(&out), stderr: format!("{}\n{}", clean_string(&err), e), @@ -75,8 +76,8 @@ pub(super) async fn run_normal( Err(_) => { job.terminate(); let _ = child.start_kill(); - let out = stdout_task.await.unwrap_or_default(); - let err = stderr_task.await.unwrap_or_default(); + let (out, err) = + join_reads(stdout_task, stderr_task, READ_GRACE, || job.terminate()).await; let _ = child.wait().await; ExecResult { stdout: clean_string(&out), diff --git a/clients/openframe-client/src/lib.rs b/clients/openframe-client/src/lib.rs index 3d70402d5..310a5fd94 100644 --- a/clients/openframe-client/src/lib.rs +++ b/clients/openframe-client/src/lib.rs @@ -38,9 +38,7 @@ pub mod executor; use crate::clients::tool_agent_file_client::ToolAgentFileClient; use crate::clients::{AuthClient, RegistrationClient, ToolApiClient}; -use crate::config::update_config::{ - DOWNLOAD_CLIENT_TIMEOUT_SECS, EXECUTION_MIN_CONCURRENCY, HTTP_CLIENT_TIMEOUT_SECS, -}; +use crate::config::update_config::{DOWNLOAD_CLIENT_TIMEOUT_SECS, HTTP_CLIENT_TIMEOUT_SECS}; use crate::listener::execution_listener::ExecutionListener; use crate::listener::openframe_client_update_listener::OpenFrameClientUpdateListener; use crate::listener::tool_agent_update_listener::ToolAgentUpdateListener; @@ -48,10 +46,11 @@ use crate::listener::tool_installation_message_listener::ToolInstallationMessage use crate::listener::tool_restart_message_listener::ToolRestartMessageListener; use crate::listener::tool_uninstall_message_listener::ToolUninstallMessageListener; use crate::logging::nats_streaming::LogStreamingRunManager; -use crate::models::{CommandMessage, ScriptMessage}; +use crate::models::{CommandMessage, ScriptMessage, ScriptScheduleExecutionMessage}; use crate::platform::DirectoryManager; use crate::platform::DmgExtractor; use crate::services::agent_configuration_service::AgentConfigurationService; +use crate::services::deactivation_service::DeactivationService; use crate::services::device_data_fetcher::DeviceDataFetcher; use crate::services::encryption_service::EncryptionService; use crate::services::execution_service::ExecutionService; @@ -67,6 +66,8 @@ use crate::services::nats_message_publisher::NatsMessagePublisher; use crate::services::openframe_client_info_service::OpenFrameClientInfoService; use crate::services::openframe_client_update_service::OpenFrameClientUpdateService; use crate::services::registration_processor::RegistrationProcessor; +use crate::services::result_outbox_run_manager::ResultOutboxRunManager; +use crate::services::result_store::ResultStore; use crate::services::shared_token_service::SharedTokenService; use crate::services::token_refresh_run_manager::TokenRefreshRunManager; use crate::services::tool_agent_update_service::ToolAgentUpdateService; @@ -159,11 +160,14 @@ pub struct Client { tool_agent_update_listener: ToolAgentUpdateListener, command_execution_listener: ExecutionListener, script_execution_listener: ExecutionListener, + script_schedule_execution_listener: ExecutionListener, tool_run_manager: ToolRunManager, token_refresh_run_manager: TokenRefreshRunManager, mesh_self_heal_service: MeshSelfHealService, tool_connection_processing_manager: ToolConnectionProcessingManager, machine_heartbeat_run_manager: MachineHeartbeatRunManager, + result_outbox_run_manager: ResultOutboxRunManager, + result_store: Arc, update_handler_service: UpdateHandlerService, openframe_client_info_service: OpenFrameClientInfoService, last_known_good_service: LastKnownGoodService, @@ -172,6 +176,7 @@ pub struct Client { agent_configuration_service: AgentConfigurationService, installed_tools_service: InstalledToolsService, initial_key_service: Arc, + deactivation_service: Arc, } impl Client { @@ -189,6 +194,9 @@ impl Client { // Perform initial health check directory_manager.perform_health_check()?; + // Detects the gateway's 410 Gone (tenant deleted) and drives backoff / stop / self-uninstall. + let deactivation_service = DeactivationService::new(&directory_manager); + // Initialize initial configuration service let initial_configuration_service = InitialConfigurationService::new(directory_manager.clone()) @@ -242,7 +250,11 @@ impl Client { RegistrationProcessor::new(registration_service, config_service.clone()); // Initialize authentication client - let auth_client = AuthClient::new(http_url.clone(), http_client.clone()); + let auth_client = AuthClient::new( + http_url.clone(), + http_client.clone(), + deactivation_service.clone(), + ); // Initialize encryption service let encryption_service = EncryptionService::new(); @@ -264,8 +276,11 @@ impl Client { // Initialize proactive token refresh run manager (keeps shared_token.enc valid // independent of NATS reconnects) - let token_refresh_run_manager = - TokenRefreshRunManager::new(auth_service.clone(), config_service.clone()); + let token_refresh_run_manager = TokenRefreshRunManager::new( + auth_service.clone(), + config_service.clone(), + deactivation_service.clone(), + ); // Initialize NATS connection manager let ws_url = format!("wss://{}", initial_configuration_service.get_server_url()?); @@ -277,6 +292,7 @@ impl Client { initial_configuration_service.clone(), auth_service.clone(), tls_config_provider, + deactivation_service.clone(), ); // Initialize tool agent file client @@ -295,6 +311,7 @@ impl Client { http_url.clone(), initial_configuration_service.clone(), config_service.clone(), + deactivation_service.clone(), )); // Initialize installed tools service @@ -349,6 +366,7 @@ impl Client { initial_configuration_service.clone(), config_service.clone(), tool_run_manager.clone(), + deactivation_service.clone(), ); // Initialize tool connection service @@ -466,25 +484,45 @@ impl Client { ); let execution_service = ExecutionService::new(); - let execution_concurrency = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(EXECUTION_MIN_CONCURRENCY) - .max(EXECUTION_MIN_CONCURRENCY); - let execution_semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency)); + + let result_store = Arc::new(ResultStore::open_or_degrade( + directory_manager + .secured_dir() + .join("scheduled_outbox.redb"), + )); + let flush_notify = Arc::new(tokio::sync::Notify::new()); + let result_outbox_run_manager = ResultOutboxRunManager::new( + result_store.clone(), + Arc::new(nats_message_publisher.clone()), + flush_notify.clone(), + ); + let result_store_for_recovery = result_store.clone(); + let command_execution_listener = ExecutionListener::::new( nats_connection_manager.clone(), nats_message_publisher.clone(), execution_service.clone(), config_service.clone(), - execution_semaphore.clone(), + result_store.clone(), + flush_notify.clone(), ); let script_execution_listener = ExecutionListener::::new( nats_connection_manager.clone(), nats_message_publisher.clone(), - execution_service, + execution_service.clone(), config_service.clone(), - execution_semaphore, + result_store.clone(), + flush_notify.clone(), ); + let script_schedule_execution_listener = + ExecutionListener::::new( + nats_connection_manager.clone(), + nats_message_publisher.clone(), + execution_service, + config_service.clone(), + result_store.clone(), + flush_notify.clone(), + ); // Initialize machine heartbeat publisher and run manager let machine_heartbeat_publisher = @@ -515,11 +553,14 @@ impl Client { tool_agent_update_listener, command_execution_listener, script_execution_listener, + script_schedule_execution_listener, tool_run_manager, token_refresh_run_manager, mesh_self_heal_service, tool_connection_processing_manager, machine_heartbeat_run_manager, + result_outbox_run_manager, + result_store: result_store_for_recovery, update_handler_service, openframe_client_info_service, last_known_good_service, @@ -527,12 +568,19 @@ impl Client { agent_configuration_service: config_service, installed_tools_service, initial_key_service, + deactivation_service, }) } pub async fn start(&self) -> Result<()> { info!("Starting OpenFrame Client"); + // Tenant-gone supervisor: stops/restarts tools and self-uninstalls off the detection path. + // Started first so its commands are consumed even if the startup auth loop (which itself + // feeds the 410s) blocks below. + self.deactivation_service + .start(self.tool_run_manager.clone()); + if let Err(e) = self .openframe_client_info_service .reconcile_version(env!("OPENFRAME_VERSION")) @@ -596,12 +644,22 @@ impl Client { // Start tool agent update listener in background self.tool_agent_update_listener.start().await?; + // Recover interrupted scheduled scripts, then start the outbox flusher, + // both strictly before any execution listener can accept new batches + if let Err(e) = self.result_store.recover().await { + error!("Failed to recover scheduled result outbox: {:#}", e); + } + self.result_outbox_run_manager.start(); + info!("Starting command execution listener..."); self.command_execution_listener.start().await?; info!("Command execution listener started"); info!("Starting script execution listener..."); self.script_execution_listener.start().await?; info!("Script execution listener started"); + info!("Starting script schedule execution listener..."); + self.script_schedule_execution_listener.start().await?; + info!("Script schedule execution listener started"); // Start tool run manager self.tool_run_manager.run().await?; diff --git a/clients/openframe-client/src/listener/execution_listener.rs b/clients/openframe-client/src/listener/execution_listener.rs index 985afeb5f..072a817a9 100644 --- a/clients/openframe-client/src/listener/execution_listener.rs +++ b/clients/openframe-client/src/listener/execution_listener.rs @@ -4,15 +4,16 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; use async_nats::Message; use futures::StreamExt; -use tokio::sync::Semaphore; +use tokio::sync::Notify; use tokio::time::Duration; use tracing::{error, info, warn}; use crate::config::update_config::RECONNECTION_DELAY_MS; -use crate::models::ExecutionMessage; +use crate::models::{ExecutionMessage, ExecutionRequest, RmmResult}; use crate::services::execution_service::ExecutionService; use crate::services::nats_connection_manager::NatsConnectionManager; use crate::services::nats_message_publisher::NatsMessagePublisher; +use crate::services::result_store::{entry_key, now_secs, JournalRecord, ResultStore}; use crate::services::AgentConfigurationService; pub struct ExecutionListener { @@ -20,7 +21,8 @@ pub struct ExecutionListener { nats_message_publisher: NatsMessagePublisher, execution_service: ExecutionService, config_service: AgentConfigurationService, - semaphore: Arc, + result_store: Arc, + flush_notify: Arc, _marker: PhantomData M>, } @@ -31,7 +33,8 @@ impl Clone for ExecutionListener { nats_message_publisher: self.nats_message_publisher.clone(), execution_service: self.execution_service.clone(), config_service: self.config_service.clone(), - semaphore: self.semaphore.clone(), + result_store: self.result_store.clone(), + flush_notify: self.flush_notify.clone(), _marker: PhantomData, } } @@ -43,14 +46,16 @@ impl ExecutionListener { nats_message_publisher: NatsMessagePublisher, execution_service: ExecutionService, config_service: AgentConfigurationService, - semaphore: Arc, + result_store: Arc, + flush_notify: Arc, ) -> Self { Self { nats_connection_manager, nats_message_publisher, execution_service, config_service, - semaphore, + result_store, + flush_notify, _marker: PhantomData, } } @@ -90,8 +95,10 @@ impl ExecutionListener { info!(subject = %subject, "Execution listener active"); + let queued = subscriber.inspect(|_| info!(kind = M::KIND, "Execution message received")); + let listener = self.clone(); - run_bounded(subscriber, self.semaphore.clone(), move |message| { + run_unbounded(queued, move |message| { let listener = listener.clone(); let machine_id = machine_id.clone(); async move { @@ -119,158 +126,179 @@ impl ExecutionListener { }; let execution_id = parsed.execution_id().to_string(); let schedule_id = parsed.schedule_id().unwrap_or("-").to_string(); - info!(kind = M::KIND, execution_id = %execution_id, schedule_id = %schedule_id, "Execution request received"); - - let request = parsed.to_request(); - let result = self.execution_service.execute(&request, machine_id).await; + let requests = parsed.to_requests(); + info!(kind = M::KIND, execution_id = %execution_id, schedule_id = %schedule_id, scripts = requests.len(), "Execution request received"); - info!( - kind = M::KIND, - execution_id = %execution_id, - schedule_id = %schedule_id, - exit_code = result.exit_code, - timed_out = result.timed_out, - execution_time_ms = result.execution_time_ms, - "Execution finished" - ); + let result_subject = format!("machine.{}.{}.result", machine_id, M::RESULT_KIND); - let result_subject = format!("machine.{}.{}.result", machine_id, M::KIND); - if let Err(e) = self - .nats_message_publisher - .publish(&result_subject, &result) - .await - { - error!(kind = M::KIND, execution_id = %execution_id, error = %e, "Failed to publish result"); + if M::DURABLE && self.result_store.enabled() { + self.handle_durable( + requests, + machine_id, + &result_subject, + &execution_id, + &schedule_id, + ) + .await; + } else { + for request in requests { + let script_id = request.script_id.unwrap_or("-").to_string(); + let result = self.execution_service.execute(&request, machine_id).await; + log_finished(&execution_id, &schedule_id, &script_id, &result); + self.publish_result(&result_subject, &result, &execution_id, &script_id) + .await; + } } Ok(()) } -} -async fn run_bounded( - stream: impl futures::Stream, - semaphore: Arc, - handler: F, -) where - T: Send + 'static, - F: Fn(T) -> Fut + Clone + Send + 'static, - Fut: std::future::Future + Send + 'static, -{ - tokio::pin!(stream); - while let Some(item) = stream.next().await { - let permit = match semaphore.clone().acquire_owned().await { - Ok(permit) => permit, - Err(_) => break, - }; - let handler = handler.clone(); - tokio::spawn(async move { - let _permit = permit; - handler(item).await; - }); + async fn publish_result( + &self, + subject: &str, + result: &RmmResult, + execution_id: &str, + script_id: &str, + ) { + if let Err(e) = self.nats_message_publisher.publish(subject, result).await { + error!(kind = M::KIND, execution_id = %execution_id, script_id = %script_id, error = %e, "Failed to publish result"); + } } -} -#[cfg(test)] -mod tests { - use super::*; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::time::{Duration as StdDuration, Instant}; + async fn handle_durable( + &self, + requests: Vec>, + machine_id: &str, + result_subject: &str, + execution_id: &str, + schedule_id: &str, + ) { + let now = now_secs(); + let records: Vec<(String, JournalRecord)> = requests + .iter() + .map(|req| { + let key = entry_key(req.execution_id, req.script_id); + let record = JournalRecord { + subject: result_subject.to_string(), + execution_id: req.execution_id.to_string(), + schedule_id: req.schedule_id.map(str::to_string), + machine_id: machine_id.to_string(), + script_id: req.script_id.map(str::to_string), + started: false, + created_at_secs: now, + }; + (key, record) + }) + .collect(); - async fn wait_for(counter: &AtomicUsize, target: usize) { - while counter.load(Ordering::SeqCst) < target { - tokio::time::sleep(StdDuration::from_millis(5)).await; + match self + .result_store + .journal_batch(execution_id.to_string(), records) + .await + { + Ok(true) => {} + Ok(false) => { + warn!(kind = M::KIND, execution_id = %execution_id, "Batch already in flight, skipping redelivery"); + return; + } + Err(e) => { + error!(kind = M::KIND, execution_id = %execution_id, error = %e, "Failed to persist batch, falling back to best-effort publish"); + self.publish_directly( + requests, + machine_id, + result_subject, + execution_id, + schedule_id, + ) + .await; + return; + } } - } + self.flush_notify.notify_one(); - #[tokio::test] - async fn runs_up_to_k_in_parallel() { - let k = 4; - let semaphore = Arc::new(Semaphore::new(k)); - let active = Arc::new(AtomicUsize::new(0)); - let max_active = Arc::new(AtomicUsize::new(0)); - let done = Arc::new(AtomicUsize::new(0)); - - let (a, m, d) = (active.clone(), max_active.clone(), done.clone()); - let start = Instant::now(); - run_bounded(futures::stream::iter(0..k), semaphore, move |_| { - let (a, m, d) = (a.clone(), m.clone(), d.clone()); - async move { - let now = a.fetch_add(1, Ordering::SeqCst) + 1; - m.fetch_max(now, Ordering::SeqCst); - tokio::time::sleep(StdDuration::from_millis(200)).await; - a.fetch_sub(1, Ordering::SeqCst); - d.fetch_add(1, Ordering::SeqCst); + for request in requests { + let script_id = request.script_id.unwrap_or("-").to_string(); + let key = entry_key(request.execution_id, request.script_id); + if let Err(e) = self.result_store.journal_mark_started(key.clone()).await { + warn!(kind = M::KIND, execution_id = %execution_id, script_id = %script_id, error = %e, "Failed to mark script started"); } - }) - .await; - wait_for(&done, k).await; + let result = self.execution_service.execute(&request, machine_id).await; + log_finished(execution_id, schedule_id, &script_id, &result); - assert_eq!( - max_active.load(Ordering::SeqCst), - k, - "all K should run at once" - ); - assert!( - start.elapsed() < StdDuration::from_millis(600), - "K parallel sleeps should take ~one duration, took {:?}", - start.elapsed() - ); + let bytes = ResultStore::encode_result(&result); + if let Err(e) = self + .result_store + .complete(key.clone(), result_subject.to_string(), bytes) + .await + { + error!(kind = M::KIND, execution_id = %execution_id, script_id = %script_id, error = %e, "Failed to persist result, publishing best-effort"); + match self + .nats_message_publisher + .publish(result_subject, &result) + .await + { + Ok(()) => { + if let Err(re) = self.result_store.journal_remove(key).await { + warn!(kind = M::KIND, execution_id = %execution_id, script_id = %script_id, error = %re, "Delivered best-effort but failed to clear journal entry"); + } + } + Err(pe) => { + error!(kind = M::KIND, execution_id = %execution_id, script_id = %script_id, error = %pe, "Best-effort publish also failed"); + } + } + } else { + self.flush_notify.notify_one(); + } + } } - #[tokio::test] - async fn concurrency_never_exceeds_k() { - let k = 2; - let n = 10; - let semaphore = Arc::new(Semaphore::new(k)); - let active = Arc::new(AtomicUsize::new(0)); - let max_active = Arc::new(AtomicUsize::new(0)); - let done = Arc::new(AtomicUsize::new(0)); - - let (a, m, d) = (active.clone(), max_active.clone(), done.clone()); - run_bounded(futures::stream::iter(0..n), semaphore, move |_| { - let (a, m, d) = (a.clone(), m.clone(), d.clone()); - async move { - let now = a.fetch_add(1, Ordering::SeqCst) + 1; - m.fetch_max(now, Ordering::SeqCst); - tokio::time::sleep(StdDuration::from_millis(30)).await; - a.fetch_sub(1, Ordering::SeqCst); - d.fetch_add(1, Ordering::SeqCst); + async fn publish_directly( + &self, + requests: Vec>, + machine_id: &str, + result_subject: &str, + execution_id: &str, + schedule_id: &str, + ) { + for request in requests { + let script_id = request.script_id.unwrap_or("-").to_string(); + let result = self.execution_service.execute(&request, machine_id).await; + log_finished(execution_id, schedule_id, &script_id, &result); + if let Err(e) = self + .nats_message_publisher + .publish(result_subject, &result) + .await + { + error!(kind = M::KIND, execution_id = %execution_id, script_id = %script_id, error = %e, "Failed to publish result"); } - }) - .await; - wait_for(&done, n).await; - - assert!( - max_active.load(Ordering::SeqCst) <= k, - "observed {} concurrent, cap is {}", - max_active.load(Ordering::SeqCst), - k - ); - assert_eq!(done.load(Ordering::SeqCst), n, "every item must complete"); + } } +} - #[tokio::test] - async fn permit_released_on_panic() { - let semaphore = Arc::new(Semaphore::new(1)); - let done = Arc::new(AtomicUsize::new(0)); - - let d = done.clone(); - run_bounded(futures::stream::iter(0..3usize), semaphore, move |i| { - let d = d.clone(); - async move { - if i == 0 { - panic!("intentional panic in first task"); - } - d.fetch_add(1, Ordering::SeqCst); - } - }) - .await; - wait_for(&done, 2).await; +fn log_finished(execution_id: &str, schedule_id: &str, script_id: &str, result: &RmmResult) { + info!( + execution_id = %execution_id, + schedule_id = %schedule_id, + script_id = %script_id, + exit_code = result.exit_code, + timed_out = result.timed_out, + execution_time_ms = result.execution_time_ms, + "Execution finished" + ); +} - assert_eq!( - done.load(Ordering::SeqCst), - 2, - "a panicking task must release its permit so the rest still run" - ); +async fn run_unbounded(stream: impl futures::Stream, handler: F) +where + T: Send + 'static, + F: Fn(T) -> Fut + Clone + Send + 'static, + Fut: std::future::Future + Send + 'static, +{ + tokio::pin!(stream); + while let Some(item) = stream.next().await { + tokio::spawn(handler.clone()(item)); } } + +#[cfg(test)] +#[path = "execution_listener_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/listener/execution_listener_tests.rs b/clients/openframe-client/src/listener/execution_listener_tests.rs new file mode 100644 index 000000000..06f958d81 --- /dev/null +++ b/clients/openframe-client/src/listener/execution_listener_tests.rs @@ -0,0 +1,122 @@ +use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration as StdDuration; + +async fn wait_for(counter: &AtomicUsize, target: usize) { + while counter.load(Ordering::SeqCst) < target { + tokio::time::sleep(StdDuration::from_millis(5)).await; + } +} + +#[tokio::test] +async fn runs_all_concurrently_without_a_cap() { + let n = 64; + let active = Arc::new(AtomicUsize::new(0)); + let done = Arc::new(AtomicUsize::new(0)); + + let (a, d) = (active.clone(), done.clone()); + run_unbounded(futures::stream::iter(0..n), move |_| { + let (a, d) = (a.clone(), d.clone()); + async move { + a.fetch_add(1, Ordering::SeqCst); + while a.load(Ordering::SeqCst) < n { + tokio::task::yield_now().await; + } + d.fetch_add(1, Ordering::SeqCst); + } + }) + .await; + + let drained = tokio::time::timeout(StdDuration::from_secs(5), wait_for(&done, n)).await; + assert!( + drained.is_ok(), + "all {n} tasks must be concurrently active to pass the barrier; a cap would deadlock" + ); + assert_eq!(active.load(Ordering::SeqCst), n); +} + +// A task that never finishes must not hold up any other dispatched message +// (the old bounded loop with a small cap would have blocked here). +#[tokio::test] +async fn a_stuck_task_does_not_block_the_rest() { + let n = 10; + let gate = Arc::new(Notify::new()); + let started_stuck = Arc::new(AtomicUsize::new(0)); + let done = Arc::new(AtomicUsize::new(0)); + + let (g, s, d) = (gate.clone(), started_stuck.clone(), done.clone()); + run_unbounded(futures::stream::iter(0..n), move |i| { + let (g, s, d) = (g.clone(), s.clone(), d.clone()); + async move { + if i == 0 { + s.fetch_add(1, Ordering::SeqCst); + g.notified().await; + } + d.fetch_add(1, Ordering::SeqCst); + } + }) + .await; + + // The 9 fast tasks finish while task 0 is parked on the gate. + let fast = tokio::time::timeout(StdDuration::from_secs(5), wait_for(&done, n - 1)).await; + assert!( + fast.is_ok(), + "fast tasks must not wait behind the stuck one" + ); + assert_eq!( + started_stuck.load(Ordering::SeqCst), + 1, + "the stuck task did start" + ); + assert_eq!( + done.load(Ordering::SeqCst), + n - 1, + "the stuck task is still parked" + ); + + // Releasing the gate lets the last task complete. + gate.notify_one(); + let all = tokio::time::timeout(StdDuration::from_secs(5), wait_for(&done, n)).await; + assert!(all.is_ok(), "the released task completes"); +} + +// A panic in one task is isolated by the runtime and must not stop the loop +// from dispatching or the sibling tasks from running. +#[tokio::test] +async fn a_panicking_task_does_not_stop_the_rest() { + let done = Arc::new(AtomicUsize::new(0)); + + let d = done.clone(); + run_unbounded(futures::stream::iter(0..3usize), move |i| { + let d = d.clone(); + async move { + if i == 1 { + panic!("intentional panic in one task"); + } + d.fetch_add(1, Ordering::SeqCst); + } + }) + .await; + + let ok = tokio::time::timeout(StdDuration::from_secs(5), wait_for(&done, 2)).await; + assert!(ok.is_ok(), "the two non-panicking tasks must still run"); +} + +// A large fan-out completes without leaking or wedging. +#[tokio::test] +async fn high_fan_out_all_complete() { + let n = 1000; + let done = Arc::new(AtomicUsize::new(0)); + + let d = done.clone(); + run_unbounded(futures::stream::iter(0..n), move |_| { + let d = d.clone(); + async move { + d.fetch_add(1, Ordering::SeqCst); + } + }) + .await; + + let ok = tokio::time::timeout(StdDuration::from_secs(10), wait_for(&done, n)).await; + assert!(ok.is_ok(), "all {n} dispatched tasks must complete"); +} diff --git a/clients/openframe-client/src/logging/log_parser.rs b/clients/openframe-client/src/logging/log_parser.rs index 210555d94..d23030f2e 100644 --- a/clients/openframe-client/src/logging/log_parser.rs +++ b/clients/openframe-client/src/logging/log_parser.rs @@ -145,174 +145,5 @@ impl LogDeduplicator for Vec { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_log_line() { - let line = "2026-03-18T15:44:15.099267Z INFO openframe::services::nats_connection_manager: Reauthentication successful"; - let entry = parse_log_line(line).unwrap(); - - assert_eq!(entry.ts, "2026-03-18T15:44:15.099267Z"); - assert_eq!(entry.level, "INFO"); - assert_eq!( - entry.msg, - "openframe::services::nats_connection_manager: Reauthentication successful" - ); - } - - #[test] - fn test_parse_log_line_warn() { - let line = "2026-03-17T13:20:19.591487Z WARN openframe::services::github_download_service: Download failed"; - let entry = parse_log_line(line).unwrap(); - - assert_eq!(entry.ts, "2026-03-17T13:20:19.591487Z"); - assert_eq!(entry.level, "WARN"); - assert_eq!( - entry.msg, - "openframe::services::github_download_service: Download failed" - ); - } - - #[test] - fn test_deduplicate_tool_logs_only() { - let logs = vec![ - // Client log - should NOT be deduplicated - LogEntry { - ts: "2026-03-17T13:20:19.000Z".into(), - level: "INFO".into(), - msg: "openframe::services::tool_run_manager: Starting".into(), - count: None, - }, - // Tool log - should be deduplicated - LogEntry { - ts: "2026-03-17T13:20:20.000Z".into(), - level: "INFO".into(), - msg: "[tool] Token refresh".into(), - count: None, - }, - // Same client log - should NOT be deduplicated (appears twice) - LogEntry { - ts: "2026-03-17T13:20:21.000Z".into(), - level: "INFO".into(), - msg: "openframe::services::tool_run_manager: Starting".into(), - count: None, - }, - // Same tool log - should be deduplicated with previous - LogEntry { - ts: "2026-03-17T13:20:22.000Z".into(), - level: "INFO".into(), - msg: "[tool] Token refresh".into(), - count: None, - }, - // Another tool log - LogEntry { - ts: "2026-03-17T13:20:23.000Z".into(), - level: "INFO".into(), - msg: "[tool] Done".into(), - count: None, - }, - ]; - - let deduped = logs.deduplicate(); - - assert_eq!(deduped.len(), 4); - - // First: client log (not deduplicated) - assert_eq!( - deduped[0].msg, - "openframe::services::tool_run_manager: Starting" - ); - assert_eq!(deduped[0].count, None); - - // Second: tool log (first occurrence) - assert_eq!(deduped[1].msg, "[tool] Token refresh"); - assert_eq!(deduped[1].ts, "2026-03-17T13:20:20.000Z"); - assert_eq!(deduped[1].count, Some(2)); // deduplicated: 2 occurrences - - // Third: same client log again (not deduplicated) - assert_eq!( - deduped[2].msg, - "openframe::services::tool_run_manager: Starting" - ); - assert_eq!(deduped[2].count, None); - - // Fourth: another tool log - assert_eq!(deduped[3].msg, "[tool] Done"); - assert_eq!(deduped[3].count, None); - } - - #[test] - fn test_parse_logrus_format() { - let line = - r#"time="2026-03-24T13:24:04Z" level=info msg="Agent: /Library/Application Support""#; - let entry = parse_log_line(line).unwrap(); - - assert_eq!(entry.ts, "2026-03-24T13:24:04Z"); - assert_eq!(entry.level, "INFO"); - assert_eq!(entry.msg, "[tool] Agent: /Library/Application Support"); - } - - #[test] - fn test_parse_logrus_with_stdout_prefix() { - let line = - r#"stdout: time="2026-03-24T13:24:04Z" level=info msg="Token refresh job started""#; - let entry = parse_log_line(line).unwrap(); - - assert_eq!(entry.ts, "2026-03-24T13:24:04Z"); - assert_eq!(entry.level, "INFO"); - assert_eq!(entry.msg, "[tool] Token refresh job started"); - } - - #[test] - fn test_parse_tool_level_format() { - let line = "2026-04-06T14:15:10.488Z TOOL Openframe JWT: token123"; - let entry = parse_log_line(line).unwrap(); - - assert_eq!(entry.ts, "2026-04-06T14:15:10.488Z"); - assert_eq!(entry.level, "TOOL"); - assert_eq!(entry.msg, "Openframe JWT: token123"); - } - - #[test] - fn test_tool_level_is_not_client_log() { - let entry = LogEntry { - ts: "2026-04-06T14:15:10.488Z".into(), - level: "TOOL".into(), - msg: "Connection established".into(), - count: None, - }; - - assert!(!is_client_log(&entry)); - } - - #[test] - fn test_deduplicate_tool_level_logs() { - let logs = vec![ - LogEntry { - ts: "2026-04-06T14:15:10.000Z".into(), - level: "TOOL".into(), - msg: "Connection FAILED: Network timeout".into(), - count: None, - }, - LogEntry { - ts: "2026-04-06T14:15:11.000Z".into(), - level: "TOOL".into(), - msg: "Connection FAILED: Network timeout".into(), - count: None, - }, - LogEntry { - ts: "2026-04-06T14:15:12.000Z".into(), - level: "TOOL".into(), - msg: "Connection FAILED: Network timeout".into(), - count: None, - }, - ]; - - let deduped = logs.deduplicate(); - - assert_eq!(deduped.len(), 1); - assert_eq!(deduped[0].msg, "Connection FAILED: Network timeout"); - assert_eq!(deduped[0].count, Some(3)); - } -} +#[path = "log_parser_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/logging/log_parser_tests.rs b/clients/openframe-client/src/logging/log_parser_tests.rs new file mode 100644 index 000000000..76a3dea7d --- /dev/null +++ b/clients/openframe-client/src/logging/log_parser_tests.rs @@ -0,0 +1,168 @@ +use super::*; + +#[test] +fn test_parse_log_line() { + let line = "2026-03-18T15:44:15.099267Z INFO openframe::services::nats_connection_manager: Reauthentication successful"; + let entry = parse_log_line(line).unwrap(); + + assert_eq!(entry.ts, "2026-03-18T15:44:15.099267Z"); + assert_eq!(entry.level, "INFO"); + assert_eq!( + entry.msg, + "openframe::services::nats_connection_manager: Reauthentication successful" + ); +} + +#[test] +fn test_parse_log_line_warn() { + let line = "2026-03-17T13:20:19.591487Z WARN openframe::services::github_download_service: Download failed"; + let entry = parse_log_line(line).unwrap(); + + assert_eq!(entry.ts, "2026-03-17T13:20:19.591487Z"); + assert_eq!(entry.level, "WARN"); + assert_eq!( + entry.msg, + "openframe::services::github_download_service: Download failed" + ); +} + +#[test] +fn test_deduplicate_tool_logs_only() { + let logs = vec![ + // Client log - should NOT be deduplicated + LogEntry { + ts: "2026-03-17T13:20:19.000Z".into(), + level: "INFO".into(), + msg: "openframe::services::tool_run_manager: Starting".into(), + count: None, + }, + // Tool log - should be deduplicated + LogEntry { + ts: "2026-03-17T13:20:20.000Z".into(), + level: "INFO".into(), + msg: "[tool] Token refresh".into(), + count: None, + }, + // Same client log - should NOT be deduplicated (appears twice) + LogEntry { + ts: "2026-03-17T13:20:21.000Z".into(), + level: "INFO".into(), + msg: "openframe::services::tool_run_manager: Starting".into(), + count: None, + }, + // Same tool log - should be deduplicated with previous + LogEntry { + ts: "2026-03-17T13:20:22.000Z".into(), + level: "INFO".into(), + msg: "[tool] Token refresh".into(), + count: None, + }, + // Another tool log + LogEntry { + ts: "2026-03-17T13:20:23.000Z".into(), + level: "INFO".into(), + msg: "[tool] Done".into(), + count: None, + }, + ]; + + let deduped = logs.deduplicate(); + + assert_eq!(deduped.len(), 4); + + // First: client log (not deduplicated) + assert_eq!( + deduped[0].msg, + "openframe::services::tool_run_manager: Starting" + ); + assert_eq!(deduped[0].count, None); + + // Second: tool log (first occurrence) + assert_eq!(deduped[1].msg, "[tool] Token refresh"); + assert_eq!(deduped[1].ts, "2026-03-17T13:20:20.000Z"); + assert_eq!(deduped[1].count, Some(2)); // deduplicated: 2 occurrences + + // Third: same client log again (not deduplicated) + assert_eq!( + deduped[2].msg, + "openframe::services::tool_run_manager: Starting" + ); + assert_eq!(deduped[2].count, None); + + // Fourth: another tool log + assert_eq!(deduped[3].msg, "[tool] Done"); + assert_eq!(deduped[3].count, None); +} + +#[test] +fn test_parse_logrus_format() { + let line = + r#"time="2026-03-24T13:24:04Z" level=info msg="Agent: /Library/Application Support""#; + let entry = parse_log_line(line).unwrap(); + + assert_eq!(entry.ts, "2026-03-24T13:24:04Z"); + assert_eq!(entry.level, "INFO"); + assert_eq!(entry.msg, "[tool] Agent: /Library/Application Support"); +} + +#[test] +fn test_parse_logrus_with_stdout_prefix() { + let line = r#"stdout: time="2026-03-24T13:24:04Z" level=info msg="Token refresh job started""#; + let entry = parse_log_line(line).unwrap(); + + assert_eq!(entry.ts, "2026-03-24T13:24:04Z"); + assert_eq!(entry.level, "INFO"); + assert_eq!(entry.msg, "[tool] Token refresh job started"); +} + +#[test] +fn test_parse_tool_level_format() { + let line = "2026-04-06T14:15:10.488Z TOOL Openframe JWT: token123"; + let entry = parse_log_line(line).unwrap(); + + assert_eq!(entry.ts, "2026-04-06T14:15:10.488Z"); + assert_eq!(entry.level, "TOOL"); + assert_eq!(entry.msg, "Openframe JWT: token123"); +} + +#[test] +fn test_tool_level_is_not_client_log() { + let entry = LogEntry { + ts: "2026-04-06T14:15:10.488Z".into(), + level: "TOOL".into(), + msg: "Connection established".into(), + count: None, + }; + + assert!(!is_client_log(&entry)); +} + +#[test] +fn test_deduplicate_tool_level_logs() { + let logs = vec![ + LogEntry { + ts: "2026-04-06T14:15:10.000Z".into(), + level: "TOOL".into(), + msg: "Connection FAILED: Network timeout".into(), + count: None, + }, + LogEntry { + ts: "2026-04-06T14:15:11.000Z".into(), + level: "TOOL".into(), + msg: "Connection FAILED: Network timeout".into(), + count: None, + }, + LogEntry { + ts: "2026-04-06T14:15:12.000Z".into(), + level: "TOOL".into(), + msg: "Connection FAILED: Network timeout".into(), + count: None, + }, + ]; + + let deduped = logs.deduplicate(); + + assert_eq!(deduped.len(), 1); + assert_eq!(deduped[0].msg, "Connection FAILED: Network timeout"); + assert_eq!(deduped[0].count, Some(3)); +} diff --git a/clients/openframe-client/src/logging/log_source.rs b/clients/openframe-client/src/logging/log_source.rs index 91be9fe7f..6f129d5d1 100644 --- a/clients/openframe-client/src/logging/log_source.rs +++ b/clients/openframe-client/src/logging/log_source.rs @@ -200,112 +200,5 @@ impl Default for LogSourceRegistry { } #[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - use tempfile::TempDir; - - #[test] - fn test_file_log_source_reads_and_commits() { - let tmp = TempDir::new().unwrap(); - let log_path = tmp.path().join("test.log"); - let offset_path = tmp.path().join("offset"); - - let mut file = File::create(&log_path).unwrap(); - writeln!( - file, - "2026-04-06T14:15:10.488Z INFO openframe::test: message 1" - ) - .unwrap(); - writeln!( - file, - "2026-04-06T14:15:11.488Z WARN openframe::test: message 2" - ) - .unwrap(); - - let mut source = - FileLogSource::new(LogSourceKind::Openframe, log_path, offset_path.clone()); - let entries = source.read(10).unwrap(); - - assert_eq!(entries.len(), 2); - source.commit(); - assert!(offset_path.exists()); - } - - #[test] - fn test_file_log_source_returns_error_on_missing_file() { - let tmp = TempDir::new().unwrap(); - let log_path = tmp.path().join("nonexistent.log"); - let offset_path = tmp.path().join("offset"); - - let mut source = FileLogSource::new(LogSourceKind::Meshcentral, log_path, offset_path); - assert!(source.read(10).is_err()); - } - - #[test] - fn test_file_log_source_rollback() { - let tmp = TempDir::new().unwrap(); - let log_path = tmp.path().join("test.log"); - let offset_path = tmp.path().join("offset"); - - let mut file = File::create(&log_path).unwrap(); - writeln!(file, "2026-04-06T14:15:10.488Z INFO openframe::test: msg").unwrap(); - - let mut source = FileLogSource::new(LogSourceKind::Openframe, log_path, offset_path); - - let entries1 = source.read(10).unwrap(); - source.rollback(); - let entries2 = source.read(10).unwrap(); - - assert_eq!(entries1[0].msg, entries2[0].msg); - } - - #[test] - fn test_registry_distributes_reads_across_sources() { - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; - - struct MockLogSource { - name: String, - logs_available: Arc, - } - - impl MockLogSource { - fn new(name: &str, available: usize) -> Self { - Self { - name: name.to_string(), - logs_available: Arc::new(AtomicUsize::new(available)), - } - } - } - - impl LogSource for MockLogSource { - fn name(&self) -> &str { - &self.name - } - fn read(&mut self, max_count: usize) -> Result> { - let available = self.logs_available.load(Ordering::SeqCst); - let to_read = max_count.min(available); - self.logs_available.fetch_sub(to_read, Ordering::SeqCst); - - Ok((0..to_read) - .map(|i| LogEntry { - ts: format!("2026-04-06T14:15:{:02}.000Z", i), - level: "INFO".to_string(), - msg: format!("{}::log_{}", self.name, i), - count: None, - }) - .collect()) - } - fn commit(&mut self) {} - fn rollback(&mut self) {} - } - - let mut registry = LogSourceRegistry::new(); - registry.register(Box::new(MockLogSource::new("source1", 100))); - registry.register(Box::new(MockLogSource::new("source2", 100))); - - let logs = registry.read_all(50); - assert!(logs.len() <= 50); - } -} +#[path = "log_source_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/logging/log_source_tests.rs b/clients/openframe-client/src/logging/log_source_tests.rs new file mode 100644 index 000000000..29e369457 --- /dev/null +++ b/clients/openframe-client/src/logging/log_source_tests.rs @@ -0,0 +1,106 @@ +use super::*; +use std::io::Write; +use tempfile::TempDir; + +#[test] +fn test_file_log_source_reads_and_commits() { + let tmp = TempDir::new().unwrap(); + let log_path = tmp.path().join("test.log"); + let offset_path = tmp.path().join("offset"); + + let mut file = File::create(&log_path).unwrap(); + writeln!( + file, + "2026-04-06T14:15:10.488Z INFO openframe::test: message 1" + ) + .unwrap(); + writeln!( + file, + "2026-04-06T14:15:11.488Z WARN openframe::test: message 2" + ) + .unwrap(); + + let mut source = FileLogSource::new(LogSourceKind::Openframe, log_path, offset_path.clone()); + let entries = source.read(10).unwrap(); + + assert_eq!(entries.len(), 2); + source.commit(); + assert!(offset_path.exists()); +} + +#[test] +fn test_file_log_source_returns_error_on_missing_file() { + let tmp = TempDir::new().unwrap(); + let log_path = tmp.path().join("nonexistent.log"); + let offset_path = tmp.path().join("offset"); + + let mut source = FileLogSource::new(LogSourceKind::Meshcentral, log_path, offset_path); + assert!(source.read(10).is_err()); +} + +#[test] +fn test_file_log_source_rollback() { + let tmp = TempDir::new().unwrap(); + let log_path = tmp.path().join("test.log"); + let offset_path = tmp.path().join("offset"); + + let mut file = File::create(&log_path).unwrap(); + writeln!(file, "2026-04-06T14:15:10.488Z INFO openframe::test: msg").unwrap(); + + let mut source = FileLogSource::new(LogSourceKind::Openframe, log_path, offset_path); + + let entries1 = source.read(10).unwrap(); + source.rollback(); + let entries2 = source.read(10).unwrap(); + + assert_eq!(entries1[0].msg, entries2[0].msg); +} + +#[test] +fn test_registry_distributes_reads_across_sources() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + struct MockLogSource { + name: String, + logs_available: Arc, + } + + impl MockLogSource { + fn new(name: &str, available: usize) -> Self { + Self { + name: name.to_string(), + logs_available: Arc::new(AtomicUsize::new(available)), + } + } + } + + impl LogSource for MockLogSource { + fn name(&self) -> &str { + &self.name + } + fn read(&mut self, max_count: usize) -> Result> { + let available = self.logs_available.load(Ordering::SeqCst); + let to_read = max_count.min(available); + self.logs_available.fetch_sub(to_read, Ordering::SeqCst); + + Ok((0..to_read) + .map(|i| LogEntry { + ts: format!("2026-04-06T14:15:{:02}.000Z", i), + level: "INFO".to_string(), + msg: format!("{}::log_{}", self.name, i), + count: None, + }) + .collect()) + } + fn commit(&mut self) {} + fn rollback(&mut self) {} + } + + let mut registry = LogSourceRegistry::new(); + registry.register(Box::new(MockLogSource::new("source1", 100))); + registry.register(Box::new(MockLogSource::new("source2", 100))); + + let logs = registry.read_all(50); + assert!(logs.len() <= 50); +} diff --git a/clients/openframe-client/src/logging/mod.rs b/clients/openframe-client/src/logging/mod.rs index 58036d646..472d93bf8 100644 --- a/clients/openframe-client/src/logging/mod.rs +++ b/clients/openframe-client/src/logging/mod.rs @@ -417,48 +417,5 @@ pub fn get_log_file_path(dir_manager: &DirectoryManager) -> PathBuf { } #[cfg(test)] -mod tests { - use super::*; - use std::io::Read; - use tempfile::tempdir; - use tracing::{debug, error, info, trace, warn}; - - #[test] - fn test_structured_logging() -> std::io::Result<()> { - let temp_dir = tempdir()?; - let log_file = temp_dir.path().join("test.log"); - - let json_layer = JsonLayer::new(log_file.clone())?; - let subscriber = Registry::default().with(json_layer); - - tracing::subscriber::set_global_default(subscriber).expect("Failed to set subscriber"); - - // Log messages with different levels and context - error!(error = "test error", "Error message"); - warn!(user = "test_user", "Warning message"); - info!(request_id = 123, "Info message"); - debug!(status = "pending", "Debug message"); - trace!(correlation_id = "abc", "Trace message"); - - // Read and verify log file contents - let mut file = std::fs::File::open(log_file)?; - let mut contents = String::new(); - file.read_to_string(&mut contents)?; - - // Verify each log level appears in the file - assert!(contents.contains(r#""level":"ERROR"#)); - assert!(contents.contains(r#""level":"WARN"#)); - assert!(contents.contains(r#""level":"INFO"#)); - assert!(contents.contains(r#""level":"DEBUG"#)); - assert!(contents.contains(r#""level":"TRACE"#)); - - // Verify custom fields are included - assert!(contents.contains(r#""error":"test error"#)); - assert!(contents.contains(r#""user":"test_user"#)); - assert!(contents.contains(r#""request_id":"123"#)); - assert!(contents.contains(r#""status":"pending"#)); - assert!(contents.contains(r#""correlation_id":"abc"#)); - - Ok(()) - } -} +#[path = "mod_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/logging/mod_tests.rs b/clients/openframe-client/src/logging/mod_tests.rs new file mode 100644 index 000000000..60927aa42 --- /dev/null +++ b/clients/openframe-client/src/logging/mod_tests.rs @@ -0,0 +1,43 @@ +use super::*; +use std::io::Read; +use tempfile::tempdir; +use tracing::{debug, error, info, trace, warn}; + +#[test] +fn test_structured_logging() -> std::io::Result<()> { + let temp_dir = tempdir()?; + let log_file = temp_dir.path().join("test.log"); + + let json_layer = JsonLayer::new(log_file.clone())?; + let subscriber = Registry::default().with(json_layer); + + tracing::subscriber::set_global_default(subscriber).expect("Failed to set subscriber"); + + // Log messages with different levels and context + error!(error = "test error", "Error message"); + warn!(user = "test_user", "Warning message"); + info!(request_id = 123, "Info message"); + debug!(status = "pending", "Debug message"); + trace!(correlation_id = "abc", "Trace message"); + + // Read and verify log file contents + let mut file = std::fs::File::open(log_file)?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + + // Verify each log level appears in the file + assert!(contents.contains(r#""level":"ERROR"#)); + assert!(contents.contains(r#""level":"WARN"#)); + assert!(contents.contains(r#""level":"INFO"#)); + assert!(contents.contains(r#""level":"DEBUG"#)); + assert!(contents.contains(r#""level":"TRACE"#)); + + // Verify custom fields are included + assert!(contents.contains(r#""error":"test error"#)); + assert!(contents.contains(r#""user":"test_user"#)); + assert!(contents.contains(r#""request_id":"123"#)); + assert!(contents.contains(r#""status":"pending"#)); + assert!(contents.contains(r#""correlation_id":"abc"#)); + + Ok(()) +} diff --git a/clients/openframe-client/src/logging/platform.rs b/clients/openframe-client/src/logging/platform.rs index 26bc7fae3..49a39cec9 100644 --- a/clients/openframe-client/src/logging/platform.rs +++ b/clients/openframe-client/src/logging/platform.rs @@ -88,20 +88,6 @@ fn can_write_to_directory(path: &PathBuf) -> bool { } #[cfg(test)] -mod tests { - use super::*; +#[path = "platform_tests.rs"] +mod tests; - #[test] - fn test_get_log_directory() { - let log_dir = get_log_directory(); - - #[cfg(target_os = "windows")] - assert!(log_dir.to_string_lossy().contains("OpenFrame\\logs")); - - #[cfg(target_os = "macos")] - assert_eq!(log_dir.to_string_lossy(), "/Library/Logs/OpenFrame"); - - #[cfg(target_os = "linux")] - assert_eq!(log_dir.to_string_lossy(), "/var/log/openframe"); - } -} diff --git a/clients/openframe-client/src/logging/platform_tests.rs b/clients/openframe-client/src/logging/platform_tests.rs new file mode 100644 index 000000000..5d7e52ba4 --- /dev/null +++ b/clients/openframe-client/src/logging/platform_tests.rs @@ -0,0 +1,15 @@ +use super::*; + +#[test] +fn test_get_log_directory() { + let log_dir = get_log_directory(); + + #[cfg(target_os = "windows")] + assert!(log_dir.to_string_lossy().contains("OpenFrame\\logs")); + + #[cfg(target_os = "macos")] + assert_eq!(log_dir.to_string_lossy(), "/Library/Logs/OpenFrame"); + + #[cfg(target_os = "linux")] + assert_eq!(log_dir.to_string_lossy(), "/var/log/openframe"); +} diff --git a/clients/openframe-client/src/models/device_tag.rs b/clients/openframe-client/src/models/device_tag.rs index b72670837..4a7942871 100644 --- a/clients/openframe-client/src/models/device_tag.rs +++ b/clients/openframe-client/src/models/device_tag.rs @@ -34,77 +34,5 @@ impl DeviceTag { } #[cfg(test)] -mod tests { - use super::*; - #[test] - fn test_parse_multiple_values_same_key() { - let raw = vec![ - "site=CHICAGO".to_string(), - "site=NEW YORK".to_string(), - "site=LA".to_string(), - ]; - let tags = DeviceTag::parse_from_cli(raw); - - assert_eq!(tags.len(), 1); - assert_eq!(tags[0].key, "site"); - assert!(tags[0].values.contains(&"CHICAGO".to_string())); - assert!(tags[0].values.contains(&"NEW YORK".to_string())); - assert!(tags[0].values.contains(&"LA".to_string())); - } - - #[test] - fn test_parse_multiple_keys() { - let raw = vec!["site=CHICAGO".to_string(), "env=production".to_string()]; - let tags = DeviceTag::parse_from_cli(raw); - - assert_eq!(tags.len(), 2); - } - - #[test] - fn test_parse_empty() { - let raw: Vec = vec![]; - let tags = DeviceTag::parse_from_cli(raw); - - assert!(tags.is_empty()); - } - - #[test] - fn test_parse_invalid_format_ignored() { - let raw = vec!["site=CHICAGO".to_string(), "invalid_no_equals".to_string()]; - let tags = DeviceTag::parse_from_cli(raw); - - assert_eq!(tags.len(), 1); - assert_eq!(tags[0].key, "site"); - } - - #[test] - fn test_parse_empty_value_skipped() { - let raw = vec![ - "site=CHICAGO".to_string(), - "empty=".to_string(), - "env=production".to_string(), - ]; - let tags = DeviceTag::parse_from_cli(raw); - - assert_eq!(tags.len(), 2); - assert!(tags.iter().all(|t| t.key != "empty")); - } - - #[test] - fn test_parse_empty_key_skipped() { - let raw = vec!["site=CHICAGO".to_string(), "=nokey".to_string()]; - let tags = DeviceTag::parse_from_cli(raw); - - assert_eq!(tags.len(), 1); - assert_eq!(tags[0].key, "site"); - } - - #[test] - fn test_parse_whitespace_only_value_skipped() { - let raw = vec!["site=CHICAGO".to_string(), "empty= ".to_string()]; - let tags = DeviceTag::parse_from_cli(raw); - - assert_eq!(tags.len(), 1); - assert_eq!(tags[0].key, "site"); - } -} +#[path = "device_tag_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/models/device_tag_tests.rs b/clients/openframe-client/src/models/device_tag_tests.rs new file mode 100644 index 000000000..4025fb2eb --- /dev/null +++ b/clients/openframe-client/src/models/device_tag_tests.rs @@ -0,0 +1,72 @@ +use super::*; +#[test] +fn test_parse_multiple_values_same_key() { + let raw = vec![ + "site=CHICAGO".to_string(), + "site=NEW YORK".to_string(), + "site=LA".to_string(), + ]; + let tags = DeviceTag::parse_from_cli(raw); + + assert_eq!(tags.len(), 1); + assert_eq!(tags[0].key, "site"); + assert!(tags[0].values.contains(&"CHICAGO".to_string())); + assert!(tags[0].values.contains(&"NEW YORK".to_string())); + assert!(tags[0].values.contains(&"LA".to_string())); +} + +#[test] +fn test_parse_multiple_keys() { + let raw = vec!["site=CHICAGO".to_string(), "env=production".to_string()]; + let tags = DeviceTag::parse_from_cli(raw); + + assert_eq!(tags.len(), 2); +} + +#[test] +fn test_parse_empty() { + let raw: Vec = vec![]; + let tags = DeviceTag::parse_from_cli(raw); + + assert!(tags.is_empty()); +} + +#[test] +fn test_parse_invalid_format_ignored() { + let raw = vec!["site=CHICAGO".to_string(), "invalid_no_equals".to_string()]; + let tags = DeviceTag::parse_from_cli(raw); + + assert_eq!(tags.len(), 1); + assert_eq!(tags[0].key, "site"); +} + +#[test] +fn test_parse_empty_value_skipped() { + let raw = vec![ + "site=CHICAGO".to_string(), + "empty=".to_string(), + "env=production".to_string(), + ]; + let tags = DeviceTag::parse_from_cli(raw); + + assert_eq!(tags.len(), 2); + assert!(tags.iter().all(|t| t.key != "empty")); +} + +#[test] +fn test_parse_empty_key_skipped() { + let raw = vec!["site=CHICAGO".to_string(), "=nokey".to_string()]; + let tags = DeviceTag::parse_from_cli(raw); + + assert_eq!(tags.len(), 1); + assert_eq!(tags[0].key, "site"); +} + +#[test] +fn test_parse_whitespace_only_value_skipped() { + let raw = vec!["site=CHICAGO".to_string(), "empty= ".to_string()]; + let tags = DeviceTag::parse_from_cli(raw); + + assert_eq!(tags.len(), 1); + assert_eq!(tags[0].key, "site"); +} diff --git a/clients/openframe-client/src/models/execution.rs b/clients/openframe-client/src/models/execution.rs index 06e43e24c..bfcec7da9 100644 --- a/clients/openframe-client/src/models/execution.rs +++ b/clients/openframe-client/src/models/execution.rs @@ -41,6 +41,48 @@ pub struct ScriptEnvVar { pub secret: bool, } +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScriptSpec { + pub code: String, + pub shell: ScriptShell, + #[serde(default)] + pub privilege_level: Option, + #[serde(default)] + pub args: Option>, + #[serde(default)] + pub timeout_seconds: Option, + #[serde(default)] + pub env_vars: Option>, +} + +impl ScriptSpec { + fn to_request<'a>( + &'a self, + execution_id: &'a str, + script_id: Option<&'a str>, + schedule_id: Option<&'a str>, + ) -> ExecutionRequest<'a> { + ExecutionRequest { + execution_id, + code: &self.code, + shell: self.shell, + privilege: self.privilege_level.unwrap_or(PrivilegeLevel::Admin), + args: self.args.as_deref().unwrap_or(&[]), + timeout_secs: self.timeout_seconds.unwrap_or_else(default_timeout), + env_vars: self + .env_vars + .as_deref() + .unwrap_or(&[]) + .iter() + .map(|e| format!("{}={}", e.name, e.value)) + .collect(), + script_id, + schedule_id, + } + } +} + #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ScriptMessage { @@ -51,16 +93,30 @@ pub struct ScriptMessage { pub schedule_id: Option, #[serde(default)] pub script_id: Option, - pub code: String, - pub shell: ScriptShell, + #[serde(flatten)] + pub spec: ScriptSpec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScriptScheduleExecutionItem { + pub script_id: String, + #[serde(flatten)] + pub spec: ScriptSpec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScriptScheduleExecutionMessage { + pub execution_id: String, #[serde(default)] - pub privilege_level: Option, + pub schedule_id: Option, #[serde(default)] - pub args: Vec, - #[serde(default = "default_timeout")] - pub timeout_seconds: u64, + pub machine_id: Option, #[serde(default)] - pub env_vars: Vec, + pub initiated_by: Option, + #[serde(default)] + pub scripts: Vec, } #[derive(Debug, Clone, Deserialize)] @@ -110,10 +166,12 @@ pub struct ExecutionRequest<'a> { pub trait ExecutionMessage: Sized + Send { const KIND: &'static str; + const RESULT_KIND: &'static str = Self::KIND; + const DURABLE: bool = false; fn from_payload(payload: &str) -> Result; fn execution_id(&self) -> &str; - fn to_request(&self) -> ExecutionRequest<'_>; + fn to_requests(&self) -> Vec>; fn schedule_id(&self) -> Option<&str> { None @@ -131,8 +189,8 @@ impl ExecutionMessage for CommandMessage { &self.execution_id } - fn to_request(&self) -> ExecutionRequest<'_> { - ExecutionRequest { + fn to_requests(&self) -> Vec> { + vec![ExecutionRequest { execution_id: &self.execution_id, code: &self.code, shell: self.shell, @@ -142,7 +200,7 @@ impl ExecutionMessage for CommandMessage { env_vars: Vec::new(), script_id: None, schedule_id: None, - } + }] } } @@ -161,102 +219,46 @@ impl ExecutionMessage for ScriptMessage { self.schedule_id.as_deref() } - fn to_request(&self) -> ExecutionRequest<'_> { - ExecutionRequest { - execution_id: &self.execution_id, - code: &self.code, - shell: self.shell, - privilege: self.privilege_level.unwrap_or(PrivilegeLevel::Admin), - args: &self.args, - timeout_secs: self.timeout_seconds, - env_vars: self - .env_vars - .iter() - .map(|e| format!("{}={}", e.name, e.value)) - .collect(), - script_id: self.script_id.as_deref(), - schedule_id: self.schedule_id.as_deref(), - } + fn to_requests(&self) -> Vec> { + vec![self.spec.to_request( + &self.execution_id, + self.script_id.as_deref(), + self.schedule_id.as_deref(), + )] } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_command_message_camel_case() { - let m = CommandMessage::from_payload( - r#"{"executionId":"e","code":"echo hi","shell":"BASH","privilegeLevel":"ADMIN","timeout":30}"#, - ) - .unwrap(); - assert_eq!(m.execution_id, "e"); - assert!(matches!(m.shell, ScriptShell::Bash)); - assert_eq!(m.timeout, 30); - } - - #[test] - fn parses_script_message_with_env_and_args() { - let m = ScriptMessage::from_payload( - r#"{"executionId":"e","machineId":"mac","scheduleId":"sch-1","scriptId":"scr-1","code":"x","shell":"POWERSHELL","privilegeLevel":"USER","args":["-v"],"timeoutSeconds":60,"envVars":[{"name":"FOO","value":"bar"}]}"#, - ) - .unwrap(); - let req = m.to_request(); - assert_eq!(req.timeout_secs, 60); - assert_eq!(req.args, &["-v".to_string()]); - assert_eq!(req.env_vars, vec!["FOO=bar".to_string()]); - assert!(matches!(req.privilege, PrivilegeLevel::User)); - assert_eq!(req.script_id, Some("scr-1")); - assert_eq!(req.schedule_id, Some("sch-1")); - } +impl ExecutionMessage for ScriptScheduleExecutionMessage { + const KIND: &'static str = "script-schedule-execution"; + const RESULT_KIND: &'static str = ScriptMessage::KIND; + const DURABLE: bool = true; - #[test] - fn script_message_without_script_id_still_parses() { - let m = ScriptMessage::from_payload(r#"{"executionId":"e","code":"x","shell":"BASH"}"#) - .unwrap(); - assert!(m.script_id.is_none()); - assert!(m.to_request().script_id.is_none()); + fn from_payload(payload: &str) -> Result { + Ok(serde_json::from_str(payload)?) } - #[test] - fn missing_privilege_defaults_to_admin() { - let m = CommandMessage::from_payload(r#"{"executionId":"e","code":"x","shell":"CMD"}"#) - .unwrap(); - assert!(matches!(m.to_request().privilege, PrivilegeLevel::Admin)); - assert_eq!(m.timeout, 900); + fn execution_id(&self) -> &str { + &self.execution_id } - #[test] - fn result_serializes_snake_case() { - let r = RmmResult { - execution_id: "e".into(), - machine_id: "m".into(), - stdout: "o".into(), - stderr: String::new(), - exit_code: 0, - execution_time_ms: 1, - timed_out: false, - error: None, - script_id: Some("scr-1".into()), - schedule_id: None, - }; - let v = serde_json::to_value(&r).unwrap(); - assert!(v.get("execution_id").is_some()); - assert!(v.get("exit_code").is_some()); - assert!(v.get("execution_time_ms").is_some()); - assert!(v.get("timed_out").is_some()); - assert!(v.get("error").is_none(), "None error must be omitted"); - assert_eq!(v.get("script_id").and_then(|s| s.as_str()), Some("scr-1")); - assert!( - v.get("schedule_id").is_none(), - "None schedule_id must be omitted" - ); + fn schedule_id(&self) -> Option<&str> { + self.schedule_id.as_deref() } - #[test] - fn shell_maps_to_param() { - assert_eq!(ScriptShell::Powershell.as_param(), "powershell"); - assert_eq!(ScriptShell::Cmd.as_param(), "cmd"); - assert_eq!(ScriptShell::Shell.as_param(), "sh"); + fn to_requests(&self) -> Vec> { + self.scripts + .iter() + .map(|item| { + item.spec.to_request( + &self.execution_id, + Some(&item.script_id), + self.schedule_id.as_deref(), + ) + }) + .collect() } } + +#[cfg(test)] +#[path = "execution_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/models/execution_tests.rs b/clients/openframe-client/src/models/execution_tests.rs new file mode 100644 index 000000000..983e9c8b3 --- /dev/null +++ b/clients/openframe-client/src/models/execution_tests.rs @@ -0,0 +1,150 @@ +use super::*; + +#[test] +fn parses_command_message_camel_case() { + let m = CommandMessage::from_payload( + r#"{"executionId":"e","code":"echo hi","shell":"BASH","privilegeLevel":"ADMIN","timeout":30}"#, + ) + .unwrap(); + assert_eq!(m.execution_id, "e"); + assert!(matches!(m.shell, ScriptShell::Bash)); + assert_eq!(m.timeout, 30); +} + +#[test] +fn parses_script_message_with_env_and_args() { + let m = ScriptMessage::from_payload( + r#"{"executionId":"e","machineId":"mac","scheduleId":"sch-1","scriptId":"scr-1","code":"x","shell":"POWERSHELL","privilegeLevel":"USER","args":["-v"],"timeoutSeconds":60,"envVars":[{"name":"FOO","value":"bar"}]}"#, + ) + .unwrap(); + let req = &m.to_requests()[0]; + assert_eq!(req.timeout_secs, 60); + assert_eq!(req.args, &["-v".to_string()]); + assert_eq!(req.env_vars, vec!["FOO=bar".to_string()]); + assert!(matches!(req.privilege, PrivilegeLevel::User)); + assert_eq!(req.script_id, Some("scr-1")); + assert_eq!(req.schedule_id, Some("sch-1")); +} + +#[test] +fn script_message_without_script_id_still_parses() { + let m = + ScriptMessage::from_payload(r#"{"executionId":"e","code":"x","shell":"BASH"}"#).unwrap(); + assert!(m.script_id.is_none()); + assert!(m.to_requests()[0].script_id.is_none()); +} + +#[test] +fn missing_privilege_defaults_to_admin() { + let m = + CommandMessage::from_payload(r#"{"executionId":"e","code":"x","shell":"CMD"}"#).unwrap(); + assert!(matches!( + m.to_requests()[0].privilege, + PrivilegeLevel::Admin + )); + assert_eq!(m.timeout, 900); +} + +#[test] +fn result_serializes_snake_case() { + let r = RmmResult { + execution_id: "e".into(), + machine_id: "m".into(), + stdout: "o".into(), + stderr: String::new(), + exit_code: 0, + execution_time_ms: 1, + timed_out: false, + error: None, + script_id: Some("scr-1".into()), + schedule_id: None, + }; + let v = serde_json::to_value(&r).unwrap(); + assert!(v.get("execution_id").is_some()); + assert!(v.get("exit_code").is_some()); + assert!(v.get("execution_time_ms").is_some()); + assert!(v.get("timed_out").is_some()); + assert!(v.get("error").is_none(), "None error must be omitted"); + assert_eq!(v.get("script_id").and_then(|s| s.as_str()), Some("scr-1")); + assert!( + v.get("schedule_id").is_none(), + "None schedule_id must be omitted" + ); +} + +#[test] +fn parses_schedule_batch_preserving_order() { + let m = ScriptScheduleExecutionMessage::from_payload( + r#"{"executionId":"ex-1","scheduleId":"sch-1","machineId":"mac","initiatedBy":"user@x","scripts":[ + {"scriptId":"a","code":"1","shell":"BASH","timeoutSeconds":10}, + {"scriptId":"b","code":"2","shell":"BASH","privilegeLevel":"USER","args":["-v"],"envVars":[{"name":"FOO","value":"bar"}]} + ]}"#, + ) + .unwrap(); + let reqs = m.to_requests(); + assert_eq!(reqs.len(), 2); + assert_eq!(reqs[0].script_id, Some("a")); + assert_eq!(reqs[0].code, "1"); + assert_eq!(reqs[0].timeout_secs, 10); + assert!(matches!(reqs[0].privilege, PrivilegeLevel::Admin)); + assert_eq!(reqs[1].script_id, Some("b")); + assert_eq!(reqs[1].args, &["-v".to_string()]); + assert_eq!(reqs[1].env_vars, vec!["FOO=bar".to_string()]); + assert!(reqs.iter().all(|r| r.execution_id == "ex-1")); + assert!(reqs.iter().all(|r| r.schedule_id == Some("sch-1"))); +} + +#[test] +fn schedule_batch_tolerates_explicit_nulls() { + let m = ScriptScheduleExecutionMessage::from_payload( + r#"{"executionId":"ex-1","scheduleId":null,"machineId":null,"initiatedBy":null,"scripts":[ + {"scriptId":"a","code":"x","shell":"BASH","privilegeLevel":null,"args":null,"timeoutSeconds":null,"envVars":null} + ]}"#, + ) + .unwrap(); + let reqs = m.to_requests(); + assert_eq!(reqs.len(), 1); + assert_eq!(reqs[0].timeout_secs, 900); + assert!(reqs[0].args.is_empty()); + assert!(reqs[0].env_vars.is_empty()); + assert!(matches!(reqs[0].privilege, PrivilegeLevel::Admin)); + assert_eq!(reqs[0].schedule_id, None); +} + +#[test] +fn schedule_batch_runs_every_script() { + let n = 120; + let scripts: Vec = (0..n) + .map(|i| format!(r#"{{"scriptId":"s{}","code":"x","shell":"BASH"}}"#, i)) + .collect(); + let m = ScriptScheduleExecutionMessage::from_payload(&format!( + r#"{{"executionId":"ex-1","scripts":[{}]}}"#, + scripts.join(",") + )) + .unwrap(); + let reqs = m.to_requests(); + assert_eq!(reqs.len(), n); + assert_eq!(reqs[0].script_id, Some("s0")); + assert_eq!(reqs[n - 1].script_id, Some("s119")); +} + +#[test] +fn schedule_results_reuse_the_script_execution_subject() { + assert_eq!( + ScriptScheduleExecutionMessage::KIND, + "script-schedule-execution" + ); + assert_eq!( + ScriptScheduleExecutionMessage::RESULT_KIND, + "script-execution" + ); + assert_eq!(ScriptMessage::RESULT_KIND, ScriptMessage::KIND); + assert_eq!(CommandMessage::RESULT_KIND, CommandMessage::KIND); +} + +#[test] +fn shell_maps_to_param() { + assert_eq!(ScriptShell::Powershell.as_param(), "powershell"); + assert_eq!(ScriptShell::Cmd.as_param(), "cmd"); + assert_eq!(ScriptShell::Shell.as_param(), "sh"); +} diff --git a/clients/openframe-client/src/models/mod.rs b/clients/openframe-client/src/models/mod.rs index e09ea1ea3..74358a2b7 100644 --- a/clients/openframe-client/src/models/mod.rs +++ b/clients/openframe-client/src/models/mod.rs @@ -23,7 +23,8 @@ pub mod update_state; pub use execution::{ CommandMessage, ExecutionMessage, ExecutionRequest, PrivilegeLevel, RmmResult, ScriptEnvVar, - ScriptMessage, ScriptShell, + ScriptMessage, ScriptScheduleExecutionItem, ScriptScheduleExecutionMessage, ScriptShell, + ScriptSpec, }; pub use agent_configuration::AgentConfiguration; diff --git a/clients/openframe-client/src/platform/binary_writer.rs b/clients/openframe-client/src/platform/binary_writer.rs index 133beb95d..e37de32b3 100644 --- a/clients/openframe-client/src/platform/binary_writer.rs +++ b/clients/openframe-client/src/platform/binary_writer.rs @@ -1,5 +1,5 @@ use anyhow::{Context, Result}; -use std::path::Path; +use std::path::{Path, PathBuf}; use tokio::fs::{self, File}; use tokio::io::AsyncWriteExt; use tracing::{info, warn}; @@ -7,6 +7,13 @@ use tracing::{info, warn}; #[cfg(target_family = "unix")] use std::os::unix::fs::PermissionsExt; +/// Path of the `.old` rename-aside `write_executable` creates when the target binary is locked. +pub fn aside_path(path: &Path) -> PathBuf { + let mut aside = path.as_os_str().to_os_string(); + aside.push(".old"); + PathBuf::from(aside) +} + pub async fn write_executable(bytes: &[u8], path: &Path) -> Result<()> { if let Some(parent) = path.parent() { fs::create_dir_all(parent) @@ -26,9 +33,7 @@ pub async fn write_executable(bytes: &[u8], path: &Path) -> Result<()> { #[cfg(target_os = "windows")] { - let mut aside = path.as_os_str().to_os_string(); - aside.push(".old"); - let aside = std::path::PathBuf::from(aside); + let aside = aside_path(path); let _ = fs::remove_file(&aside).await; match fs::rename(path, &aside).await { Ok(()) => info!( @@ -76,3 +81,7 @@ pub async fn set_executable_permissions(path: &Path) -> Result<()> { Ok(()) } + +#[cfg(test)] +#[path = "binary_writer_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/platform/binary_writer_tests.rs b/clients/openframe-client/src/platform/binary_writer_tests.rs new file mode 100644 index 000000000..ec1a09e12 --- /dev/null +++ b/clients/openframe-client/src/platform/binary_writer_tests.rs @@ -0,0 +1,13 @@ +use super::*; + +#[test] +fn aside_path_appends_old_to_full_filename() { + assert_eq!( + aside_path(Path::new(r"C:\x\agent.exe")), + PathBuf::from(r"C:\x\agent.exe.old") + ); + assert_eq!( + aside_path(Path::new("/x/agent")), + PathBuf::from("/x/agent.old") + ); +} diff --git a/clients/openframe-client/src/platform/directories.rs b/clients/openframe-client/src/platform/directories.rs index 4d59714ae..6df1c9fb0 100644 --- a/clients/openframe-client/src/platform/directories.rs +++ b/clients/openframe-client/src/platform/directories.rs @@ -884,268 +884,5 @@ pub async fn remove_app_bundle_path(executable_path: &Path) -> Result<()> { } #[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn test_directory_creation() { - let temp_dir = tempdir().unwrap(); - let logs_dir = temp_dir.path().join("logs"); - let app_dir = temp_dir.path().join("app"); - let secured_dir = temp_dir.path().join("secured"); - - let manager = DirectoryManager::with_custom_dirs( - logs_dir.clone(), - app_dir.clone(), - secured_dir.clone(), - ); - - // Test directory creation - assert!(manager.ensure_directories().is_ok()); - assert!(logs_dir.exists()); - assert!(app_dir.exists()); - } - - #[test] - #[cfg_attr( - not(windows), - ignore = "requires root-owned secured dir; not applicable to non-root unix runs" - )] - fn test_directory_permissions() { - let temp_dir = tempdir().unwrap(); - let logs_dir = temp_dir.path().join("logs"); - let app_dir = temp_dir.path().join("app"); - let secured_dir = temp_dir.path().join("secured"); - - let manager = DirectoryManager::with_custom_dirs( - logs_dir.clone(), - app_dir.clone(), - secured_dir.clone(), - ); - - // Create directories first - assert!(manager.ensure_directories().is_ok()); - - // Test permission validation and fixing - assert!(manager.validate_permissions().is_ok()); - - // Test user directory creation - let user_manager = DirectoryManager::with_user_logs_dir(); - if let Some(user_logs) = &user_manager.user_logs_dir { - assert!( - user_logs.to_string_lossy().contains("OpenFrame") - || user_logs.to_string_lossy().contains("openframe") - ); - } - } - - #[test] - fn test_file_permissions() { - let temp_dir = tempdir().unwrap(); - let logs_dir = temp_dir.path().join("logs"); - let app_dir = temp_dir.path().join("app"); - let secured_dir = temp_dir.path().join("secured"); - - let manager = DirectoryManager::with_custom_dirs( - logs_dir.clone(), - app_dir.clone(), - secured_dir.clone(), - ); - - // Create directories first - assert!(manager.ensure_directories().is_ok()); - - // Create a test file in the logs directory - let test_file = logs_dir.join("test.log"); - fs::write(&test_file, "test").unwrap(); - - // Apply file permissions - let file_perms = Permissions::file(); - assert!(file_perms.apply(&test_file).is_ok()); - - // Verify file permissions - #[cfg(unix)] - { - if unsafe { libc::geteuid() } == 0 { - // Only run this check if we're root, otherwise it will fail - let metadata = fs::metadata(&test_file).unwrap(); - assert_eq!(metadata.permissions().mode() & 0o777, 0o644); - } - } - } - - #[test] - #[allow(unused_variables)] // manager used only by unix assertion below - fn test_error_handling() { - // Test with a non-existent directory - let non_existent = PathBuf::from("/non_existent_dir_for_test"); - - let manager = DirectoryManager::with_custom_dirs( - non_existent.clone(), - non_existent.clone(), - non_existent.clone(), - ); - - // This should fail on validate because we can't create the directory - #[cfg(unix)] - if unsafe { libc::geteuid() } != 0 { - // We expect this to fail if we're not root - assert!(manager.validate_permissions().is_err()); - } - } - - #[test] - fn test_user_logs_directory() { - let manager = DirectoryManager::with_user_logs_dir(); - - // Ensure the user logs directory exists - assert!(manager.user_logs_dir.is_some()); - - #[cfg(target_os = "macos")] - { - let user_logs = manager.user_logs_dir.unwrap(); - assert!(user_logs - .to_string_lossy() - .contains("Library/Logs/OpenFrame")); - } - - #[cfg(target_os = "windows")] - { - let user_logs = manager.user_logs_dir.unwrap(); - assert!(user_logs.to_string_lossy().contains("OpenFrame\\Logs")); - } - - #[cfg(target_os = "linux")] - { - let user_logs = manager.user_logs_dir.unwrap(); - assert!(user_logs - .to_string_lossy() - .contains(".local/share/openframe/logs")); - } - } - - #[test] - #[cfg_attr( - not(windows), - ignore = "requires root-owned secured dir; not applicable to non-root unix runs" - )] - fn test_health_check() { - let temp_dir = tempdir().unwrap(); - let logs_dir = temp_dir.path().join("logs"); - let app_dir = temp_dir.path().join("app"); - let secured_dir = temp_dir.path().join("secured"); - - let manager = DirectoryManager::with_custom_dirs( - logs_dir.clone(), - app_dir.clone(), - secured_dir.clone(), - ); - - // Test health check - assert!(manager.perform_health_check().is_ok()); - assert!(logs_dir.exists()); - assert!(app_dir.exists()); - - // Intentionally corrupt permissions to test fixing - #[cfg(unix)] - { - if unsafe { libc::geteuid() } == 0 { - // Only run this check if we're root, otherwise it will fail - use std::os::unix::fs::PermissionsExt; - let bad_perms = fs::Permissions::from_mode(0o700); - fs::set_permissions(&logs_dir, bad_perms).unwrap(); - - // Health check should fix the permissions - assert!(manager.perform_health_check().is_ok()); - - // Verify permissions were fixed - let metadata = fs::metadata(&logs_dir).unwrap(); - assert_eq!(metadata.permissions().mode() & 0o777, 0o755); - } - } - } - - #[test] - fn test_write_permissions() { - let temp_dir = tempdir().unwrap(); - let logs_dir = temp_dir.path().join("logs"); - let app_dir = temp_dir.path().join("app"); - let secured_dir = temp_dir.path().join("secured"); - - let manager = DirectoryManager::with_custom_dirs( - logs_dir.clone(), - app_dir.clone(), - secured_dir.clone(), - ); - - // Create directories first - assert!(manager.ensure_directories().is_ok()); - - // Test write permissions - assert!(manager.can_write_to_directory(&logs_dir)); - assert!(manager.can_write_to_directory(&app_dir)); - } - - #[test] - fn test_get_logs_directory() { - let logs_dir = get_logs_directory(); - - #[cfg(target_os = "macos")] - assert_eq!(logs_dir, PathBuf::from("/Library/Logs/OpenFrame")); - - #[cfg(target_os = "linux")] - assert_eq!(logs_dir, PathBuf::from("/var/log/openframe")); - - #[cfg(target_os = "windows")] - { - let program_data = std::env::var_os("ProgramData").unwrap_or_default(); - let expected = PathBuf::from(program_data).join("OpenFrame").join("logs"); - assert_eq!(logs_dir, expected); - } - } - - #[test] - fn test_get_app_support_directory() { - let app_dir = get_app_support_directory(); - - #[cfg(target_os = "macos")] - assert_eq!( - app_dir, - PathBuf::from("/Library/Application Support/OpenFrame") - ); - - #[cfg(target_os = "linux")] - assert_eq!(app_dir, PathBuf::from("/var/lib/openframe")); - - #[cfg(target_os = "windows")] - { - let program_data = std::env::var_os("ProgramData").unwrap_or_default(); - let expected = PathBuf::from(program_data).join("OpenFrame"); - assert_eq!(app_dir, expected); - } - } - - #[test] - fn test_get_secured_directory() { - let secured_dir = get_secured_directory(); - - #[cfg(target_os = "macos")] - assert_eq!( - secured_dir, - PathBuf::from("/Library/Application Support/OpenFrame/secured") - ); - - #[cfg(target_os = "linux")] - assert_eq!(secured_dir, PathBuf::from("/var/lib/openframe/secured")); - - #[cfg(target_os = "windows")] - { - let program_data = std::env::var_os("ProgramData").unwrap_or_default(); - let expected = PathBuf::from(program_data) - .join("OpenFrame") - .join("secured"); - assert_eq!(secured_dir, expected); - } - } -} +#[path = "directories_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/platform/directories_tests.rs b/clients/openframe-client/src/platform/directories_tests.rs new file mode 100644 index 000000000..3a4ca3277 --- /dev/null +++ b/clients/openframe-client/src/platform/directories_tests.rs @@ -0,0 +1,248 @@ +use super::*; +use tempfile::tempdir; + +#[test] +fn test_directory_creation() { + let temp_dir = tempdir().unwrap(); + let logs_dir = temp_dir.path().join("logs"); + let app_dir = temp_dir.path().join("app"); + let secured_dir = temp_dir.path().join("secured"); + + let manager = + DirectoryManager::with_custom_dirs(logs_dir.clone(), app_dir.clone(), secured_dir.clone()); + + // Test directory creation + assert!(manager.ensure_directories().is_ok()); + assert!(logs_dir.exists()); + assert!(app_dir.exists()); +} + +#[test] +#[cfg_attr( + not(windows), + ignore = "requires root-owned secured dir; not applicable to non-root unix runs" +)] +fn test_directory_permissions() { + let temp_dir = tempdir().unwrap(); + let logs_dir = temp_dir.path().join("logs"); + let app_dir = temp_dir.path().join("app"); + let secured_dir = temp_dir.path().join("secured"); + + let manager = + DirectoryManager::with_custom_dirs(logs_dir.clone(), app_dir.clone(), secured_dir.clone()); + + // Create directories first + assert!(manager.ensure_directories().is_ok()); + + // Test permission validation and fixing + assert!(manager.validate_permissions().is_ok()); + + // Test user directory creation + let user_manager = DirectoryManager::with_user_logs_dir(); + if let Some(user_logs) = &user_manager.user_logs_dir { + assert!( + user_logs.to_string_lossy().contains("OpenFrame") + || user_logs.to_string_lossy().contains("openframe") + ); + } +} + +#[test] +fn test_file_permissions() { + let temp_dir = tempdir().unwrap(); + let logs_dir = temp_dir.path().join("logs"); + let app_dir = temp_dir.path().join("app"); + let secured_dir = temp_dir.path().join("secured"); + + let manager = + DirectoryManager::with_custom_dirs(logs_dir.clone(), app_dir.clone(), secured_dir.clone()); + + // Create directories first + assert!(manager.ensure_directories().is_ok()); + + // Create a test file in the logs directory + let test_file = logs_dir.join("test.log"); + fs::write(&test_file, "test").unwrap(); + + // Apply file permissions + let file_perms = Permissions::file(); + assert!(file_perms.apply(&test_file).is_ok()); + + // Verify file permissions + #[cfg(unix)] + { + if unsafe { libc::geteuid() } == 0 { + // Only run this check if we're root, otherwise it will fail + let metadata = fs::metadata(&test_file).unwrap(); + assert_eq!(metadata.permissions().mode() & 0o777, 0o644); + } + } +} + +#[test] +#[allow(unused_variables)] // manager used only by unix assertion below +fn test_error_handling() { + // Test with a non-existent directory + let non_existent = PathBuf::from("/non_existent_dir_for_test"); + + let manager = DirectoryManager::with_custom_dirs( + non_existent.clone(), + non_existent.clone(), + non_existent.clone(), + ); + + // This should fail on validate because we can't create the directory + #[cfg(unix)] + if unsafe { libc::geteuid() } != 0 { + // We expect this to fail if we're not root + assert!(manager.validate_permissions().is_err()); + } +} + +#[test] +fn test_user_logs_directory() { + let manager = DirectoryManager::with_user_logs_dir(); + + // Ensure the user logs directory exists + assert!(manager.user_logs_dir.is_some()); + + #[cfg(target_os = "macos")] + { + let user_logs = manager.user_logs_dir.unwrap(); + assert!(user_logs + .to_string_lossy() + .contains("Library/Logs/OpenFrame")); + } + + #[cfg(target_os = "windows")] + { + let user_logs = manager.user_logs_dir.unwrap(); + assert!(user_logs.to_string_lossy().contains("OpenFrame\\Logs")); + } + + #[cfg(target_os = "linux")] + { + let user_logs = manager.user_logs_dir.unwrap(); + assert!(user_logs + .to_string_lossy() + .contains(".local/share/openframe/logs")); + } +} + +#[test] +#[cfg_attr( + not(windows), + ignore = "requires root-owned secured dir; not applicable to non-root unix runs" +)] +fn test_health_check() { + let temp_dir = tempdir().unwrap(); + let logs_dir = temp_dir.path().join("logs"); + let app_dir = temp_dir.path().join("app"); + let secured_dir = temp_dir.path().join("secured"); + + let manager = + DirectoryManager::with_custom_dirs(logs_dir.clone(), app_dir.clone(), secured_dir.clone()); + + // Test health check + assert!(manager.perform_health_check().is_ok()); + assert!(logs_dir.exists()); + assert!(app_dir.exists()); + + // Intentionally corrupt permissions to test fixing + #[cfg(unix)] + { + if unsafe { libc::geteuid() } == 0 { + // Only run this check if we're root, otherwise it will fail + use std::os::unix::fs::PermissionsExt; + let bad_perms = fs::Permissions::from_mode(0o700); + fs::set_permissions(&logs_dir, bad_perms).unwrap(); + + // Health check should fix the permissions + assert!(manager.perform_health_check().is_ok()); + + // Verify permissions were fixed + let metadata = fs::metadata(&logs_dir).unwrap(); + assert_eq!(metadata.permissions().mode() & 0o777, 0o755); + } + } +} + +#[test] +fn test_write_permissions() { + let temp_dir = tempdir().unwrap(); + let logs_dir = temp_dir.path().join("logs"); + let app_dir = temp_dir.path().join("app"); + let secured_dir = temp_dir.path().join("secured"); + + let manager = + DirectoryManager::with_custom_dirs(logs_dir.clone(), app_dir.clone(), secured_dir.clone()); + + // Create directories first + assert!(manager.ensure_directories().is_ok()); + + // Test write permissions + assert!(manager.can_write_to_directory(&logs_dir)); + assert!(manager.can_write_to_directory(&app_dir)); +} + +#[test] +fn test_get_logs_directory() { + let logs_dir = get_logs_directory(); + + #[cfg(target_os = "macos")] + assert_eq!(logs_dir, PathBuf::from("/Library/Logs/OpenFrame")); + + #[cfg(target_os = "linux")] + assert_eq!(logs_dir, PathBuf::from("/var/log/openframe")); + + #[cfg(target_os = "windows")] + { + let program_data = std::env::var_os("ProgramData").unwrap_or_default(); + let expected = PathBuf::from(program_data).join("OpenFrame").join("logs"); + assert_eq!(logs_dir, expected); + } +} + +#[test] +fn test_get_app_support_directory() { + let app_dir = get_app_support_directory(); + + #[cfg(target_os = "macos")] + assert_eq!( + app_dir, + PathBuf::from("/Library/Application Support/OpenFrame") + ); + + #[cfg(target_os = "linux")] + assert_eq!(app_dir, PathBuf::from("/var/lib/openframe")); + + #[cfg(target_os = "windows")] + { + let program_data = std::env::var_os("ProgramData").unwrap_or_default(); + let expected = PathBuf::from(program_data).join("OpenFrame"); + assert_eq!(app_dir, expected); + } +} + +#[test] +fn test_get_secured_directory() { + let secured_dir = get_secured_directory(); + + #[cfg(target_os = "macos")] + assert_eq!( + secured_dir, + PathBuf::from("/Library/Application Support/OpenFrame/secured") + ); + + #[cfg(target_os = "linux")] + assert_eq!(secured_dir, PathBuf::from("/var/lib/openframe/secured")); + + #[cfg(target_os = "windows")] + { + let program_data = std::env::var_os("ProgramData").unwrap_or_default(); + let expected = PathBuf::from(program_data) + .join("OpenFrame") + .join("secured"); + assert_eq!(secured_dir, expected); + } +} diff --git a/clients/openframe-client/src/platform/mod.rs b/clients/openframe-client/src/platform/mod.rs index a560fcb5b..76371fa6a 100644 --- a/clients/openframe-client/src/platform/mod.rs +++ b/clients/openframe-client/src/platform/mod.rs @@ -39,6 +39,7 @@ pub use installation_detector::detect_actual_installation; pub use permissions::{Capability, PermissionError, PermissionUtils, Permissions}; #[cfg(target_os = "windows")] pub use powershell::get_powershell_path; +pub(crate) use tool_updater::clear_aside_binary; pub use tool_updater::{ create_migrator, create_updater, needs_migration, run_migration, run_update, ToolUpdater, ToolUpdaterDeps, UpdateContext, diff --git a/clients/openframe-client/src/platform/permissions.rs b/clients/openframe-client/src/platform/permissions.rs index e975608a2..149b9b17b 100644 --- a/clients/openframe-client/src/platform/permissions.rs +++ b/clients/openframe-client/src/platform/permissions.rs @@ -551,113 +551,5 @@ pub enum Capability { } #[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn test_permissions_creation() { - let dir_perms = Permissions::directory(); - assert_eq!(dir_perms.mode, 0o755); - - let file_perms = Permissions::file(); - assert_eq!(file_perms.mode, 0o644); - } - - #[cfg(unix)] - #[test] - fn test_permissions_verification() { - if unsafe { libc::geteuid() } == 0 { - let temp = tempdir().unwrap(); - let test_path = temp.path().join("test_file"); - fs::write(&test_path, "test").unwrap(); - - let perms = Permissions::file(); - assert!(perms.apply(&test_path).is_ok()); - assert!(perms.verify(&test_path).unwrap()); - } - } - - #[test] - fn test_is_admin() { - // This just verifies the function runs without errors - let is_admin = PermissionUtils::is_admin(); - println!("Running with admin privileges: {}", is_admin); - } - - #[test] - fn test_has_capability() { - // Test all capabilities - for cap in &[ - Capability::ManageServices, - Capability::WriteSystemDirectories, - Capability::ReadSystemLogs, - Capability::WriteSystemLogs, - ] { - let has_cap = PermissionUtils::has_capability(*cap); - println!("Has capability {:?}: {}", cap, has_cap); - } - } - - #[test] - #[ignore = "triggers an interactive macOS osascript / Windows UAC admin prompt; hangs in headless CI"] - fn test_ensure_admin() { - // This should return Ok if already admin, or attempt to get privileges - let result = PermissionUtils::ensure_admin(); - - if PermissionUtils::is_admin() { - assert!(result.is_ok()); - } else { - // The function might return Ok if the user granted privileges via the prompt, - // or an error if they declined or if there was an issue with the prompt - println!("Result of ensure_admin when not admin: {:?}", result); - } - } - - #[test] - fn test_run_command() { - // Test running a simple command that should work on all platforms - // On Windows, use "cmd /c echo test" - // On Unix, use "echo test" - #[cfg(target_os = "windows")] - { - let result = PermissionUtils::run_command("cmd", &["/c", "echo", "test"]); - assert!(result.is_ok()); - } - - #[cfg(unix)] - { - let result = PermissionUtils::run_command("echo", &["test"]); - assert!(result.is_ok()); - } - } - - #[test] - fn test_cross_platform_permissions() { - // Create a temporary file and test platform-agnostic permissions - let temp = tempdir().unwrap(); - let test_path = temp.path().join("test_file"); - fs::write(&test_path, "test").unwrap(); - - // Test applying permissions - let perms = Permissions::file(); - let result = perms.apply(&test_path); - assert!(result.is_ok()); - - // Test verifying permissions - should pass on all platforms - // even though the exact permission representation differs - let verify_result = perms.verify(&test_path); - assert!(verify_result.is_ok()); - - // Test retrieving permissions from a path - let retrieved_perms = Permissions::from_path(&test_path); - assert!(retrieved_perms.is_ok()); - } - - #[test] - fn test_can_read_system_logs() { - // Just verify the function runs without errors - let can_read = PermissionUtils::has_capability(Capability::ReadSystemLogs); - println!("Can read system logs: {}", can_read); - } -} +#[path = "permissions_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/platform/permissions_tests.rs b/clients/openframe-client/src/platform/permissions_tests.rs new file mode 100644 index 000000000..5fc9a7ebc --- /dev/null +++ b/clients/openframe-client/src/platform/permissions_tests.rs @@ -0,0 +1,108 @@ +use super::*; +use tempfile::tempdir; + +#[test] +fn test_permissions_creation() { + let dir_perms = Permissions::directory(); + assert_eq!(dir_perms.mode, 0o755); + + let file_perms = Permissions::file(); + assert_eq!(file_perms.mode, 0o644); +} + +#[cfg(unix)] +#[test] +fn test_permissions_verification() { + if unsafe { libc::geteuid() } == 0 { + let temp = tempdir().unwrap(); + let test_path = temp.path().join("test_file"); + fs::write(&test_path, "test").unwrap(); + + let perms = Permissions::file(); + assert!(perms.apply(&test_path).is_ok()); + assert!(perms.verify(&test_path).unwrap()); + } +} + +#[test] +fn test_is_admin() { + // This just verifies the function runs without errors + let is_admin = PermissionUtils::is_admin(); + println!("Running with admin privileges: {}", is_admin); +} + +#[test] +fn test_has_capability() { + // Test all capabilities + for cap in &[ + Capability::ManageServices, + Capability::WriteSystemDirectories, + Capability::ReadSystemLogs, + Capability::WriteSystemLogs, + ] { + let has_cap = PermissionUtils::has_capability(*cap); + println!("Has capability {:?}: {}", cap, has_cap); + } +} + +#[test] +#[ignore = "triggers an interactive macOS osascript / Windows UAC admin prompt; hangs in headless CI"] +fn test_ensure_admin() { + // This should return Ok if already admin, or attempt to get privileges + let result = PermissionUtils::ensure_admin(); + + if PermissionUtils::is_admin() { + assert!(result.is_ok()); + } else { + // The function might return Ok if the user granted privileges via the prompt, + // or an error if they declined or if there was an issue with the prompt + println!("Result of ensure_admin when not admin: {:?}", result); + } +} + +#[test] +fn test_run_command() { + // Test running a simple command that should work on all platforms + // On Windows, use "cmd /c echo test" + // On Unix, use "echo test" + #[cfg(target_os = "windows")] + { + let result = PermissionUtils::run_command("cmd", &["/c", "echo", "test"]); + assert!(result.is_ok()); + } + + #[cfg(unix)] + { + let result = PermissionUtils::run_command("echo", &["test"]); + assert!(result.is_ok()); + } +} + +#[test] +fn test_cross_platform_permissions() { + // Create a temporary file and test platform-agnostic permissions + let temp = tempdir().unwrap(); + let test_path = temp.path().join("test_file"); + fs::write(&test_path, "test").unwrap(); + + // Test applying permissions + let perms = Permissions::file(); + let result = perms.apply(&test_path); + assert!(result.is_ok()); + + // Test verifying permissions - should pass on all platforms + // even though the exact permission representation differs + let verify_result = perms.verify(&test_path); + assert!(verify_result.is_ok()); + + // Test retrieving permissions from a path + let retrieved_perms = Permissions::from_path(&test_path); + assert!(retrieved_perms.is_ok()); +} + +#[test] +fn test_can_read_system_logs() { + // Just verify the function runs without errors + let can_read = PermissionUtils::has_capability(Capability::ReadSystemLogs); + println!("Can read system logs: {}", can_read); +} diff --git a/clients/openframe-client/src/platform/system_service.rs b/clients/openframe-client/src/platform/system_service.rs index dc641ad79..8f6fe4453 100644 --- a/clients/openframe-client/src/platform/system_service.rs +++ b/clients/openframe-client/src/platform/system_service.rs @@ -2,7 +2,8 @@ #[cfg(target_os = "windows")] use crate::config::service_stop::{ - PROCESS_CHECK_INTERVAL_MS, SERVICE_FORCE_KILL_MAX_ATTEMPTS, SERVICE_START_MAX_ATTEMPTS, + PROCESS_CHECK_INTERVAL_MS, SCM_QUERY_MAX_CONSECUTIVE_FAILURES, SCM_QUERY_TIMEOUT_SECS, + SERVICE_FORCE_KILL_MAX_ATTEMPTS, SERVICE_START_CALL_TIMEOUT_SECS, SERVICE_START_MAX_ATTEMPTS, SERVICE_STOP_CALL_TIMEOUT_SECS, SERVICE_STOP_MAX_ATTEMPTS, }; use anyhow::{Context, Result}; @@ -11,6 +12,45 @@ use tokio::process::Command; use tokio::time::{sleep, Duration}; use tracing::{error, info, warn}; +/// Shared pool bounding in-flight SCM calls; timed-out calls can park at most SCM_MAX_IN_FLIGHT threads. +#[cfg(target_os = "windows")] +fn scm_pool() -> &'static crate::utils::timed_permit_pool::TimedPermitPool { + use crate::utils::timed_permit_pool::TimedPermitPool; + static POOL: std::sync::OnceLock = std::sync::OnceLock::new(); + POOL.get_or_init(|| TimedPermitPool::new(crate::config::service_stop::SCM_MAX_IN_FLIGHT)) +} + +/// Run a blocking SCM call off-runtime with a timeout, so a wedged SCM can never hang an async task. +#[cfg(target_os = "windows")] +async fn scm_call_timed(service_name: &str, what: &str, timeout_secs: u64, f: F) -> Result +where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, +{ + scm_pool() + .call( + &format!("SCM {} for service {}", what, service_name), + Duration::from_secs(timeout_secs), + f, + ) + .await +} + +/// `query_service_status_windows` off-runtime with a timeout; the outer Err is an unresponsive SCM. +#[cfg(target_os = "windows")] +async fn query_service_status_timed( + service_name: &str, +) -> Result> { + let name = service_name.to_string(); + scm_call_timed( + service_name, + "status query", + SCM_QUERY_TIMEOUT_SECS, + move || query_service_status_windows(&name), + ) + .await +} + /// Start a macOS service via launchctl load #[cfg(target_os = "macos")] pub async fn start_service(service_name: &str) -> Result<()> { @@ -39,18 +79,37 @@ pub async fn start_service(service_name: &str) -> Result<()> { let mut last_err = String::new(); for attempt in 1..=SERVICE_START_MAX_ATTEMPTS { - match try_start_service_windows(service_name) { - Ok(()) if wait_for_service_running_windows(service_name).await => { - info!("Service {} confirmed running", service_name); - return Ok(()); - } - Ok(()) => { - last_err = "service did not reach RUNNING after start".to_string(); - warn!( - "Start attempt {}/{} for service {}: {}", - attempt, SERVICE_START_MAX_ATTEMPTS, service_name, last_err - ); - } + let name = service_name.to_string(); + let start_result = scm_call_timed( + service_name, + "start call", + SERVICE_START_CALL_TIMEOUT_SECS, + move || try_start_service_windows(&name), + ) + .await + .map_err(|e| format!("{e:#}")) + .and_then(|r| r); + match start_result { + Ok(()) => match wait_for_service_running_windows(service_name).await { + Some(true) => { + info!("Service {} confirmed running", service_name); + return Ok(()); + } + Some(false) => { + last_err = "service did not reach RUNNING after start".to_string(); + warn!( + "Start attempt {}/{} for service {}: {}", + attempt, SERVICE_START_MAX_ATTEMPTS, service_name, last_err + ); + } + None => { + last_err = "could not confirm RUNNING (SCM unresponsive)".to_string(); + warn!( + "Start attempt {}/{} for service {}: {}", + attempt, SERVICE_START_MAX_ATTEMPTS, service_name, last_err + ); + } + }, Err(e) => { last_err = e; warn!( @@ -72,18 +131,29 @@ pub async fn start_service(service_name: &str) -> Result<()> { ) } +/// Some(true) = RUNNING, Some(false) = polls exhausted without RUNNING, None = SCM unresponsive. #[cfg(target_os = "windows")] -async fn wait_for_service_running_windows(service_name: &str) -> bool { +async fn wait_for_service_running_windows(service_name: &str) -> Option { use windows_service::service::ServiceState; + let mut query_failures = 0u32; for _ in 1..=SERVICE_STOP_MAX_ATTEMPTS { sleep(Duration::from_millis(PROCESS_CHECK_INTERVAL_MS)).await; - if let Ok(status) = query_service_status_windows(service_name) { - if status.current_state == ServiceState::Running { - return true; + match query_service_status_timed(service_name).await { + Ok(Ok(status)) if status.current_state == ServiceState::Running => return Some(true), + Ok(_) => query_failures = 0, + Err(e) => { + query_failures += 1; + warn!( + "Status query for service {} failed while awaiting RUNNING: {e:#}", + service_name + ); + if query_failures >= SCM_QUERY_MAX_CONSECUTIVE_FAILURES { + return None; + } } } } - false + Some(false) } #[cfg(target_os = "windows")] @@ -167,7 +237,7 @@ pub async fn verify_service_running(service_name: &str) -> Result<()> { #[cfg(target_os = "windows")] { use windows_service::service::ServiceState; - if let Ok(status) = query_service_status_windows(service_name) { + if let Ok(Ok(status)) = query_service_status_timed(service_name).await { if status.current_state == ServiceState::Running { return Ok(()); } @@ -192,7 +262,16 @@ pub async fn verify_service_running(service_name: &str) -> Result<()> { pub async fn service_clear_for_install(service_name: &str) -> bool { #[cfg(target_os = "windows")] { - service_stopped_or_missing(&query_service_status_windows(service_name)) + match query_service_status_timed(service_name).await { + Ok(status) => service_stopped_or_missing(&status), + Err(e) => { + warn!( + "Status query for service {} failed during install check: {e:#}", + service_name + ); + false + } + } } #[cfg(not(target_os = "windows"))] { @@ -211,33 +290,26 @@ async fn stop_service_windows(service_name: &str, allow_delete: bool) -> Result< info!("Stopping Windows service via SCM: {}", service_name); let svc = service_name.to_string(); - let stop_call = tokio::task::spawn_blocking(move || { - let service = - open_service_windows(&svc, ServiceAccess::QUERY_STATUS | ServiceAccess::STOP)?; - service.stop() - }); - - let stop_result = match tokio::time::timeout( - Duration::from_secs(SERVICE_STOP_CALL_TIMEOUT_SECS), - stop_call, + let stop_result = match scm_call_timed( + service_name, + "stop call", + SERVICE_STOP_CALL_TIMEOUT_SECS, + move || { + let service = + open_service_windows(&svc, ServiceAccess::QUERY_STATUS | ServiceAccess::STOP)?; + service.stop() + }, ) .await { - Err(_elapsed) => { + Err(e) => { warn!( - "service.stop() for {} timed out after {}s; force-killing service process", - service_name, SERVICE_STOP_CALL_TIMEOUT_SECS - ); - return force_stop_service_windows(service_name, allow_delete).await; - } - Ok(Err(join_err)) => { - error!( - "service.stop() task for {} failed: {}; force-killing service process", - service_name, join_err + "service.stop() for {} did not complete ({e:#}); force-killing service process", + service_name ); return force_stop_service_windows(service_name, allow_delete).await; } - Ok(Ok(result)) => result, + Ok(result) => result, }; match stop_result { @@ -292,8 +364,17 @@ async fn stop_service_windows(service_name: &str, allow_delete: bool) -> Result< #[cfg(target_os = "windows")] async fn force_stop_service_windows(service_name: &str, allow_delete: bool) -> Result<()> { for attempt in 1..=SERVICE_FORCE_KILL_MAX_ATTEMPTS { - let status = query_service_status_windows(service_name); - if service_stopped_or_missing(&status) { + let status = match query_service_status_timed(service_name).await { + Ok(status) => Some(status), + Err(e) => { + warn!( + "Status query for service {} failed during force-stop (attempt {}/{}): {e:#}", + service_name, attempt, SERVICE_FORCE_KILL_MAX_ATTEMPTS + ); + None + } + }; + if status.as_ref().is_some_and(service_stopped_or_missing) { info!( "Service {} is no longer running (force-stop attempt {})", service_name, attempt @@ -301,7 +382,11 @@ async fn force_stop_service_windows(service_name: &str, allow_delete: bool) -> R return Ok(()); } - match status.as_ref().ok().and_then(|s| s.process_id) { + match status + .as_ref() + .and_then(|s| s.as_ref().ok()) + .and_then(|s| s.process_id) + { Some(pid) => { info!( "Force-killing service {} process tree (pid {}, attempt {}/{})", @@ -336,7 +421,22 @@ async fn force_stop_service_windows(service_name: &str, allow_delete: bool) -> R // other transitional states) the PID is hidden, so the SCM-PID path can never // act on a wedged service. Fall back to the service's configured image path // and kill any live process running from it directly. - match service_image_exe_path_windows(service_name) { + let name = service_name.to_string(); + let image_path = scm_call_timed( + service_name, + "config query", + SCM_QUERY_TIMEOUT_SECS, + move || service_image_exe_path_windows(&name), + ) + .await + .unwrap_or_else(|e| { + warn!( + "Config query for service {} failed during force-stop: {e:#}", + service_name + ); + None + }); + match image_path { Some(exe) => { let killed = kill_processes_by_exe_path_windows(&exe).await; if killed > 0 { @@ -348,10 +448,7 @@ async fn force_stop_service_windows(service_name: &str, allow_delete: bool) -> R } } None => { - let state = status - .as_ref() - .map(|s| format!("{:?}", s.current_state)) - .unwrap_or_else(|_| "unqueryable".to_string()); + let state = service_state_label(status.as_ref()); info!("Service {} has no reportable PID and no resolvable image path (state {}, attempt {}/{}); waiting", service_name, state, attempt, SERVICE_FORCE_KILL_MAX_ATTEMPTS); } @@ -362,16 +459,13 @@ async fn force_stop_service_windows(service_name: &str, allow_delete: bool) -> R sleep(Duration::from_millis(PROCESS_CHECK_INTERVAL_MS)).await; } - let status = query_service_status_windows(service_name); - if service_stopped_or_missing(&status) { + let status = query_service_status_timed(service_name).await.ok(); + if status.as_ref().is_some_and(service_stopped_or_missing) { info!("Service {} force-stopped successfully", service_name); return Ok(()); } - let state = status - .as_ref() - .map(|s| format!("{:?}", s.current_state)) - .unwrap_or_else(|_| "unqueryable".to_string()); + let state = service_state_label(status.as_ref()); // Only delete when the caller will recreate the service (install/reinstall/uninstall). The // update/restore paths pass allow_delete=false: deleting there would brick the tool because @@ -416,15 +510,35 @@ async fn force_stop_service_windows(service_name: &str, allow_delete: bool) -> R #[cfg(target_os = "windows")] async fn wait_for_service_stop_windows(service_name: &str) -> Result { + let mut query_failures = 0u32; for attempt in 1..=SERVICE_STOP_MAX_ATTEMPTS { sleep(Duration::from_millis(PROCESS_CHECK_INTERVAL_MS)).await; - if service_stopped_or_missing(&query_service_status_windows(service_name)) { - info!( - "Service {} confirmed stopped after {} attempts", - service_name, attempt - ); - return Ok(true); + match query_service_status_timed(service_name).await { + Ok(status) => { + query_failures = 0; + if service_stopped_or_missing(&status) { + info!( + "Service {} confirmed stopped after {} attempts", + service_name, attempt + ); + return Ok(true); + } + } + Err(e) => { + query_failures += 1; + warn!( + "Status query for service {} failed while awaiting stop: {e:#}", + service_name + ); + if query_failures >= SCM_QUERY_MAX_CONSECUTIVE_FAILURES { + warn!( + "SCM unresponsive for service {}; escalating to force-stop", + service_name + ); + return Ok(false); + } + } } } @@ -554,11 +668,24 @@ fn service_stopped_or_missing( } } +/// Human-readable state for logs; None or an SCM-level error reads as "unqueryable". +#[cfg(target_os = "windows")] +fn service_state_label( + status: Option<&windows_service::Result>, +) -> String { + status + .and_then(|s| s.as_ref().ok()) + .map(|s| format!("{:?}", s.current_state)) + .unwrap_or_else(|| "unqueryable".to_string()) +} + +// Sync SCM call on the caller's thread; not timeboxed (callers are off the restart path). #[cfg(target_os = "windows")] pub fn service_exists(service_name: &str) -> bool { query_service_status_windows(service_name).is_ok() } +// Sync SCM call on the caller's thread; not timeboxed (callers are off the restart path). #[cfg(target_os = "windows")] pub fn service_not_stopped(service_name: &str) -> bool { use windows_service::service::ServiceState; @@ -645,28 +772,54 @@ async fn kill_processes_by_exe_path_windows(exe_path: &std::path::Path) -> usize async fn delete_service_windows(service_name: &str) -> Result<()> { use windows_service::service::ServiceAccess; - if service_missing_windows(service_name) { + if service_missing_timed(service_name) + .await + .unwrap_or_else(|e| { + warn!( + "Existence query for service {} failed before delete: {e:#}", + service_name + ); + false + }) + { return Ok(()); } - // Scope the handle so it is closed before we poll — SCM only finalizes removal once the - // last open handle is released. - { - let service = open_service_windows(service_name, ServiceAccess::DELETE) - .with_context(|| format!("open service {} for deletion", service_name))?; - service - .delete() - .with_context(|| format!("DeleteService failed for {}", service_name))?; - } + // The handle is opened, used, and dropped inside the closure — SCM only finalizes removal + // once the last open handle is released. + let name = service_name.to_string(); + scm_call_timed( + service_name, + "delete call", + SCM_QUERY_TIMEOUT_SECS, + move || open_service_windows(&name, ServiceAccess::DELETE)?.delete(), + ) + .await? + .with_context(|| format!("open/DeleteService failed for {}", service_name))?; + let mut query_failures = 0u32; for _ in 1..=SERVICE_STOP_MAX_ATTEMPTS { - if service_missing_windows(service_name) { - return Ok(()); + match service_missing_timed(service_name).await { + Ok(true) => return Ok(()), + Ok(false) => query_failures = 0, + Err(e) => { + query_failures += 1; + warn!( + "Existence query for service {} failed while confirming deletion: {e:#}", + service_name + ); + if query_failures >= SCM_QUERY_MAX_CONSECUTIVE_FAILURES { + return Err(anyhow::anyhow!( + "SCM unresponsive while confirming deletion of service {}", + service_name + )); + } + } } sleep(Duration::from_millis(PROCESS_CHECK_INTERVAL_MS)).await; } - if service_missing_windows(service_name) { + if service_missing_timed(service_name).await.unwrap_or(false) { Ok(()) } else { Err(anyhow::anyhow!( @@ -675,3 +828,16 @@ async fn delete_service_windows(service_name: &str) -> Result<()> { )) } } + +/// `service_missing_windows` off-runtime with a timeout; the outer Err is an unresponsive SCM. +#[cfg(target_os = "windows")] +async fn service_missing_timed(service_name: &str) -> Result { + let name = service_name.to_string(); + scm_call_timed( + service_name, + "existence query", + SCM_QUERY_TIMEOUT_SECS, + move || service_missing_windows(&name), + ) + .await +} diff --git a/clients/openframe-client/src/platform/tool_updater/gui_app.rs b/clients/openframe-client/src/platform/tool_updater/gui_app.rs index 0ec138526..1d44999ee 100644 --- a/clients/openframe-client/src/platform/tool_updater/gui_app.rs +++ b/clients/openframe-client/src/platform/tool_updater/gui_app.rs @@ -163,28 +163,5 @@ impl ToolUpdater for GuiAppToolUpdater { } #[cfg(test)] -mod tests { - use super::*; - use crate::platform::DirectoryManager; - use std::path::Path; - - #[test] - fn test_find_app_bundle_path() { - assert_eq!( - DirectoryManager::find_app_bundle_path(Path::new("/Applications/FAE Chat.app")), - Some(PathBuf::from("/Applications/FAE Chat.app")) - ); - - assert_eq!( - DirectoryManager::find_app_bundle_path(Path::new( - "/Applications/FAE Chat.app/Contents/MacOS/FAE Chat" - )), - Some(PathBuf::from("/Applications/FAE Chat.app")) - ); - - assert_eq!( - DirectoryManager::find_app_bundle_path(Path::new("/usr/bin/some-binary")), - None - ); - } -} +#[path = "gui_app_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/platform/tool_updater/gui_app_tests.rs b/clients/openframe-client/src/platform/tool_updater/gui_app_tests.rs new file mode 100644 index 000000000..7d74c8762 --- /dev/null +++ b/clients/openframe-client/src/platform/tool_updater/gui_app_tests.rs @@ -0,0 +1,23 @@ +use super::*; +use crate::platform::DirectoryManager; +use std::path::Path; + +#[test] +fn test_find_app_bundle_path() { + assert_eq!( + DirectoryManager::find_app_bundle_path(Path::new("/Applications/FAE Chat.app")), + Some(PathBuf::from("/Applications/FAE Chat.app")) + ); + + assert_eq!( + DirectoryManager::find_app_bundle_path(Path::new( + "/Applications/FAE Chat.app/Contents/MacOS/FAE Chat" + )), + Some(PathBuf::from("/Applications/FAE Chat.app")) + ); + + assert_eq!( + DirectoryManager::find_app_bundle_path(Path::new("/usr/bin/some-binary")), + None + ); +} diff --git a/clients/openframe-client/src/platform/tool_updater/mod.rs b/clients/openframe-client/src/platform/tool_updater/mod.rs index 62769f02a..92bfe63b3 100644 --- a/clients/openframe-client/src/platform/tool_updater/mod.rs +++ b/clients/openframe-client/src/platform/tool_updater/mod.rs @@ -169,6 +169,35 @@ pub async fn run_migration( } } +/// Delete a leftover `.old` rename-aside; returns false when it is locked, i.e. a process from a previous update still executes it. +pub(crate) async fn clear_aside_binary(exec_path: &Path, tool_agent_id: &str) -> bool { + let aside = binary_writer::aside_path(exec_path); + if !aside.exists() { + return true; + } + match fs::remove_file(&aside).await { + Ok(()) => { + info!(tool_id = %tool_agent_id, "Removed stale aside binary: {}", aside.display()); + true + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => true, + Err(e) => { + error!(tool_id = %tool_agent_id, + "Cannot remove {} — a process from a previous update is likely still running it: {e}", + aside.display()); + false + } + } +} + +/// Post-kill visibility check: a surviving process keeps running the old version until it exits. +pub(crate) async fn log_update_survivors(deps: &ToolUpdaterDeps, tool: &InstalledTool) { + if deps.tool_kill_service.is_installed_tool_running(tool).await { + error!(tool_id = %tool.tool_agent_id, + "Tool process still running after stop — update proceeds, but the running instance stays on the old version until it exits"); + } +} + pub(crate) async fn backup_binary( source_path: &Path, tool_agent_id: &str, diff --git a/clients/openframe-client/src/platform/tool_updater/service.rs b/clients/openframe-client/src/platform/tool_updater/service.rs index d91853241..95adf0b37 100644 --- a/clients/openframe-client/src/platform/tool_updater/service.rs +++ b/clients/openframe-client/src/platform/tool_updater/service.rs @@ -4,14 +4,13 @@ use std::path::PathBuf; use tracing::{info, warn}; use super::{ - backup_binary, cleanup_backup, download_and_write_binary, restore_from_backup, ToolUpdater, - ToolUpdaterDeps, UpdateContext, + backup_binary, cleanup_backup, clear_aside_binary, download_and_write_binary, + log_update_survivors, restore_from_backup, ToolUpdater, ToolUpdaterDeps, UpdateContext, }; use crate::models::{DownloadConfiguration, Installation, InstalledTool}; #[cfg(target_os = "macos")] -use crate::platform::remove_app_bundle_path; -#[allow(unused_imports)] // binary_writer used by macos-only app-bundle path -use crate::platform::{binary_writer, system_service, DirectoryManager}; +use crate::platform::{binary_writer, remove_app_bundle_path}; +use crate::platform::{system_service, DirectoryManager}; pub struct ServiceToolUpdater { deps: ToolUpdaterDeps, @@ -44,10 +43,58 @@ impl ServiceToolUpdater { } /// Check if download config targets an .app bundle - #[allow(dead_code)] // called only by the macos app-bundle path + #[cfg(target_os = "macos")] fn is_app_bundle_download(config: &DownloadConfiguration) -> bool { config.target_file_name.contains(".app/") } + + /// Best-effort bounce of a service still executing the pre-update image; never fails the update. + #[cfg(target_os = "windows")] + async fn remediate_orphaned_service( + &self, + tool: &InstalledTool, + service_name: &str, + exec_path: &std::path::Path, + ) { + let tool_agent_id = &tool.tool_agent_id; + + if let Err(e) = self + .deps + .tool_kill_service + .stop_installed_tool(tool, false) + .await + { + warn!(tool_id = %tool_agent_id, "Orphan remediation: service stop failed: {:#}", e); + } + if let Err(e) = self.deps.tool_kill_service.stop_tool(tool_agent_id).await { + warn!(tool_id = %tool_agent_id, "Orphan remediation: process kill failed: {:#}", e); + } + + // Only abstain when a tool process truly survives; a non-process lock holder (e.g. AV scan) must not block the start. + if !clear_aside_binary(exec_path, tool_agent_id).await { + if self + .deps + .tool_kill_service + .is_installed_tool_running(tool) + .await + { + tracing::error!(tool_id = %tool_agent_id, + "Orphan remediation: pre-update process would not die — leaving service stopped until it exits"); + return; + } + warn!(tool_id = %tool_agent_id, + "Orphan remediation: .old still locked but no tool process is running — starting the service anyway"); + } + + match system_service::start_service(service_name).await { + Ok(()) => { + info!(tool_id = %tool_agent_id, "Service {service_name} restarted on the updated binary") + } + Err(e) => { + tracing::error!(tool_id = %tool_agent_id, "Orphan remediation: failed to start service: {:#}", e) + } + } + } } #[async_trait] @@ -76,6 +123,8 @@ impl ToolUpdater for ServiceToolUpdater { tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; let exec_path = self.resolve_executable_path(tool); + clear_aside_binary(&exec_path, tool_agent_id).await; + log_update_survivors(&self.deps, tool).await; // Skip backup for .app bundles on macOS - they're protected and too large let backup_path = if DirectoryManager::is_app_bundle_path(&exec_path) { @@ -139,6 +188,25 @@ impl ToolUpdater for ServiceToolUpdater { info!(tool_id = %tool_agent_id, "Finalizing Service tool update"); if let Installation::Service { service_name, .. } = &tool.installation { + // A locked .old means a pre-update process is still executing, whatever SCM reports — remediate before any start. + #[cfg(target_os = "windows")] + { + let exec_path = self.resolve_executable_path(tool); + if !clear_aside_binary(&exec_path, tool_agent_id).await { + tracing::error!(tool_id = %tool_agent_id, + "A pre-update process still holds the old {service_name} binary — remediating before start"); + self.remediate_orphaned_service(tool, service_name, &exec_path) + .await; + cleanup_backup(ctx.backup_path.as_ref(), tool_agent_id).await; + return Ok(()); + } + if system_service::service_not_stopped(service_name) { + // Benign: SCM recovery already restarted it on the new binary; start_service below no-ops on RUNNING. + info!(tool_id = %tool_agent_id, + "Service {service_name} already active with the updated binary"); + } + } + info!(tool_id = %tool_agent_id, "Starting service: {}", service_name); system_service::start_service(service_name) .await diff --git a/clients/openframe-client/src/platform/tool_updater/standard.rs b/clients/openframe-client/src/platform/tool_updater/standard.rs index 5c0e416e2..ae4852204 100644 --- a/clients/openframe-client/src/platform/tool_updater/standard.rs +++ b/clients/openframe-client/src/platform/tool_updater/standard.rs @@ -3,8 +3,8 @@ use async_trait::async_trait; use tracing::info; use super::{ - backup_binary, cleanup_backup, download_and_write_binary, restore_from_backup, ToolUpdater, - ToolUpdaterDeps, UpdateContext, + backup_binary, cleanup_backup, clear_aside_binary, download_and_write_binary, + log_update_survivors, restore_from_backup, ToolUpdater, ToolUpdaterDeps, UpdateContext, }; use crate::models::{DownloadConfiguration, Installation, InstalledTool}; @@ -32,6 +32,9 @@ impl ToolUpdater for StandardToolUpdater { .with_context(|| format!("Failed to stop tool: {}", tool_agent_id))?; let agent_path = self.deps.directory_manager.get_agent_path(tool_agent_id); + clear_aside_binary(&agent_path, tool_agent_id).await; + log_update_survivors(&self.deps, tool).await; + let backup_path = backup_binary(&agent_path, tool_agent_id).await?; Ok(UpdateContext { diff --git a/clients/openframe-client/src/platform/uninstall.rs b/clients/openframe-client/src/platform/uninstall.rs index 4ae76cd54..49565a815 100644 --- a/clients/openframe-client/src/platform/uninstall.rs +++ b/clients/openframe-client/src/platform/uninstall.rs @@ -13,6 +13,76 @@ const SERVICE_NAME: &str = "client"; const DISPLAY_NAME: &str = "OpenFrame Client Service"; const DESCRIPTION: &str = "OpenFrame client service for remote management and monitoring"; +/// CLI subcommand the detached process runs to remove the client. +const UNINSTALL_SUBCOMMAND: &str = "uninstall"; + +/// Spawn a detached `openframe-client uninstall` that survives this service being stopped. +/// Self-uninstall stops the `com.openframe.client` service (our own process), so it must run +/// out-of-process. macOS + Windows only — Linux client self-uninstall is unsupported. +pub fn spawn_detached_uninstall(install_path: &Path) -> Result<()> { + #[cfg(target_os = "macos")] + { + spawn_detached_uninstall_macos(install_path) + } + #[cfg(target_os = "windows")] + { + spawn_detached_uninstall_windows(install_path) + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + let _ = install_path; + anyhow::bail!("client self-uninstall is not supported on this platform") + } +} + +/// Detached child in its own session (setsid) so it outlives launchd stopping our own +/// `com.openframe.client` service — no launchd job/plist needed. +#[cfg(target_os = "macos")] +fn spawn_detached_uninstall_macos(install_path: &Path) -> Result<()> { + use std::os::unix::process::CommandExt; + use std::process::{Command, Stdio}; + + let child = unsafe { + Command::new(install_path) + .arg(UNINSTALL_SUBCOMMAND) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + // SAFETY: setsid() is async-signal-safe; it moves the child into a new session so + // launchd stopping our service can't signal it away mid-uninstall. + .pre_exec(|| { + if libc::setsid() == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }) + .spawn() + } + .context("Failed to spawn detached self-uninstall process")?; + + info!("Self-uninstall process launched (PID: {})", child.id()); + Ok(()) +} + +/// Detached child (inherits LocalSystem) so it survives the SCM stopping our service. +#[cfg(target_os = "windows")] +fn spawn_detached_uninstall_windows(install_path: &Path) -> Result<()> { + use std::os::windows::process::CommandExt; + use std::process::Command; + + const CREATE_NO_WINDOW: u32 = 0x0800_0000; + const DETACHED_PROCESS: u32 = 0x0000_0008; + + let child = Command::new(install_path) + .arg(UNINSTALL_SUBCOMMAND) + .creation_flags(CREATE_NO_WINDOW | DETACHED_PROCESS) + .spawn() + .context("Failed to spawn detached self-uninstall process")?; + + info!("Self-uninstall process launched (PID: {})", child.id()); + Ok(()) +} + pub fn orbit_dir() -> std::path::PathBuf { #[cfg(target_os = "windows")] { diff --git a/clients/openframe-client/src/platform/update_scripts/mod.rs b/clients/openframe-client/src/platform/update_scripts/mod.rs index 132c4bbc0..8291bef1d 100644 --- a/clients/openframe-client/src/platform/update_scripts/mod.rs +++ b/clients/openframe-client/src/platform/update_scripts/mod.rs @@ -11,29 +11,5 @@ pub use windows::UPDATE_SCRIPT_WINDOWS; pub use macos::{UPDATER_PLIST_TEMPLATE, UPDATE_SCRIPT_MACOS}; #[cfg(test)] -mod tests { - // Windows PowerShell 5.1 reads BOM-less script files as ANSI: a multi-byte - // UTF-8 character can decode into a smart quote (e.g. 0x94 from an em-dash) - // that terminates a string early and structurally breaks the script. - #[cfg(target_os = "windows")] - #[test] - fn windows_update_script_is_ascii() { - assert!( - super::windows::UPDATE_SCRIPT_WINDOWS.is_ascii(), - "UPDATE_SCRIPT_WINDOWS must stay pure ASCII" - ); - } - - #[cfg(target_os = "macos")] - #[test] - fn macos_update_script_is_ascii() { - assert!( - super::macos::UPDATE_SCRIPT_MACOS.is_ascii(), - "UPDATE_SCRIPT_MACOS must stay pure ASCII" - ); - assert!( - super::macos::UPDATER_PLIST_TEMPLATE.is_ascii(), - "UPDATER_PLIST_TEMPLATE must stay pure ASCII" - ); - } -} +#[path = "mod_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/platform/update_scripts/mod_tests.rs b/clients/openframe-client/src/platform/update_scripts/mod_tests.rs new file mode 100644 index 000000000..22c0265ba --- /dev/null +++ b/clients/openframe-client/src/platform/update_scripts/mod_tests.rs @@ -0,0 +1,24 @@ +// Windows PowerShell 5.1 reads BOM-less script files as ANSI: a multi-byte +// UTF-8 character can decode into a smart quote (e.g. 0x94 from an em-dash) +// that terminates a string early and structurally breaks the script. +#[cfg(target_os = "windows")] +#[test] +fn windows_update_script_is_ascii() { + assert!( + super::windows::UPDATE_SCRIPT_WINDOWS.is_ascii(), + "UPDATE_SCRIPT_WINDOWS must stay pure ASCII" + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn macos_update_script_is_ascii() { + assert!( + super::macos::UPDATE_SCRIPT_MACOS.is_ascii(), + "UPDATE_SCRIPT_MACOS must stay pure ASCII" + ); + assert!( + super::macos::UPDATER_PLIST_TEMPLATE.is_ascii(), + "UPDATER_PLIST_TEMPLATE must stay pure ASCII" + ); +} diff --git a/clients/openframe-client/src/service.rs b/clients/openframe-client/src/service.rs index 26271975c..80f20cde2 100644 --- a/clients/openframe-client/src/service.rs +++ b/clients/openframe-client/src/service.rs @@ -6,8 +6,10 @@ use tracing::{error, info, warn}; use crate::installation_initial_config_service::{ InstallConfigParams, InstallationInitialConfigService, }; +use crate::platform::machine_info_persistence::{self, PersistedMachineInfo}; use crate::platform::permissions::{Capability, PermissionUtils}; use crate::service_adapter::{CrossPlatformServiceManager, RecoveryConfig, ServiceConfig}; +use crate::services::{AgentConfigurationService, AgentRegistrationService}; use crate::{platform::DirectoryManager, Client}; #[cfg(windows)] @@ -187,6 +189,41 @@ impl Service { if Self::is_installed() { info!("Existing Installation Detected\n"); info!("An existing OpenFrame installation was found\n"); + + // Pre-1.0.0 clients never wrote the machine-info store; backfill it before uninstall wipes their agent config. + info!("Checking if persisted machine info exists..."); + let persisted_machine_info = AgentRegistrationService::read_persisted_credentials() + .await + .unwrap_or_else(|e| { + warn!("Failed to read persisted machine info: {}", e); + None + }); + if persisted_machine_info.is_none() { + info!("No persisted machine info found, trying to read from agent config..."); + match AgentConfigurationService::new(DirectoryManager::new()) + .and_then(|config_service| config_service.get_registration_credentials()) + { + Ok((machine_id, client_secret)) + if !machine_id.trim().is_empty() && !client_secret.trim().is_empty() => + { + let machine_info = PersistedMachineInfo { + machine_id, + client_secret, + }; + match machine_info_persistence::write(&machine_info) { + Ok(()) => info!("Machine info persisted successfully"), + Err(e) => error!("Failed to persist machine info: {}", e), + } + } + Ok(_) => { + info!("Agent config has no registration credentials, skipping backfill") + } + Err(e) => warn!("Failed to read machine info from agent config: {}", e), + } + } else { + info!("Persisted machine info was found"); + } + info!("To proceed with the new installation, the old version must be removed\n"); info!("Uninstalling existing installation..."); diff --git a/clients/openframe-client/src/service_adapter.rs b/clients/openframe-client/src/service_adapter.rs index 7726dab8e..a4900ea12 100644 --- a/clients/openframe-client/src/service_adapter.rs +++ b/clients/openframe-client/src/service_adapter.rs @@ -2,8 +2,8 @@ use anyhow::{Context, Result}; #[allow(unused_imports)] // Dictionary used by macos-only plist config use plist::Dictionary; use service_manager::{ - ServiceInstallCtx, ServiceLabel, ServiceManager, ServiceStartCtx, ServiceStopCtx, - ServiceUninstallCtx, + RestartPolicy, ServiceInstallCtx, ServiceLabel, ServiceManager, ServiceStartCtx, + ServiceStopCtx, ServiceUninstallCtx, }; use std::ffi::OsString; use std::path::PathBuf; @@ -171,6 +171,20 @@ impl CrossPlatformServiceManager { let environment = self.config.environment_vars.clone(); // Create the installation context with full configuration + // sc.exe cannot encode restart policies (0.11 warns about it); Windows restart comes from the SCM recovery actions applied right after install. + #[cfg(windows)] + let restart_policy = RestartPolicy::Never; + #[cfg(not(windows))] + let restart_policy = if self.config.restart_on_crash { + RestartPolicy::OnFailure { + delay_secs: None, + max_retries: None, + reset_after_secs: None, + } + } else { + RestartPolicy::Never + }; + let mut ctx = ServiceInstallCtx { label: label.clone(), program: self.config.exec_path.clone(), @@ -180,7 +194,7 @@ impl CrossPlatformServiceManager { working_directory: Some(working_dir), environment: Some(environment), autostart: self.config.run_at_load, - disable_restart_on_failure: !self.config.restart_on_crash, + restart_policy, }; // Apply platform-specific configuration diff --git a/clients/openframe-client/src/services/.deactivation_service.md b/clients/openframe-client/src/services/.deactivation_service.md new file mode 100644 index 000000000..395cc6564 --- /dev/null +++ b/clients/openframe-client/src/services/.deactivation_service.md @@ -0,0 +1,54 @@ + +Handles tenant deactivation lifecycle for the OpenFrame client agent: detects gateway `410 Gone` responses, backs off, stops managed tools, and triggers a detached self-uninstall after the tenant has remained gone for a configurable wall-clock window. + +## Key Components + +| Item | Description | +|------|-------------| +| `DeactivationService` | Primary struct managing detection state, suspension flag, and command dispatch | +| `Phase` | Internal FSM states: `Healthy` → `Suspended` | +| `DeactivationCommand` | Channel messages: `StopTools`, `RestartTools`, `Uninstall` | +| `State` | Mutable inner state: consecutive 410 count, `gone_since` timestamp, backoff, uninstall flag | +| `GONE_MARKER_FILE` | Persisted RFC-3339 timestamp in the secured directory, survives restarts | + +**Key thresholds:** + +| Constant | Value | Purpose | +|----------|-------|---------| +| `STOP_TOOLS_AFTER_CONSECUTIVE_GONE` | 5 | 410s before suspension | +| `UNINSTALL_AFTER` | 2 hours | Wall-clock gone duration before self-uninstall | +| `POST_RESTART_GRACE` | 1 hour | Minimum uptime before honoring deadline | +| `PROBE_BACKOFF_MAX` | 30 minutes | Exponential backoff ceiling | + +## Usage Example + +```rust +// Construct once, share via Arc +let svc = DeactivationService::new(&directory_manager); + +// Start the supervisor after the tool manager is available +svc.clone().start(tool_run_manager); + +// Feed every gateway response status +svc.on_gateway_status(StatusCode::GONE).await; // records 410 +svc.on_gateway_status(StatusCode::OK).await; // clears state + +// Gate outbound calls while suspended +if !svc.is_suspended() { + make_outbound_call().await; +} + +// Exponential delay between suspension probes +let delay = svc.next_probe_delay().await; +tokio::time::sleep(delay).await; + +// Manually trigger uninstall (e.g. from a backend decommission message) +svc.request_uninstall().await; +``` + +## Notes + +- **Platform:** Active only on macOS and Windows (`ENABLED` compile-time flag); Linux self-uninstall is unsupported. +- **Persistence:** The `tenant_gone_since` marker file survives reboots but requires a fresh live 410 to advance toward uninstall — a stale marker alone cannot trigger removal. +- **Clock safety:** `POST_RESTART_GRACE` uses `Instant` (monotonic) to prevent a forwarded system clock from shortcutting the deadline. +- **Idempotent uninstall:** `request_uninstall` sets a boolean flag before sending the command, making all callers safe to call concurrently. \ No newline at end of file diff --git a/clients/openframe-client/src/services/.mesh_self_heal_service.md b/clients/openframe-client/src/services/.mesh_self_heal_service.md index b411d1089..3028ac349 100644 --- a/clients/openframe-client/src/services/.mesh_self_heal_service.md +++ b/clients/openframe-client/src/services/.mesh_self_heal_service.md @@ -1,29 +1,31 @@ - -Monitors the MeshCentral agent for connectivity failures and silently wedged states, automatically refreshing configuration and restarting the agent when unhealthy conditions are detected. + +Monitors the MeshCentral agent's health by continuously scanning its log file for connection failure/success markers and healing (refreshing the `.msh`, then restarting the agent) when it becomes stuck, silent, or loses its ServerID. ## Key Components -- **`MeshSelfHealService`** — Main service struct holding references to directory management, tool lifecycle services, HTTP client, and configuration providers. -- **`run()`** — Spawns the background watcher task via `tokio::spawn`. -- **`watch()`** — Core polling loop that reads new log lines every 30 seconds, tracks failure/healthy markers, detects stuck (`10 min`) and silent (`90 min`) states, and triggers healing actions under a 1-hour cooldown. -- **`restart_agent()`** — Delegates to `ToolRestartService::restart_guarded`; falls back to a process kill if the agent is absent from the registry. -- **`try_refresh_msh()`** — Fetches a fresh `.msh` config from the server's `/generate-msh` endpoint, performs an atomic write (`tmp` → rename) only when `MeshID` or `ServerID` has changed. -- **`current_msh_missing_serverid()`** — Quick check that gates an immediate heal when the on-disk `.msh` lacks a `ServerID` field. +| Item | Description | +|------|-------------| +| `MeshSelfHealService` | Main service struct holding references to directory management, tool lifecycle, config, and an HTTP client | +| `run()` | Spawns the background watcher loop as a detached Tokio task | +| `watch()` | Core polling loop — reads new log lines every 30s, tracks health state, and triggers heal actions | +| `heal()` | Single heal path used by every unhealthy branch (missing ServerID, stuck, silent): refreshes the `.msh`, then restarts the agent so it re-imports it | +| `restart_agent()` | Attempts a guarded restart via `ToolRestartService`, falling back to a process kill if the agent is unregistered | +| `try_refresh_msh()` | Fetches a fresh `.msh` config from `/generate-msh` and atomically rewrites the file if `MeshID` or `ServerID` changed | +| `current_msh_missing_serverid()` | Returns `true` when the on-disk `.msh` lacks a `ServerID` (agent cannot authenticate) | -### Key Constants +### Timing Constants | Constant | Value | Purpose | -|---|---|---| -| `POLL_INTERVAL` | 30 s | Log scan frequency | -| `STUCK_DURATION` | 10 min | Max time in failure state before restart | -| `SILENCE_DURATION` | 90 min | Max log inactivity before restart | -| `ACTION_COOLDOWN` | 60 min | Minimum gap between heal attempts | -| `HTTP_TIMEOUT` | 30 s | Timeout for `/generate-msh` fetch | +|----------|-------|---------| +| `POLL_INTERVAL` | 30s | Log scan frequency | +| `STUCK_DURATION` | 10m | Max time a failure marker may persist without a healthy connect | +| `SILENCE_DURATION` | 90m | Max total log silence before assuming the agent is dead | +| `ACTION_COOLDOWN` | 60m | Minimum gap between heal attempts | ## Usage Example ```rust -let svc = MeshSelfHealService::new( +let heal = MeshSelfHealService::new( directory_manager, installed_tools, tool_kill, @@ -31,10 +33,11 @@ let svc = MeshSelfHealService::new( initial_config, agent_config, tool_run_manager, + Arc::clone(&deactivation), ); -// Spawns the background watcher; returns immediately. -svc.run().await?; +// Spawns background watcher; returns immediately. +heal.run().await?; ``` -The watcher is entirely self-contained after `run()` returns. Host suspension is detected via `Instant` drift and resets all timers to prevent spurious restarts after a sleep/resume cycle. \ No newline at end of file +The watcher exits only on an unexpected panic; the `run()` caller logs an error if that occurs. The service skips all heal logic while `DeactivationService::is_suspended()` is true (tenant offboarded) and resets timers after a detected host suspend to avoid false-positive restarts. \ No newline at end of file diff --git a/clients/openframe-client/src/services/.result_outbox_run_manager.md b/clients/openframe-client/src/services/.result_outbox_run_manager.md new file mode 100644 index 000000000..376b17e91 --- /dev/null +++ b/clients/openframe-client/src/services/.result_outbox_run_manager.md @@ -0,0 +1,51 @@ + +Manages a background task that periodically flushes pending outbox entries from a durable result store by publishing them to a message broker, implementing a reliable outbox pattern for result delivery. + +## Key Components + +### `ResultOutboxRunManager` +The primary struct holding shared references to: +- `store` — `Arc`: durable store of pending result payloads +- `publisher` — `Arc

`: message publisher implementing `ResultPublisher` +- `notify` — `Arc`: allows external callers to trigger an immediate flush + +### `new(store, publisher, notify) -> Self` +Constructor that takes shared ownership of each dependency. + +### `start(&self)` +Spawns a Tokio background task that: +1. **No-ops** if the store is not durable (`store.enabled()` returns false) +2. **Wakes** on either a periodic interval (`OUTBOX_FLUSH_INTERVAL_SECS`) or an explicit `Notify` signal +3. **Iterates** over pending `(key, subject)` pairs from the store +4. **Skips** entries exceeding `OUTBOX_MAX_PAYLOAD_BYTES` to prevent blocking healthy results +5. **Publishes** each payload with a `FLUSH_PUBLISH_TIMEOUT_SECS` deadline, removing the entry on success +6. **Breaks** the inner loop and retries on the next tick if any publish or load error occurs + +## Usage Example + +```rust +let manager = ResultOutboxRunManager::new( + Arc::clone(&result_store), + Arc::clone(&nats_publisher), + Arc::clone(¬ify), +); + +// Starts the background flush loop; call notify.notify_one() +// elsewhere to trigger an immediate flush without waiting for +// the next interval tick. +manager.start(); +``` + +## Configuration + +All tuning constants are sourced from `update_config`: + +| Constant | Purpose | +|---|---| +| `OUTBOX_FLUSH_INTERVAL_SECS` | Polling interval between flush attempts | +| `FLUSH_PUBLISH_TIMEOUT_SECS` | Per-message publish deadline | +| `OUTBOX_MAX_PAYLOAD_BYTES` | Maximum allowed payload size per entry | + +## Source + +[`result_outbox_run_manager.rs`](https://github.com/flamingo-stack/openframe-oss-tenant/blob/main/result_outbox_run_manager.rs) \ No newline at end of file diff --git a/clients/openframe-client/src/services/.result_store.md b/clients/openframe-client/src/services/.result_store.md new file mode 100644 index 000000000..8506dba21 --- /dev/null +++ b/clients/openframe-client/src/services/.result_store.md @@ -0,0 +1,59 @@ + +Persistent result store for RMM script execution outcomes, using an embedded [redb](https://github.com/cberner/redb) database to provide durable journal and outbox semantics across agent restarts. + +## Key Components + +### `ResultPublisher` trait +Async trait for publishing serialized result bytes to a named subject (e.g., a NATS subject). Implementors handle the actual transport layer. + +### `JournalRecord` +Tracks in-flight script executions. Records whether execution has `started` (used to distinguish interrupted vs. never-started jobs during recovery). + +### `ResultStore` +Core struct wrapping an optional `Arc`. Degrades gracefully — if the database file is corrupt, it is quarantined (renamed to `.redb.corrupt`) and a fresh DB is opened. If that also fails, the store runs in a disabled/non-durable mode. + +**Key methods:** + +| Method | Description | +|---|---| +| `open_or_degrade` | Opens or creates the redb database with automatic corruption recovery | +| `journal_batch` | Atomically inserts a batch of `JournalRecord` entries if the execution ID is new | +| `journal_mark_started` | Marks a journal entry as started before script execution begins | +| `complete` | Moves a result from journal → outbox (meta + payload tables), then prunes if over capacity | +| `recover` | On startup, converts any unfinished journal entries into error results in the outbox | +| `encode_result` | Serializes an `RmmResult` to JSON, truncating `stdout`/`stderr` to fit `OUTBOX_MAX_PAYLOAD_BYTES` | +| `pending_keys` | Lists outbox entries awaiting publication | +| `load_payload` / `remove` | Fetch and acknowledge delivered outbox payloads | +| `prune_oldest` | Drops the oldest entries when outbox exceeds `OUTBOX_MAX_ENTRIES` | + +### Helper functions +- `entry_key(execution_id, script_id)` — Constructs a composite key for multi-script batches. +- `now_secs()` — Returns current Unix timestamp in seconds. + +## Usage Example + +```rust +use std::path::PathBuf; + +let store = ResultStore::open_or_degrade(PathBuf::from("/var/lib/agent/results.redb")); + +// On startup: recover interrupted executions into the outbox +let recovered = store.recover().await?; + +// Before executing a script: write a journal entry +let key = entry_key(&execution_id, Some(&script_id)); +store.journal_batch(execution_id.clone(), vec![(key.clone(), record)]).await?; +store.journal_mark_started(key.clone()).await?; + +// After execution: move result to outbox +let bytes = ResultStore::encode_result(&rmm_result); +store.complete(key, "results.machine-1".to_string(), bytes).await?; + +// Drain outbox via publisher +for (key, subject) in store.pending_keys().await? { + if let Some(payload) = store.load_payload(key.clone()).await? { + publisher.publish_raw(&subject, &payload).await?; + store.remove(key).await?; + } +} +``` \ No newline at end of file diff --git a/clients/openframe-client/src/services/.tool_connection_processing_manager.md b/clients/openframe-client/src/services/.tool_connection_processing_manager.md index cc1387ede..ec370e61d 100644 --- a/clients/openframe-client/src/services/.tool_connection_processing_manager.md +++ b/clients/openframe-client/src/services/.tool_connection_processing_manager.md @@ -1,73 +1,58 @@ - -Manages the lifecycle of tool connection processing for installed tools, coordinating agent ID resolution, connection publishing, and retry logic with backoff strategies. + +Manages the lifecycle of tool connection processing, orchestrating agent ID resolution, command execution, and connection publishing for installed MSP tools. ## Key Components -### `ToolConnectionProcessingManager` (struct) -The central manager that orchestrates tool connection setup. Holds shared state via `Arc>>` to track which tools are currently being processed, preventing duplicate processing. +### `ToolConnectionProcessingManager` -### Key Methods +Main struct coordinating tool connection workflows with the following dependencies: -| Method | Description | -|---|---| -| `new(...)` | Constructs the manager with all required service dependencies | -| `run()` | Processes connections for all installed tools on startup | -| `run_new_tool(installed_tool)` | Triggers connection processing for a newly installed tool | -| `try_mark_running(tool_id)` | Acquires a processing lock for a tool; returns `false` if already running | -| `clear_running_tool(tool_id)` | Releases the processing lock after completion or failure | -| `process_tool(tool)` | Spawns an async task that resolves the agent ID and publishes the tool connection | +- `InstalledToolsService` – retrieves installed tools +- `ToolCommandParamsResolver` – resolves command argument placeholders +- `ToolConnectionMessagePublisher` – publishes connection events +- `AgentConfigurationService` – provides machine identity +- `ToolConnectionService` – persists connection records +- `ToolRunManager` – detects in-progress tool updates +- `running_tools: Arc>>` – deduplicates concurrent processing +- `wake_signals: Arc>>>` – lets `run_new_tool` nudge a live loop into an immediate re-publish -### Constants +### Key Constants | Constant | Value | Purpose | |---|---|---| -| `RETRY_DELAY_SECONDS` | `15` | Standard delay between retry attempts | -| `AGENT_ID_MAX_FAST_RETRIES` | `5` | Max consecutive fast failures before degraded backoff | -| `AGENT_ID_DEGRADED_BACKOFF_SECONDS` | `300` | Backoff delay once resolution is treated as degraded | +| `RETRY_DELAY_SECONDS` | `15` | Standard retry interval | +| `AGENT_ID_COMMAND_TIMEOUT_SECONDS` | `15` | agentId command timeout; the spawned process is killed on expiry | +| `AGENT_ID_MAX_FAST_RETRIES` | `5` | Failures before degraded backoff | +| `AGENT_ID_DEGRADED_BACKOFF_SECONDS` | `300` | Backoff delay after repeated failures | +| `REPUBLISH_INTERVAL_SECONDS` | `3600` | Cadence of the periodic re-resolve + re-publish | + +### Public Methods + +- **`run()`** – Spawns a resolve/publish loop for every installed tool on startup, skipping in-flight tools +- **`run_new_tool(installed_tool)`** – Processes a single newly installed tool; if its loop is already alive (reinstall), wakes it for an immediate re-publish instead + +### Private Methods + +- **`try_mark_running(tool_id)`** – Atomically marks a tool as running and registers its wake handle, returning it (or `None` if already in flight) +- **`process_tool(tool)`** – Spawns a loop that resolves the agent ID via command execution (15s timeout, `kill_on_drop` so a timed-out command can't leak a live process), publishes and persists the `ToolConnection`, then repeats hourly (or when woken) so a re-keyed agent heals without a restart; exits when the tool is uninstalled ## Usage Example ```rust let manager = ToolConnectionProcessingManager::new( installed_tools_service, - params_processor, - tool_connection_publisher, + params_resolver, + connection_publisher, config_service, - tool_connection_service, - tool_run_manager, + connection_service, + run_manager, ); -// Process all existing tools on startup +// On agent startup – process all installed tools manager.run().await?; -// Handle a newly installed tool at runtime -manager.run_new_tool(installed_tool).await?; - -// Release a tool's processing lock externally if needed -manager.clear_running_tool("my-tool-id").await; -``` - -## Retry & Backoff Behavior - -The `process_tool` loop implements a two-tier retry strategy: - -```mermaid -graph TD - A[Start process_tool] --> B{Tool updating?} - B -->|Yes| C[Wait 15s] - C --> B - B -->|No| D[Resolve agent ID via command] - D --> E{Success?} - E -->|Yes| F[Publish tool connection] - E -->|No| G{Failures exceeded max fast retries?} - G -->|No| H[Wait 15s, retry] - G -->|Yes| I[Wait 300s degraded backoff, retry] - H --> D - I --> D - F --> J{Published?} - J -->|No| K[Wait 15s, retry publish] - K --> F - J -->|Yes| L[Save ToolConnection record] +// On new tool installation event +manager.run_new_tool(newly_installed_tool).await?; ``` -The manager defers processing when a tool is mid-update (`ToolRunManager.is_updating`), and escalates to a 5-minute backoff after `AGENT_ID_MAX_FAST_RETRIES` consecutive agent ID resolution failures to prevent tight retry loops on persistently unhealthy agents. \ No newline at end of file +The agent ID resolution loop defers processing while a tool update is in progress, backs off exponentially after `AGENT_ID_MAX_FAST_RETRIES` consecutive failures, and transitions to a 5-minute degraded interval to avoid tight spinning on persistently unhealthy agents. \ No newline at end of file diff --git a/clients/openframe-client/src/services/.tool_connection_service.md b/clients/openframe-client/src/services/.tool_connection_service.md index 7d55a03cb..9ef4a0de9 100644 --- a/clients/openframe-client/src/services/.tool_connection_service.md +++ b/clients/openframe-client/src/services/.tool_connection_service.md @@ -1,43 +1,43 @@ - -Manages persistent storage of `ToolConnection` records in a JSON file located in the secured application directory, providing CRUD operations keyed on `tool_agent_id`. + +A service for managing persistent storage of tool connections in JSON format, handling CRUD operations for agent tool configurations. ## Key Components -| Item | Description | -|---|---| -| `ToolConnectionService` | Cloneable service struct holding the path to `tool_connections.json` | -| `new()` | Constructs the service, resolving the secured directory via `DirectoryManager` and ensuring it exists | -| `save()` | Upserts a `ToolConnection` — updates in place if `tool_agent_id` matches, otherwise appends | -| `get_all()` | Reads and deserializes all connections; returns an empty `Vec` if the file does not yet exist | -| `exists_by_tool_agent_id()` | Returns `true` if any stored connection matches the given `tool_agent_id` | -| `delete_by_tool_agent_id()` | Removes the matching connection and persists; returns `true` if a record was deleted | -| `persist()` | Private helper that serializes the full list to pretty-printed JSON and writes it atomically | +- **`ToolConnectionService`** - Main service struct that manages tool connection persistence +- **`new()`** - Creates service instance with secured directory storage path +- **`save()`** - Upserts tool connections (updates existing or creates new) +- **`get_all()`** - Retrieves all stored tool connections +- **`delete_by_tool_agent_id()`** - Removes connection by agent ID +- **`persist()`** - Internal method for writing connections to JSON file ## Usage Example ```rust +use anyhow::Result; use crate::platform::directories::DirectoryManager; -use crate::services::tool_connection_service::ToolConnectionService; use crate::models::tool_connection::ToolConnection; -let dir_manager = DirectoryManager::new()?; -let svc = ToolConnectionService::new(dir_manager)?; - -// Upsert a connection -let conn = ToolConnection { - tool_agent_id: "agent-123".to_string(), - // ...other fields -}; -svc.save(conn).await?; - -// Check existence -let found = svc.exists_by_tool_agent_id("agent-123").await?; - -// List all -let all = svc.get_all().await?; - -// Remove by ID -let deleted = svc.delete_by_tool_agent_id("agent-123").await?; +async fn manage_tool_connections() -> Result<()> { + let dir_manager = DirectoryManager::new()?; + let service = ToolConnectionService::new(dir_manager)?; + + // Create a new tool connection + let connection = ToolConnection { + tool_agent_id: "agent_123".to_string(), + // ... other fields + }; + + // Save the connection + service.save(connection).await?; + + // Get all connections + let connections = service.get_all().await?; + + // Delete connection + let deleted = service.delete_by_tool_agent_id("agent_123").await?; + + Ok(()) +} ``` -> **Storage:** All data is written to `/tool_connections.json` as a pretty-printed JSON array. No database dependency is required. \ No newline at end of file +The service uses file-based JSON storage in a secured directory, automatically handling serialization/deserialization and directory creation. All operations are async and return `Result` for proper error handling. \ No newline at end of file diff --git a/clients/openframe-client/src/services/agent_configuration_service.rs b/clients/openframe-client/src/services/agent_configuration_service.rs index 34a5e076c..585691ab9 100644 --- a/clients/openframe-client/src/services/agent_configuration_service.rs +++ b/clients/openframe-client/src/services/agent_configuration_service.rs @@ -61,6 +61,11 @@ impl AgentConfigurationService { Ok(config.machine_id.clone()) } + pub fn get_registration_credentials(&self) -> Result<(String, String)> { + let config = self.get()?; + Ok((config.machine_id, config.client_secret)) + } + pub async fn get_client_credentials(&self) -> Result<(String, String)> { let config = self.get()?; Ok((config.client_id.clone(), config.client_secret.clone())) diff --git a/clients/openframe-client/src/services/agent_registration_service.rs b/clients/openframe-client/src/services/agent_registration_service.rs index f9d573dd9..adfe60ab1 100644 --- a/clients/openframe-client/src/services/agent_registration_service.rs +++ b/clients/openframe-client/src/services/agent_registration_service.rs @@ -78,7 +78,7 @@ impl AgentRegistrationService { Ok(response) } - async fn read_persisted_credentials() -> Result> { + pub async fn read_persisted_credentials() -> Result> { let mut attempt = 0; loop { match machine_info_persistence::read() { @@ -143,6 +143,10 @@ impl AgentRegistrationService { fn build_registration_request(&self) -> Result { let hostname = self.device_data_fetcher.get_hostname().unwrap_or_default(); + if hostname.is_empty() { + warn!("Could not resolve any hostname — registering with an empty one"); + } + info!("Registering with hostname: '{}'", hostname); let agent_version = self .device_data_fetcher .get_agent_version() diff --git a/clients/openframe-client/src/services/deactivation_service.rs b/clients/openframe-client/src/services/deactivation_service.rs new file mode 100644 index 000000000..bdd08e486 --- /dev/null +++ b/clients/openframe-client/src/services/deactivation_service.rs @@ -0,0 +1,329 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use chrono::{DateTime, Utc}; +use reqwest::StatusCode; +use tokio::sync::mpsc; +use tokio::sync::Mutex as AsyncMutex; +use tracing::{error, info, warn}; + +use crate::platform::DirectoryManager; +use crate::services::tool_run_manager::ToolRunManager; + +/// Consecutive `410 Gone` responses before we stop tools and gate outbound calls. +const STOP_TOOLS_AFTER_CONSECUTIVE_GONE: u32 = 5; +/// First delay between probes once suspended. +const PROBE_BACKOFF_INITIAL: Duration = Duration::from_secs(60); +/// Cap on the exponential probe backoff. +const PROBE_BACKOFF_MAX: Duration = Duration::from_secs(30 * 60); +/// Total time the tenant must stay gone (since the first 410) before self-uninstall. +const UNINSTALL_AFTER: Duration = Duration::from_secs(2 * 60 * 60); +/// After any (re)start, require this much fresh confirmation before uninstalling, so a +/// reboot near the deadline can't let a single 410 wipe the device. +const POST_RESTART_GRACE: Duration = Duration::from_secs(60 * 60); +/// Persisted marker (in the secured dir) holding the first-410 timestamp across restarts. +const GONE_MARKER_FILE: &str = "tenant_gone_since"; + +/// Feature is macOS + Windows only (Linux client self-uninstall is unsupported). +const ENABLED: bool = cfg!(any(target_os = "macos", target_os = "windows")); + +/// Self-uninstall is terminal — retry the detached spawn before giving up. +const UNINSTALL_SPAWN_ATTEMPTS: u32 = 3; +const UNINSTALL_SPAWN_RETRY_DELAY: Duration = Duration::from_secs(30); + +/// Action the internal supervisor executes, kept off the detection path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DeactivationCommand { + StopTools, + RestartTools, + Uninstall, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { + Healthy, + Suspended, +} + +struct State { + phase: Phase, + consecutive_gone: u32, + /// First 410 of the current gone episode (persisted, reset on recovery). + gone_since: Option>, + probe_backoff: Duration, + uninstall_triggered: bool, + /// Monotonic start of this run. The 24h deadline itself is wall-clock ([`State::gone_since`]) + /// because it must track real time across restarts; this monotonic grace floor bounds it so a + /// wrong or fast-forwarded system clock can't shortcut the uninstall. `Instant` pauses during + /// OS suspend on macOS/Linux (not Windows), which here can only ever delay uninstall. + process_started: Instant, +} + +/// Detects the gateway's `410 Gone` (tenant deleted), backs off, stops tools, and — after +/// the tenant has stayed gone for [`UNINSTALL_AFTER`] — triggers client self-uninstall. +pub struct DeactivationService { + suspended: AtomicBool, + state: AsyncMutex, + commands_tx: mpsc::UnboundedSender, + commands_rx: std::sync::Mutex>>, + secured_dir: PathBuf, +} + +impl DeactivationService { + pub fn new(directory_manager: &DirectoryManager) -> Arc { + let (commands_tx, commands_rx) = mpsc::unbounded_channel(); + let secured_dir = directory_manager.secured_dir().to_path_buf(); + let gone_since = if ENABLED { + load_marker(&secured_dir) + } else { + None + }; + + if let Some(ts) = gone_since { + warn!( + target: "deactivation", + reason = "marker_loaded", + gone_since = %ts, + "Loaded persisted tenant-gone marker; a live 410 must re-confirm before any uninstall" + ); + } + + Arc::new(Self { + suspended: AtomicBool::new(false), + state: AsyncMutex::new(State { + phase: Phase::Healthy, + consecutive_gone: 0, + gone_since, + probe_backoff: PROBE_BACKOFF_INITIAL, + uninstall_triggered: false, + process_started: Instant::now(), + }), + commands_tx, + commands_rx: std::sync::Mutex::new(Some(commands_rx)), + secured_dir, + }) + } + + /// True while outbound calls are gated (tenant presumed gone). Always false when disabled. + pub fn is_suspended(&self) -> bool { + self.suspended.load(Ordering::Acquire) + } + + /// Feed a gateway HTTP status: `410` confirms gone, `2xx` confirms healthy, everything + /// else (5xx / timeouts / auth errors) is transient and leaves the state unchanged. + pub async fn on_gateway_status(&self, status: StatusCode) { + if !ENABLED { + return; + } + if status == StatusCode::GONE { + self.record_gone().await; + } else if status.is_success() { + self.record_healthy().await; + } + } + + /// Delay before the next suspension probe, doubling up to [`PROBE_BACKOFF_MAX`]. + pub async fn next_probe_delay(&self) -> Duration { + let mut st = self.state.lock().await; + let delay = st.probe_backoff; + st.probe_backoff = (st.probe_backoff * 2).min(PROBE_BACKOFF_MAX); + delay + } + + /// Spawn the supervisor that executes deactivation decisions off the detection path: stop / + /// restart the managed tools and launch the detached self-uninstall. Call once, after the + /// tool run manager exists. No-op if already started. + pub fn start(&self, tool_run_manager: ToolRunManager) { + let Some(mut commands) = self.commands_rx.lock().ok().and_then(|mut g| g.take()) else { + return; + }; + tokio::spawn(async move { + while let Some(cmd) = commands.recv().await { + match cmd { + DeactivationCommand::StopTools => { + if let Err(e) = tool_run_manager.stop_all().await { + error!(target: "deactivation", "failed to stop tools: {e:#}"); + } + } + DeactivationCommand::RestartTools => { + if let Err(e) = tool_run_manager.restart_all().await { + error!(target: "deactivation", "failed to restart tools: {e:#}"); + } + } + DeactivationCommand::Uninstall => Self::launch_self_uninstall().await, + } + } + }); + } + + /// Spawn the detached `openframe-client uninstall`, retrying since this is terminal. + async fn launch_self_uninstall() { + let install_path = crate::service::Service::get_install_location(); + info!(target: "deactivation", "launching client self-uninstall"); + for attempt in 1..=UNINSTALL_SPAWN_ATTEMPTS { + match crate::platform::uninstall::spawn_detached_uninstall(&install_path) { + Ok(()) => return, + Err(e) => { + error!(target: "deactivation", "self-uninstall launch attempt {attempt}/{UNINSTALL_SPAWN_ATTEMPTS} failed: {e:#}"); + tokio::time::sleep(UNINSTALL_SPAWN_RETRY_DELAY).await; + } + } + } + error!(target: "deactivation", "self-uninstall could not be launched after {UNINSTALL_SPAWN_ATTEMPTS} attempts"); + } + + async fn record_gone(&self) { + let now = Utc::now(); + let due = { + let mut st = self.state.lock().await; + if st.uninstall_triggered { + return; + } + + if st.gone_since.is_none() { + st.gone_since = Some(now); + save_marker(&self.secured_dir, now); + warn!( + target: "deactivation", + reason = "backoff_started", + "First 410 Gone from gateway (tenant may be deleted); will stop tools after {} consecutive", + STOP_TOOLS_AFTER_CONSECUTIVE_GONE + ); + } + st.consecutive_gone = st.consecutive_gone.saturating_add(1); + + if st.phase == Phase::Healthy + && st.consecutive_gone >= STOP_TOOLS_AFTER_CONSECUTIVE_GONE + { + st.phase = Phase::Suspended; + st.probe_backoff = PROBE_BACKOFF_INITIAL; + self.suspended.store(true, Ordering::Release); + warn!( + target: "deactivation", + reason = "calls_stopped", + consecutive = st.consecutive_gone, + "Tenant gone confirmed; stopping all tools and gating outbound calls to a single backoff probe" + ); + let _ = self.commands_tx.send(DeactivationCommand::StopTools); + } + + Self::uninstall_due(&st, now) + }; + + // Deadline reached — funnel through the shared entry point (after dropping the lock). + if due { + self.request_uninstall().await; + } + } + + async fn record_healthy(&self) { + let mut st = self.state.lock().await; + if st.uninstall_triggered { + return; + } + + let was_suspended = st.phase == Phase::Suspended; + let had_marker = st.gone_since.is_some(); + + st.consecutive_gone = 0; + st.gone_since = None; + st.probe_backoff = PROBE_BACKOFF_INITIAL; + if had_marker { + remove_marker(&self.secured_dir); + } + + if was_suspended { + st.phase = Phase::Healthy; + self.suspended.store(false, Ordering::Release); + warn!( + target: "deactivation", + reason = "recovered", + "Gateway healthy again (tenant restored); resuming outbound calls and restarting tools" + ); + let _ = self.commands_tx.send(DeactivationCommand::RestartTools); + } else if had_marker { + info!( + target: "deactivation", + reason = "recovered", + "Gateway healthy; cleared tenant-gone marker before tools were stopped" + ); + } + } + + /// Whether the tenant has been gone past [`UNINSTALL_AFTER`] and the post-restart grace. + /// Only ever evaluated on a fresh 410 (a network outage produces none), so a stale marker + /// alone can't trigger uninstall. + fn uninstall_due(st: &State, now: DateTime) -> bool { + if st.phase != Phase::Suspended { + return false; + } + let Some(gone_since) = st.gone_since else { + return false; + }; + if now.signed_duration_since(gone_since) + < chrono::Duration::seconds(UNINSTALL_AFTER.as_secs() as i64) + { + return false; + } + if st.process_started.elapsed() < POST_RESTART_GRACE { + info!( + target: "deactivation", + reason = "grace_deferred", + "Uninstall deadline reached but within post-restart grace; deferring self-uninstall" + ); + return false; + } + true + } + + /// Trigger the detached self-uninstall (idempotent). Public so every path funnels through the + /// same supervisor: the 410 deadline above, or a future backend "decommission" message. + pub async fn request_uninstall(&self) { + { + let mut st = self.state.lock().await; + if st.uninstall_triggered { + return; + } + st.uninstall_triggered = true; + } + error!( + target: "deactivation", + reason = "self_destroy_triggered", + "Triggering client self-uninstall" + ); + let _ = self.commands_tx.send(DeactivationCommand::Uninstall); + } +} + +fn marker_path(secured_dir: &Path) -> PathBuf { + secured_dir.join(GONE_MARKER_FILE) +} + +fn load_marker(secured_dir: &Path) -> Option> { + let raw = std::fs::read_to_string(marker_path(secured_dir)).ok()?; + match DateTime::parse_from_rfc3339(raw.trim()) { + Ok(dt) => Some(dt.with_timezone(&Utc)), + Err(e) => { + // A corrupt marker silently resets the 24h clock — leave a trail. + warn!(target: "deactivation", "Failed to parse tenant-gone marker '{}': {e:#}", raw.trim()); + None + } + } +} + +fn save_marker(secured_dir: &Path, ts: DateTime) { + if let Err(e) = std::fs::write(marker_path(secured_dir), ts.to_rfc3339()) { + warn!(target: "deactivation", "Failed to persist tenant-gone marker: {e:#}"); + } +} + +fn remove_marker(secured_dir: &Path) { + let path = marker_path(secured_dir); + if path.exists() { + if let Err(e) = std::fs::remove_file(&path) { + warn!(target: "deactivation", "Failed to remove tenant-gone marker: {e:#}"); + } + } +} diff --git a/clients/openframe-client/src/services/device_data_fetcher.rs b/clients/openframe-client/src/services/device_data_fetcher.rs index 31fd7619f..64b751460 100644 --- a/clients/openframe-client/src/services/device_data_fetcher.rs +++ b/clients/openframe-client/src/services/device_data_fetcher.rs @@ -17,14 +17,78 @@ impl DeviceDataFetcher { pub fn get_hostname(&self) -> Option { match hostname::get() { Ok(hostname) => { - let hostname_str = hostname.to_string_lossy().to_string(); - Some(hostname_str) + let hostname_str = hostname.to_string_lossy().trim().to_string(); + if Self::is_valid_hostname(&hostname_str) { + info!("Resolved hostname '{}' from kernel", hostname_str); + return Some(hostname_str); + } + warn!( + "Kernel hostname '{}' is empty or looks like junk — falling back", + hostname_str + ); } Err(e) => { warn!("Failed to get hostname: {:#}", e); - None } } + + #[cfg(target_os = "macos")] + { + if let Some(name) = Self::scutil_get("LocalHostName") { + let name = format!("{}.local", name); + if Self::is_valid_hostname(&name) { + info!("Resolved hostname '{}' from LocalHostName", name); + return Some(name); + } + warn!("LocalHostName '{}' is invalid — falling back", name); + } + if let Some(name) = Self::scutil_get("ComputerName") { + if Self::is_valid_hostname(&name) { + info!("Resolved hostname '{}' from ComputerName", name); + return Some(name); + } + warn!("ComputerName '{}' is invalid — falling back", name); + } + } + + warn!("Could not resolve any hostname"); + None + } + + fn is_valid_hostname(name: &str) -> bool { + if name.is_empty() { + return false; + } + if name.parse::().is_ok() { + return false; + } + let lower = name.to_lowercase(); + lower != "localhost" && lower != "localhost.localdomain" + } + + #[cfg(target_os = "macos")] + fn scutil_get(key: &str) -> Option { + let output = std::process::Command::new("/usr/sbin/scutil") + .args(["--get", key]) + .output() + .map_err(|e| warn!("Failed to run scutil --get {}: {:#}", key, e)) + .ok()?; + + if !output.status.success() { + warn!( + "scutil --get {} failed: {}", + key, + String::from_utf8_lossy(&output.stderr).trim() + ); + return None; + } + + let value = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if value.is_empty() { + None + } else { + Some(value) + } } pub fn get_agent_version(&self) -> Option { diff --git a/clients/openframe-client/src/services/execution_service.rs b/clients/openframe-client/src/services/execution_service.rs index 2ebea4344..92bc8ad55 100644 --- a/clients/openframe-client/src/services/execution_service.rs +++ b/clients/openframe-client/src/services/execution_service.rs @@ -76,92 +76,9 @@ impl ExecutionService { } #[cfg(all(test, unix))] -mod tests { - use super::*; - use crate::models::{PrivilegeLevel, ScriptShell}; - - fn req(code: &str) -> ExecutionRequest<'_> { - ExecutionRequest { - execution_id: "exec-1", - code, - shell: ScriptShell::Bash, - privilege: PrivilegeLevel::Admin, - args: &[], - timeout_secs: 30, - env_vars: Vec::new(), - script_id: None, - schedule_id: None, - } - } - - #[tokio::test] - async fn maps_successful_execution() { - let r = ExecutionService::new() - .execute(&req("#!/bin/sh\necho hi\n"), "machine-1") - .await; - assert_eq!(r.execution_id, "exec-1"); - assert_eq!(r.machine_id, "machine-1"); - assert_eq!(r.stdout, "hi\n"); - assert_eq!(r.exit_code, 0); - assert!(!r.timed_out); - assert!(r.error.is_none()); - } - - #[tokio::test] - async fn maps_timeout() { - let mut r = req("#!/bin/sh\nsleep 5\n"); - r.timeout_secs = 1; - let r = ExecutionService::new().execute(&r, "m").await; - assert_eq!(r.exit_code, 98); - assert!(r.timed_out); - } - - #[tokio::test] - async fn maps_spawn_failure_to_error() { - let r = ExecutionService::new() - .execute(&req("#!/nonexistent/ofcmd-bad\necho hi\n"), "m") - .await; - assert_eq!(r.exit_code, 85); - assert!(r.error.is_some()); - assert!(!r.timed_out); - } -} +#[path = "execution_service_tests.rs"] +mod tests; #[cfg(all(test, windows))] -mod windows_tests { - use super::*; - use crate::models::{PrivilegeLevel, ScriptShell}; - - fn req(shell: ScriptShell, code: &str) -> ExecutionRequest<'_> { - ExecutionRequest { - execution_id: "exec-1", - code, - shell, - privilege: PrivilegeLevel::Admin, - args: &[], - timeout_secs: 30, - env_vars: Vec::new(), - script_id: None, - schedule_id: None, - } - } - - #[tokio::test] - async fn maps_powershell_execution() { - let r = ExecutionService::new() - .execute(&req(ScriptShell::Powershell, "Write-Output hi"), "m") - .await; - assert_eq!(r.exit_code, 0, "stderr: {}", r.stderr); - assert_eq!(r.stdout.trim_end(), "hi"); - assert!(r.error.is_none()); - } - - #[tokio::test] - async fn maps_unsupported_shell_to_error() { - let r = ExecutionService::new() - .execute(&req(ScriptShell::Python, "print('x')"), "m") - .await; - assert_eq!(r.exit_code, 85); - assert!(r.error.is_some()); - } -} +#[path = "execution_service_windows_tests.rs"] +mod windows_tests; diff --git a/clients/openframe-client/src/services/execution_service_tests.rs b/clients/openframe-client/src/services/execution_service_tests.rs new file mode 100644 index 000000000..98dfd3b20 --- /dev/null +++ b/clients/openframe-client/src/services/execution_service_tests.rs @@ -0,0 +1,48 @@ +use super::*; +use crate::models::{PrivilegeLevel, ScriptShell}; + +fn req(code: &str) -> ExecutionRequest<'_> { + ExecutionRequest { + execution_id: "exec-1", + code, + shell: ScriptShell::Bash, + privilege: PrivilegeLevel::Admin, + args: &[], + timeout_secs: 30, + env_vars: Vec::new(), + script_id: None, + schedule_id: None, + } +} + +#[tokio::test] +async fn maps_successful_execution() { + let r = ExecutionService::new() + .execute(&req("#!/bin/sh\necho hi\n"), "machine-1") + .await; + assert_eq!(r.execution_id, "exec-1"); + assert_eq!(r.machine_id, "machine-1"); + assert_eq!(r.stdout, "hi\n"); + assert_eq!(r.exit_code, 0); + assert!(!r.timed_out); + assert!(r.error.is_none()); +} + +#[tokio::test] +async fn maps_timeout() { + let mut r = req("#!/bin/sh\nsleep 5\n"); + r.timeout_secs = 1; + let r = ExecutionService::new().execute(&r, "m").await; + assert_eq!(r.exit_code, 98); + assert!(r.timed_out); +} + +#[tokio::test] +async fn maps_spawn_failure_to_error() { + let r = ExecutionService::new() + .execute(&req("#!/nonexistent/ofcmd-bad\necho hi\n"), "m") + .await; + assert_eq!(r.exit_code, 85); + assert!(r.error.is_some()); + assert!(!r.timed_out); +} diff --git a/clients/openframe-client/src/services/execution_service_windows_tests.rs b/clients/openframe-client/src/services/execution_service_windows_tests.rs new file mode 100644 index 000000000..7b829f06d --- /dev/null +++ b/clients/openframe-client/src/services/execution_service_windows_tests.rs @@ -0,0 +1,35 @@ +use super::*; +use crate::models::{PrivilegeLevel, ScriptShell}; + +fn req(shell: ScriptShell, code: &str) -> ExecutionRequest<'_> { + ExecutionRequest { + execution_id: "exec-1", + code, + shell, + privilege: PrivilegeLevel::Admin, + args: &[], + timeout_secs: 30, + env_vars: Vec::new(), + script_id: None, + schedule_id: None, + } +} + +#[tokio::test] +async fn maps_powershell_execution() { + let r = ExecutionService::new() + .execute(&req(ScriptShell::Powershell, "Write-Output hi"), "m") + .await; + assert_eq!(r.exit_code, 0, "stderr: {}", r.stderr); + assert_eq!(r.stdout.trim_end(), "hi"); + assert!(r.error.is_none()); +} + +#[tokio::test] +async fn maps_unsupported_shell_to_error() { + let r = ExecutionService::new() + .execute(&req(ScriptShell::Python, "print('x')"), "m") + .await; + assert_eq!(r.exit_code, 85); + assert!(r.error.is_some()); +} diff --git a/clients/openframe-client/src/services/initial_key_service.rs b/clients/openframe-client/src/services/initial_key_service.rs index 513dd1ff8..e4e6900db 100644 --- a/clients/openframe-client/src/services/initial_key_service.rs +++ b/clients/openframe-client/src/services/initial_key_service.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use tokio::time::{sleep, Duration}; use tracing::{error, info}; +use crate::services::deactivation_service::DeactivationService; use crate::services::{AgentConfigurationService, InitialConfigurationService}; const RETRY_INTERVAL_SECS: u64 = 60; @@ -19,6 +20,7 @@ pub struct InitialKeyService { base_url: String, initial_config_service: InitialConfigurationService, agent_config_service: AgentConfigurationService, + deactivation: Arc, } impl InitialKeyService { @@ -27,12 +29,14 @@ impl InitialKeyService { base_url: String, initial_config_service: InitialConfigurationService, agent_config_service: AgentConfigurationService, + deactivation: Arc, ) -> Self { Self { http_client, base_url, initial_config_service, agent_config_service, + deactivation, } } @@ -45,6 +49,11 @@ impl InitialKeyService { tokio::spawn(async move { loop { + // Tenant gone: don't hammer the gateway; the backoff probe drives recovery. + if self.deactivation.is_suspended() { + sleep(Duration::from_secs(RETRY_INTERVAL_SECS)).await; + continue; + } match self.fetch_registration_secret().await { Ok(secret) => { if let Err(e) = self.initial_config_service.update_initial_key(secret) { diff --git a/clients/openframe-client/src/services/mesh_self_heal_service.rs b/clients/openframe-client/src/services/mesh_self_heal_service.rs index fb56c81ef..aaa1eeb36 100644 --- a/clients/openframe-client/src/services/mesh_self_heal_service.rs +++ b/clients/openframe-client/src/services/mesh_self_heal_service.rs @@ -1,11 +1,16 @@ +use std::panic::AssertUnwindSafe; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::{Duration, Instant}; use anyhow::{anyhow, Result}; +use futures::FutureExt; use tokio::time::sleep; use tracing::{debug, error, info, warn}; +use crate::models::ToolRecordState; use crate::platform::DirectoryManager; +use crate::services::deactivation_service::DeactivationService; use crate::services::tool_kill_service::ToolKillService; use crate::services::tool_restart_service::{RestartOutcome, ToolRestartService}; use crate::services::tool_run_manager::ToolRunManager; @@ -32,6 +37,8 @@ const ACTION_COOLDOWN: Duration = Duration::from_secs(60 * 60); const HTTP_TIMEOUT: Duration = Duration::from_secs(30); /// How far back to look for markers when seeding health state at startup. const TAIL_BYTES: u64 = 64 * 1024; +/// Pause before respawning a watcher that exited or panicked. +const WATCHER_RESPAWN_DELAY: Duration = Duration::from_secs(60); #[derive(Clone)] pub struct MeshSelfHealService { @@ -42,10 +49,12 @@ pub struct MeshSelfHealService { initial_config: InitialConfigurationService, agent_config: AgentConfigurationService, tool_run_manager: ToolRunManager, + deactivation: Arc, http: reqwest::Client, } impl MeshSelfHealService { + #[allow(clippy::too_many_arguments)] pub fn new( directory_manager: DirectoryManager, installed_tools: InstalledToolsService, @@ -54,6 +63,7 @@ impl MeshSelfHealService { initial_config: InitialConfigurationService, agent_config: AgentConfigurationService, tool_run_manager: ToolRunManager, + deactivation: Arc, ) -> Self { Self { directory_manager, @@ -63,6 +73,7 @@ impl MeshSelfHealService { initial_config, agent_config, tool_run_manager, + deactivation, http: reqwest::Client::builder() .timeout(HTTP_TIMEOUT) .build() @@ -73,8 +84,21 @@ impl MeshSelfHealService { pub async fn run(&self) -> Result<()> { let this = self.clone(); tokio::spawn(async move { - this.watch().await; - error!("mesh self-heal watcher exited unexpectedly"); + // Un-caught, a panic dies silently (spawn swallows it, stderr isn't shipped) — capture it into tracing and respawn. + loop { + match AssertUnwindSafe(this.clone().watch()).catch_unwind().await { + Ok(()) => error!( + "mesh self-heal watcher exited unexpectedly — respawning in {}s", + WATCHER_RESPAWN_DELAY.as_secs() + ), + Err(panic) => error!( + "mesh self-heal watcher panicked: {} — respawning in {}s", + panic_message(&*panic), + WATCHER_RESPAWN_DELAY.as_secs() + ), + } + sleep(WATCHER_RESPAWN_DELAY).await; + } }); Ok(()) } @@ -109,6 +133,11 @@ impl MeshSelfHealService { let sleep_started = Instant::now(); sleep(POLL_INTERVAL).await; + // Tenant gone: agent is stopped and /generate-msh returns 410 — don't hammer it. + if self.deactivation.is_suspended() { + continue; + } + // The sleep alone overran by far ⇒ the host was suspended (Instant counts suspend on Windows) — discard timers measured across it. if sleep_started.elapsed() > POLL_INTERVAL * 5 { stuck_since = None; @@ -153,43 +182,43 @@ impl MeshSelfHealService { let silent = last_activity.elapsed() >= SILENCE_DURATION; let msh_missing_serverid = self.current_msh_missing_serverid().await; - if msh_missing_serverid || stuck { - let reason = if msh_missing_serverid { - "current .msh has no ServerID (agent cannot authenticate the server)" - .to_string() - } else { - format!("no successful connect within {}s", STUCK_DURATION.as_secs()) - }; - warn!("meshcentral-agent unhealthy: {reason} — refreshing .msh and restarting the agent"); - - // Arm the cooldown before acting so no outcome (busy, error, no-op) can spin the loop. - last_action = Some(Instant::now()); - match self.try_refresh_msh().await { - Ok(true) => info!("mesh self-heal: refreshed .msh (NodeID preserved)"), - Ok(false) => debug!("mesh self-heal: .msh already current"), - Err(e) => { - error!("mesh self-heal: .msh refresh failed (restarting anyway): {e:#}") - } - } - self.restart_agent().await; - - stuck_since = None; - last_activity = Instant::now(); + let reason = if msh_missing_serverid { + Some("current .msh is missing or has no ServerID (agent cannot authenticate the server)".to_string()) + } else if stuck { + Some(format!( + "no successful connect within {}s", + STUCK_DURATION.as_secs() + )) } else if silent { - warn!( - "meshcentral-agent silent for {}s (last_marker_healthy={last_marker_healthy}) — restarting the agent", + Some(format!( + "silent for {}s (last_marker_healthy={last_marker_healthy})", last_activity.elapsed().as_secs() - ); + )) + } else { + None + }; + if let Some(reason) = reason { + // Arm the cooldown before acting so no outcome (busy, error, no-op) can spin the loop. last_action = Some(Instant::now()); - self.restart_agent().await; - + self.heal(&reason).await; stuck_since = None; last_activity = Instant::now(); } } } + /// Single heal path for every unhealthy branch: refresh the .msh, then restart so the agent re-imports it. + async fn heal(&self, reason: &str) { + warn!("meshcentral-agent unhealthy: {reason} — refreshing .msh and restarting the agent"); + match self.try_refresh_msh().await { + Ok(true) => info!("mesh self-heal: refreshed .msh (NodeID preserved)"), + Ok(false) => debug!("mesh self-heal: .msh already current"), + Err(e) => error!("mesh self-heal: .msh refresh failed (restarting anyway): {e:#}"), + } + self.restart_agent().await; + } + /// Restart through the shared guarded flow; a missing registry entry degrades to a process kill so the OS supervisor can relaunch. async fn restart_agent(&self) { match self.tool_restart.restart_guarded(MESH_TOOL_ID).await { @@ -209,6 +238,9 @@ impl MeshSelfHealService { /// Refresh the .msh from /generate-msh; returns true when it was rewritten. async fn try_refresh_msh(&self) -> Result { + // Resolve the target path first: it fails fast when the tool isn't installed, before any network call. + let msh_path = self.mesh_msh_path().await?; + let host = self.initial_config.get_server_url()?; let url = format!("https://{host}/tools/agent/meshcentral-server/generate-msh?host={host}"); @@ -231,7 +263,6 @@ impl MeshSelfHealService { )); } - let msh_path = self.mesh_msh_path().await?; let current = tokio::fs::read_to_string(&msh_path).await.ok(); let cur_mesh = current .as_deref() @@ -268,39 +299,86 @@ impl MeshSelfHealService { ); let tmp_path = msh_path.with_extension("msh.tmp"); + if let Some(parent) = msh_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } tokio::fs::write(&tmp_path, body.as_bytes()).await?; tokio::fs::rename(&tmp_path, &msh_path).await?; Ok(true) } + /// Unhealthy when the tool is fully installed but its .msh is absent, unreadable, or lacks a ServerID. async fn current_msh_missing_serverid(&self) -> bool { - let msh_path = match self.mesh_msh_path().await { - Ok(p) => p, - Err(_) => return false, + match self + .installed_tools + .get_by_tool_agent_id(MESH_TOOL_ID) + .await + { + Ok(Some(t)) if t.state == ToolRecordState::Installed => {} + _ => return false, + } + let msh_path = match self.find_existing_msh().await { + Ok(Some(p)) => p, + Ok(None) => { + warn!("mesh self-heal: no .msh file found for {MESH_TOOL_ID}"); + return true; + } + // Transient scan error: stay passive rather than bounce a possibly healthy agent. + Err(e) => { + debug!("mesh self-heal: cannot scan for .msh: {e:#}"); + return false; + } }; match tokio::fs::read_to_string(&msh_path).await { Ok(s) => parse_msh_field(&s, "ServerID").is_none(), - Err(_) => false, + Err(e) => { + warn!("mesh self-heal: cannot read {}: {e:#}", msh_path.display()); + true + } } } + /// Existing .msh, or the path the agent itself imports (`.msh`) so a missing file can be recreated. async fn mesh_msh_path(&self) -> Result { - self.installed_tools + if let Some(p) = self.find_existing_msh().await? { + return Ok(p); + } + let tool = self + .installed_tools .get_by_tool_agent_id(MESH_TOOL_ID) .await? .ok_or_else(|| anyhow!("{MESH_TOOL_ID} is not installed"))?; + let exe = self + .directory_manager + .get_tool_executable_path(MESH_TOOL_ID, tool.installation.executable_path()); + Ok(exe.with_extension("msh")) + } + + async fn find_existing_msh(&self) -> Result> { let dir = self.directory_manager.app_support_dir().join(MESH_TOOL_ID); - let mut rd = tokio::fs::read_dir(&dir).await?; + let mut rd = match tokio::fs::read_dir(&dir).await { + Ok(rd) => rd, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.into()), + }; while let Some(entry) = rd.next_entry().await? { let p = entry.path(); if p.extension().and_then(|e| e.to_str()) == Some("msh") { - return Ok(p); + return Ok(Some(p)); } } - Err(anyhow!("no .msh found in {}", dir.display())) + Ok(None) } } +fn panic_message(payload: &(dyn std::any::Any + Send)) -> &str { + payload + .downcast_ref::<&str>() + .copied() + .or_else(|| payload.downcast_ref::().map(String::as_str)) + .unwrap_or("") +} + fn parse_msh_field(msh: &str, key: &str) -> Option { let prefix = format!("{key}="); msh.lines() @@ -378,64 +456,5 @@ async fn read_new_lines(path: &Path, offset: &mut u64) -> Result> { } #[cfg(test)] -mod tests { - use super::*; - - const FAILED_0_0_22_NO_HTTP: &str = "Connection FAILED: No HTTP response (fd=0, status=Complete/Disconnected, authState=0, connState=0, tls=down, elapsedMs=20016, attempt=ABCD1234-2100)"; - const FAILED_0_0_22_TIMEOUT: &str = "Connection FAILED: Network timeout - server unreachable or gateway blocking (tls=down, elapsedMs=21016, attempt=ABCD1234-2101)"; - const FAILED_0_0_23_PLUS: &str = "Connection FAILED (latest attempt): No HTTP response (fd=0, status=Complete/Disconnected, authState=0, connState=0, tls=down, elapsedMs=20016, attempt=ABCD1234-2102)"; - const CORE_OK: &str = "Received CoreOk from server (coreTimeout=0x0)"; - - #[test] - fn failure_marker_matches_0_0_22_formats() { - assert!(FAILED_0_0_22_NO_HTTP.contains(FAILURE_MARKER)); - assert!(FAILED_0_0_22_TIMEOUT.contains(FAILURE_MARKER)); - } - - #[test] - fn failure_marker_matches_0_0_23_plus_format() { - assert!(FAILED_0_0_23_PLUS.contains(FAILURE_MARKER)); - } - - #[test] - fn markers_ignore_unrelated_lines() { - for line in [ - "Connection: dialing uri=wss://x.openframe.ai/ws/tools/agent/meshcentral-server/agent.ashx host=x.openframe.ai port=443 family=IPv4 ip=1.2.3.4 useproxy=0 proxy=DIRECT attempt=ABCD1234-2103 suppressed=2", - "AutoRetry Connect in 299066 milliseconds", - ] { - assert!(!line.contains(FAILURE_MARKER)); - assert!(!line.contains(HEALTHY_MARKER)); - } - } - - #[test] - fn healthy_marker_matches_core_ok() { - assert!(CORE_OK.contains(HEALTHY_MARKER)); - } - - #[tokio::test] - async fn tail_seed_reports_last_marker() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("meshcentral-agent.log"); - - assert_eq!(last_marker_in_tail(&path).await, None); - - tokio::fs::write(&path, "startup\nno markers here\n") - .await - .unwrap(); - assert_eq!(last_marker_in_tail(&path).await, None); - - tokio::fs::write(&path, format!("{FAILED_0_0_22_NO_HTTP}\n{CORE_OK}\n")) - .await - .unwrap(); - assert_eq!(last_marker_in_tail(&path).await, Some(true)); - - tokio::fs::write( - &path, - format!("{CORE_OK}\n{FAILED_0_0_23_PLUS}\n{FAILED_0_0_22_TIMEOUT}\n"), - ) - .await - .unwrap(); - assert_eq!(last_marker_in_tail(&path).await, Some(false)); - } -} +#[path = "mesh_self_heal_service_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/services/mesh_self_heal_service_tests.rs b/clients/openframe-client/src/services/mesh_self_heal_service_tests.rs new file mode 100644 index 000000000..e0b50ce2d --- /dev/null +++ b/clients/openframe-client/src/services/mesh_self_heal_service_tests.rs @@ -0,0 +1,59 @@ +use super::*; + +const FAILED_0_0_22_NO_HTTP: &str = "Connection FAILED: No HTTP response (fd=0, status=Complete/Disconnected, authState=0, connState=0, tls=down, elapsedMs=20016, attempt=ABCD1234-2100)"; +const FAILED_0_0_22_TIMEOUT: &str = "Connection FAILED: Network timeout - server unreachable or gateway blocking (tls=down, elapsedMs=21016, attempt=ABCD1234-2101)"; +const FAILED_0_0_23_PLUS: &str = "Connection FAILED (latest attempt): No HTTP response (fd=0, status=Complete/Disconnected, authState=0, connState=0, tls=down, elapsedMs=20016, attempt=ABCD1234-2102)"; +const CORE_OK: &str = "Received CoreOk from server (coreTimeout=0x0)"; + +#[test] +fn failure_marker_matches_0_0_22_formats() { + assert!(FAILED_0_0_22_NO_HTTP.contains(FAILURE_MARKER)); + assert!(FAILED_0_0_22_TIMEOUT.contains(FAILURE_MARKER)); +} + +#[test] +fn failure_marker_matches_0_0_23_plus_format() { + assert!(FAILED_0_0_23_PLUS.contains(FAILURE_MARKER)); +} + +#[test] +fn markers_ignore_unrelated_lines() { + for line in [ + "Connection: dialing uri=wss://x.openframe.ai/ws/tools/agent/meshcentral-server/agent.ashx host=x.openframe.ai port=443 family=IPv4 ip=1.2.3.4 useproxy=0 proxy=DIRECT attempt=ABCD1234-2103 suppressed=2", + "AutoRetry Connect in 299066 milliseconds", + ] { + assert!(!line.contains(FAILURE_MARKER)); + assert!(!line.contains(HEALTHY_MARKER)); + } +} + +#[test] +fn healthy_marker_matches_core_ok() { + assert!(CORE_OK.contains(HEALTHY_MARKER)); +} + +#[tokio::test] +async fn tail_seed_reports_last_marker() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("meshcentral-agent.log"); + + assert_eq!(last_marker_in_tail(&path).await, None); + + tokio::fs::write(&path, "startup\nno markers here\n") + .await + .unwrap(); + assert_eq!(last_marker_in_tail(&path).await, None); + + tokio::fs::write(&path, format!("{FAILED_0_0_22_NO_HTTP}\n{CORE_OK}\n")) + .await + .unwrap(); + assert_eq!(last_marker_in_tail(&path).await, Some(true)); + + tokio::fs::write( + &path, + format!("{CORE_OK}\n{FAILED_0_0_23_PLUS}\n{FAILED_0_0_22_TIMEOUT}\n"), + ) + .await + .unwrap(); + assert_eq!(last_marker_in_tail(&path).await, Some(false)); +} diff --git a/clients/openframe-client/src/services/mod.rs b/clients/openframe-client/src/services/mod.rs index b43057fd6..d6f2f0b30 100644 --- a/clients/openframe-client/src/services/mod.rs +++ b/clients/openframe-client/src/services/mod.rs @@ -1,6 +1,7 @@ pub mod agent_auth_service; pub mod agent_configuration_service; pub mod agent_registration_service; +pub mod deactivation_service; pub mod device_data_fetcher; pub mod encryption_service; pub mod execution_service; @@ -20,6 +21,8 @@ pub mod nats_message_publisher; pub mod openframe_client_info_service; pub mod openframe_client_update_service; pub mod registration_processor; +pub mod result_outbox_run_manager; +pub mod result_store; pub mod shared_token_service; pub mod token_refresh_run_manager; pub mod tool_agent_update_service; @@ -40,6 +43,7 @@ pub mod update_state_service; pub use agent_auth_service::AgentAuthService; pub use agent_configuration_service::AgentConfigurationService; pub use agent_registration_service::AgentRegistrationService; +pub use deactivation_service::DeactivationService; pub use encryption_service::EncryptionService; pub use github_download_service::GithubDownloadService; pub use initial_configuration_service::InitialConfigurationService; diff --git a/clients/openframe-client/src/services/nats_connection_manager.rs b/clients/openframe-client/src/services/nats_connection_manager.rs index 040790f38..dc979456c 100644 --- a/clients/openframe-client/src/services/nats_connection_manager.rs +++ b/clients/openframe-client/src/services/nats_connection_manager.rs @@ -1,4 +1,5 @@ use crate::services::agent_configuration_service::AgentConfigurationService; +use crate::services::deactivation_service::DeactivationService; use crate::services::local_tls_config_provider::LocalTlsConfigProvider; use crate::services::{AgentAuthService, InitialConfigurationService}; use anyhow::{Context, Result}; @@ -10,6 +11,10 @@ use tokio::sync::broadcast; use tokio::sync::RwLock; use tracing::{info, warn}; +/// Reconnect delay while the tenant is gone (suspended): backs the 5s storm off ~60x so a +/// deleted-tenant client barely touches the gateway. Auto-reverts to 5s on recovery. +const SUSPENDED_RECONNECT_DELAY: std::time::Duration = std::time::Duration::from_secs(5 * 60); + #[derive(Clone)] pub struct NatsConnectionManager { client: Arc>>>, @@ -19,6 +24,7 @@ pub struct NatsConnectionManager { tls_config_provider: LocalTlsConfigProvider, initial_configuration_service: InitialConfigurationService, auth_service: AgentAuthService, + deactivation: Arc, } impl NatsConnectionManager { @@ -31,6 +37,7 @@ impl NatsConnectionManager { initial_configuration_service: InitialConfigurationService, auth_service: AgentAuthService, tls_config_provider: LocalTlsConfigProvider, + deactivation: Arc, ) -> Self { let (reconnect_tx, _) = broadcast::channel(16); Self { @@ -41,6 +48,7 @@ impl NatsConnectionManager { tls_config_provider, initial_configuration_service, auth_service, + deactivation, } } @@ -61,6 +69,8 @@ impl NatsConnectionManager { // Cloned dependencies for auth callback let auth_service = self.auth_service.clone(); let config_service = self.config_service.clone(); + let deactivation = self.deactivation.clone(); + let deactivation_for_delay = self.deactivation.clone(); let nats_server_url = self.nats_server_url.clone(); let nats_server_url_for_reconnect = self.nats_server_url.clone(); let reconnect_tx = self.reconnect_tx.clone(); @@ -76,6 +86,12 @@ impl NatsConnectionManager { .retry_on_initial_connect() .max_reconnects(None) .reconnect_delay_callback(move |attempt| { + // Tenant gone: async-nats can't be stopped from here (its reconnect loop never + // polls Drain), but this callback IS called per attempt — so back off hard to + // turn the 5s WS-upgrade storm into a rare probe against the gone gateway. + if deactivation_for_delay.is_suspended() { + return SUSPENDED_RECONNECT_DELAY; + } warn!( attempt = attempt, hostname = %nats_server_url_for_reconnect, @@ -100,12 +116,14 @@ impl NatsConnectionManager { info!("Starting reauthentication"); let auth_service = auth_service.clone(); let config_service = config_service.clone(); + let deactivation = deactivation.clone(); let nats_server_url = nats_server_url.clone(); async move { Self::perform_reauthentication_and_build_url( auth_service, config_service, + deactivation, nats_server_url, ) .await @@ -135,8 +153,16 @@ impl NatsConnectionManager { async fn perform_reauthentication_and_build_url( auth_service: AgentAuthService, config_service: AgentConfigurationService, + deactivation: Arc, nats_server_url: String, ) -> std::result::Result { + // Tenant gone: skip reauth so NATS reconnects fail locally instead of hammering the gateway. + if deactivation.is_suspended() { + return Err(async_nats::AuthError::new( + "client suspended (tenant gone); skipping NATS reauthentication".to_string(), + )); + } + info!( hostname = %nats_server_url, "Auth URL callback triggered - performing reauthentication" diff --git a/clients/openframe-client/src/services/nats_message_publisher.rs b/clients/openframe-client/src/services/nats_message_publisher.rs index 76c48aa71..96839e349 100644 --- a/clients/openframe-client/src/services/nats_message_publisher.rs +++ b/clients/openframe-client/src/services/nats_message_publisher.rs @@ -30,4 +30,28 @@ impl NatsMessagePublisher { .context("Failed to flush NATS publish")?; Ok(()) } + + pub async fn publish_raw(&self, subject: &str, bytes: &[u8]) -> Result<()> { + let client = self.nats_connection_manager.get_client().await?; + + client + .publish(subject.to_string(), bytes.to_vec().into()) + .await + .context("Failed to publish message to NATS")?; + client + .flush() + .await + .context("Failed to flush NATS publish")?; + Ok(()) + } +} + +impl crate::services::result_store::ResultPublisher for NatsMessagePublisher { + fn publish_raw( + &self, + subject: &str, + bytes: &[u8], + ) -> impl std::future::Future> + Send { + NatsMessagePublisher::publish_raw(self, subject, bytes) + } } diff --git a/clients/openframe-client/src/services/result_outbox_run_manager.rs b/clients/openframe-client/src/services/result_outbox_run_manager.rs new file mode 100644 index 000000000..02858c0a5 --- /dev/null +++ b/clients/openframe-client/src/services/result_outbox_run_manager.rs @@ -0,0 +1,97 @@ +use std::sync::Arc; + +use tokio::sync::Notify; +use tokio::time::{interval, timeout, Duration}; +use tracing::{error, info, warn}; + +use crate::config::update_config::{ + FLUSH_PUBLISH_TIMEOUT_SECS, OUTBOX_FLUSH_INTERVAL_SECS, OUTBOX_MAX_PAYLOAD_BYTES, +}; +use crate::services::result_store::{ResultPublisher, ResultStore}; + +pub struct ResultOutboxRunManager { + store: Arc, + publisher: Arc

, + notify: Arc, +} + +impl ResultOutboxRunManager

{ + pub fn new(store: Arc, publisher: Arc

, notify: Arc) -> Self { + Self { + store, + publisher, + notify, + } + } + + pub fn start(&self) { + if !self.store.enabled() { + info!("Result store not durable, outbox flusher disabled"); + return; + } + let store = self.store.clone(); + let publisher = self.publisher.clone(); + let notify = self.notify.clone(); + + info!("Starting result outbox run manager"); + + tokio::spawn(async move { + let mut interval = interval(Duration::from_secs(OUTBOX_FLUSH_INTERVAL_SECS)); + let publish_timeout = Duration::from_secs(FLUSH_PUBLISH_TIMEOUT_SECS); + + loop { + tokio::select! { + _ = interval.tick() => {} + _ = notify.notified() => {} + } + + let keys = match store.pending_keys().await { + Ok(keys) => keys, + Err(e) => { + error!(error = %e, "Failed to read outbox pending keys"); + continue; + } + }; + + for (key, subject) in keys { + let bytes = match store.load_payload(key.clone()).await { + Ok(Some(bytes)) => bytes, + Ok(None) => { + let _ = store.remove(key).await; + continue; + } + Err(e) => { + warn!(error = %e, "Failed to load outbox payload, retrying next tick"); + break; + } + }; + + if bytes.len() > OUTBOX_MAX_PAYLOAD_BYTES { + warn!( + key = %key, + size = bytes.len(), + "Outbox entry exceeds max payload and cannot be delivered, skipping so it does not block healthy results" + ); + continue; + } + + match timeout(publish_timeout, publisher.publish_raw(&subject, &bytes)).await { + Ok(Ok(())) => { + if let Err(e) = store.remove(key).await { + warn!(error = %e, "Delivered result but failed to remove from outbox"); + } + } + Ok(Err(e)) => { + warn!(error = %e, "Outbox publish failed, retrying next tick"); + break; + } + Err(_) => { + warn!("Outbox publish timed out, retrying next tick"); + break; + } + } + } + } + }); + } +} diff --git a/clients/openframe-client/src/services/result_store.rs b/clients/openframe-client/src/services/result_store.rs new file mode 100644 index 000000000..d3c6b0af6 --- /dev/null +++ b/clients/openframe-client/src/services/result_store.rs @@ -0,0 +1,403 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use std::future::Future; + +use anyhow::{Context, Result}; +use redb::{Database, ReadableTable, ReadableTableMetadata, TableDefinition}; +use serde::{Deserialize, Serialize}; +use tracing::{error, info, warn}; + +use crate::config::update_config::{OUTBOX_MAX_ENTRIES, OUTBOX_MAX_PAYLOAD_BYTES}; +use crate::models::RmmResult; + +const JOURNAL: TableDefinition<&str, &[u8]> = TableDefinition::new("journal"); +const OUTBOX_META: TableDefinition<&str, &[u8]> = TableDefinition::new("outbox_meta"); +const OUTBOX_PAYLOAD: TableDefinition<&str, &[u8]> = TableDefinition::new("outbox_payload"); + +const TRUNCATION_MARKER: &str = "output truncated: exceeded transport size limit"; +const ERROR_NOT_STARTED: &str = "not executed: agent restarted earlier in batch"; +const ERROR_INTERRUPTED: &str = "interrupted by agent restart; outcome unknown"; + +pub trait ResultPublisher: Send + Sync { + fn publish_raw(&self, subject: &str, bytes: &[u8]) -> impl Future> + Send; +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JournalRecord { + pub subject: String, + pub execution_id: String, + pub schedule_id: Option, + pub machine_id: String, + pub script_id: Option, + pub started: bool, + pub created_at_secs: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +struct OutboxMeta { + subject: String, + created_at_secs: u64, +} + +pub fn entry_key(execution_id: &str, script_id: Option<&str>) -> String { + match script_id { + Some(id) => format!("{}:{}", execution_id, id), + None => execution_id.to_string(), + } +} + +pub fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +pub struct ResultStore { + db: Option>, +} + +impl ResultStore { + pub fn open_or_degrade(path: PathBuf) -> Self { + match Self::open(&path) { + Ok(db) => { + info!(path = %path.display(), "Result store opened"); + Self { + db: Some(Arc::new(db)), + } + } + Err(e) => { + error!(path = %path.display(), error = %e, "Failed to open result store, quarantining and retrying"); + let corrupt = path.with_extension("redb.corrupt"); + if let Err(re) = std::fs::rename(&path, &corrupt) { + warn!(error = %re, "Failed to quarantine corrupt result store"); + } + match Self::open(&path) { + Ok(db) => Self { + db: Some(Arc::new(db)), + }, + Err(e2) => { + error!(error = %e2, "Result store unavailable, running without durability"); + Self { db: None } + } + } + } + } + } + + fn open(path: &Path) -> Result { + let db = Database::create(path).context("create redb database")?; + let txn = db.begin_write()?; + { + txn.open_table(JOURNAL)?; + txn.open_table(OUTBOX_META)?; + txn.open_table(OUTBOX_PAYLOAD)?; + } + txn.commit()?; + Ok(db) + } + + pub fn enabled(&self) -> bool { + self.db.is_some() + } + + pub fn encode_result(result: &RmmResult) -> Vec { + let full = serde_json::to_vec(result).unwrap_or_default(); + if full.len() <= OUTBOX_MAX_PAYLOAD_BYTES { + return full; + } + let mut capped = result.clone(); + capped.error = Some(TRUNCATION_MARKER.to_string()); + loop { + let bytes = serde_json::to_vec(&capped).unwrap_or_default(); + if bytes.len() <= OUTBOX_MAX_PAYLOAD_BYTES { + return bytes; + } + if !capped.stdout.is_empty() { + shrink(&mut capped.stdout); + } else if !capped.stderr.is_empty() { + shrink(&mut capped.stderr); + } else { + return bytes; + } + } + } + + async fn with_db(&self, f: F) -> Result + where + F: FnOnce(&Database) -> Result + Send + 'static, + T: Send + 'static, + { + let db = self.db.clone().context("result store is not durable")?; + tokio::task::spawn_blocking(move || f(&db)) + .await + .context("result store task join")? + } + + pub async fn journal_batch( + &self, + execution_id: String, + records: Vec<(String, JournalRecord)>, + ) -> Result { + self.with_db(move |db| { + let txn = db.begin_write()?; + let inserted; + { + let mut journal = txn.open_table(JOURNAL)?; + if journal_contains_batch(&journal, &execution_id)? { + inserted = false; + } else { + for (key, record) in &records { + let value = serde_json::to_vec(record)?; + journal.insert(key.as_str(), value.as_slice())?; + } + inserted = true; + } + } + txn.commit()?; + Ok(inserted) + }) + .await + } + + pub async fn has_batch(&self, execution_id: &str) -> Result { + let execution_id = execution_id.to_string(); + self.with_db(move |db| { + let txn = db.begin_read()?; + let journal = txn.open_table(JOURNAL)?; + journal_contains_batch(&journal, &execution_id) + }) + .await + } + + pub async fn journal_remove(&self, key: String) -> Result<()> { + self.with_db(move |db| { + let txn = db.begin_write()?; + { + txn.open_table(JOURNAL)?.remove(key.as_str())?; + } + txn.commit()?; + Ok(()) + }) + .await + } + + pub async fn journal_mark_started(&self, key: String) -> Result<()> { + self.with_db(move |db| { + let txn = db.begin_write()?; + { + let mut journal = txn.open_table(JOURNAL)?; + let current = journal.get(key.as_str())?.map(|v| v.value().to_vec()); + if let Some(bytes) = current { + let mut record: JournalRecord = serde_json::from_slice(&bytes)?; + record.started = true; + journal.insert(key.as_str(), serde_json::to_vec(&record)?.as_slice())?; + } + } + txn.commit()?; + Ok(()) + }) + .await + } + + pub async fn complete(&self, key: String, subject: String, bytes: Vec) -> Result<()> { + self.with_db(move |db| { + let txn = db.begin_write()?; + { + let mut meta = txn.open_table(OUTBOX_META)?; + let mut payload = txn.open_table(OUTBOX_PAYLOAD)?; + let m = OutboxMeta { + subject, + created_at_secs: now_secs(), + }; + meta.insert(key.as_str(), serde_json::to_vec(&m)?.as_slice())?; + payload.insert(key.as_str(), bytes.as_slice())?; + let mut journal = txn.open_table(JOURNAL)?; + journal.remove(key.as_str())?; + } + txn.commit()?; + Ok(()) + }) + .await?; + self.prune_over_cap().await; + Ok(()) + } + + pub async fn recover(&self) -> Result { + if !self.enabled() { + return Ok(0); + } + let records = self + .with_db(|db| { + let txn = db.begin_read()?; + let journal = txn.open_table(JOURNAL)?; + let mut out = Vec::new(); + for entry in journal.iter()? { + let (key, value) = entry?; + let record: JournalRecord = serde_json::from_slice(value.value())?; + out.push((key.value().to_string(), record)); + } + Ok(out) + }) + .await?; + + let count = records.len(); + for (key, record) in records { + let error = if record.started { + ERROR_INTERRUPTED + } else { + ERROR_NOT_STARTED + }; + let result = RmmResult { + execution_id: record.execution_id.clone(), + machine_id: record.machine_id.clone(), + stdout: String::new(), + stderr: String::new(), + exit_code: 85, + execution_time_ms: 0, + timed_out: false, + error: Some(error.to_string()), + script_id: record.script_id.clone(), + schedule_id: record.schedule_id.clone(), + }; + let bytes = Self::encode_result(&result); + self.complete(key, record.subject, bytes).await?; + } + if count > 0 { + info!( + count, + "Recovered interrupted scheduled scripts into the outbox" + ); + } + Ok(count) + } + + pub async fn pending_keys(&self) -> Result> { + self.with_db(|db| { + let txn = db.begin_read()?; + let meta = txn.open_table(OUTBOX_META)?; + let mut out = Vec::new(); + for entry in meta.iter()? { + let (key, value) = entry?; + let m: OutboxMeta = serde_json::from_slice(value.value())?; + out.push((key.value().to_string(), m.subject)); + } + Ok(out) + }) + .await + } + + pub async fn load_payload(&self, key: String) -> Result>> { + self.with_db(move |db| { + let txn = db.begin_read()?; + let payload = txn.open_table(OUTBOX_PAYLOAD)?; + Ok(payload.get(key.as_str())?.map(|v| v.value().to_vec())) + }) + .await + } + + pub async fn remove(&self, key: String) -> Result<()> { + self.with_db(move |db| { + let txn = db.begin_write()?; + { + let mut meta = txn.open_table(OUTBOX_META)?; + let mut payload = txn.open_table(OUTBOX_PAYLOAD)?; + meta.remove(key.as_str())?; + payload.remove(key.as_str())?; + } + txn.commit()?; + Ok(()) + }) + .await + } + + pub async fn len(&self) -> Result { + self.with_db(|db| { + let txn = db.begin_read()?; + let meta = txn.open_table(OUTBOX_META)?; + Ok(meta.len()? as usize) + }) + .await + } + + async fn prune_over_cap(&self) { + match self.prune_oldest(OUTBOX_MAX_ENTRIES).await { + Ok(0) => {} + Ok(dropped) => warn!( + dropped, + cap = OUTBOX_MAX_ENTRIES, + "Outbox over capacity, dropped oldest results" + ), + Err(e) => warn!(error = %e, "Failed to prune outbox"), + } + } + + pub async fn prune_oldest(&self, keep_newest: usize) -> Result { + self.with_db(move |db| { + { + let rtxn = db.begin_read()?; + let meta = rtxn.open_table(OUTBOX_META)?; + if (meta.len()? as usize) <= keep_newest { + return Ok(0); + } + } + let txn = db.begin_write()?; + let dropped; + { + let mut meta = txn.open_table(OUTBOX_META)?; + let total = meta.len()? as usize; + if total <= keep_newest { + return Ok(0); + } + let mut entries = Vec::with_capacity(total); + for entry in meta.iter()? { + let (key, value) = entry?; + let m: OutboxMeta = serde_json::from_slice(value.value())?; + entries.push((key.value().to_string(), m.created_at_secs)); + } + entries.sort_by_key(|(_, created)| *created); + let to_drop = total - keep_newest; + let victims: Vec = + entries.into_iter().take(to_drop).map(|(k, _)| k).collect(); + let mut payload = txn.open_table(OUTBOX_PAYLOAD)?; + for key in &victims { + meta.remove(key.as_str())?; + payload.remove(key.as_str())?; + } + dropped = victims.len(); + } + txn.commit()?; + Ok(dropped) + }) + .await + } +} + +fn journal_contains_batch( + journal: &impl ReadableTable<&'static str, &'static [u8]>, + execution_id: &str, +) -> Result { + let prefix = format!("{}:", execution_id); + for entry in journal.iter()? { + let (key, _) = entry?; + let key = key.value(); + if key == execution_id || key.starts_with(&prefix) { + return Ok(true); + } + } + Ok(false) +} + +fn shrink(s: &mut String) { + let mut n = s.len().saturating_sub(s.len() / 4 + 1); + while n > 0 && !s.is_char_boundary(n) { + n -= 1; + } + s.truncate(n); +} + +#[cfg(test)] +#[path = "result_store_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/services/result_store_tests.rs b/clients/openframe-client/src/services/result_store_tests.rs new file mode 100644 index 000000000..ffb89b7cf --- /dev/null +++ b/clients/openframe-client/src/services/result_store_tests.rs @@ -0,0 +1,180 @@ +use super::*; +use tempfile::tempdir; + +fn store() -> (ResultStore, tempfile::TempDir) { + let dir = tempdir().unwrap(); + let path = dir.path().join("outbox.redb"); + (ResultStore::open_or_degrade(path), dir) +} + +fn record(execution_id: &str, script_id: &str) -> (String, JournalRecord) { + let key = entry_key(execution_id, Some(script_id)); + ( + key, + JournalRecord { + subject: "machine.m.script-execution.result".to_string(), + execution_id: execution_id.to_string(), + schedule_id: Some("sch".to_string()), + machine_id: "m".to_string(), + script_id: Some(script_id.to_string()), + started: false, + created_at_secs: now_secs(), + }, + ) +} + +fn result(execution_id: &str, script_id: &str) -> RmmResult { + RmmResult { + execution_id: execution_id.to_string(), + machine_id: "m".to_string(), + stdout: "ok".to_string(), + stderr: String::new(), + exit_code: 0, + execution_time_ms: 1, + timed_out: false, + error: None, + script_id: Some(script_id.to_string()), + schedule_id: Some("sch".to_string()), + } +} + +#[tokio::test] +async fn invariant_journal_xor_outbox() { + let (s, _d) = store(); + let (key, rec) = record("ex", "a"); + assert!(s + .journal_batch("ex".to_string(), vec![(key.clone(), rec)]) + .await + .unwrap()); + + assert!(s.has_batch("ex").await.unwrap()); + assert!(s.pending_keys().await.unwrap().is_empty()); + + let bytes = ResultStore::encode_result(&result("ex", "a")); + s.complete(key.clone(), "subj".to_string(), bytes) + .await + .unwrap(); + + assert!(!s.has_batch("ex").await.unwrap()); + let pending = s.pending_keys().await.unwrap(); + assert_eq!(pending, vec![(key, "subj".to_string())]); +} + +#[tokio::test] +async fn journal_batch_rejects_duplicate_execution() { + let (s, _d) = store(); + let (k1, r1) = record("ex", "a"); + assert!(s + .journal_batch("ex".to_string(), vec![(k1, r1)]) + .await + .unwrap()); + + let (k2, r2) = record("ex", "b"); + assert!( + !s.journal_batch("ex".to_string(), vec![(k2, r2)]) + .await + .unwrap(), + "a second batch with the same execution_id must be rejected" + ); + assert!(s.has_batch("ex").await.unwrap()); +} + +#[tokio::test] +async fn write_ahead_survives_reopen() { + let dir = tempdir().unwrap(); + let path = dir.path().join("outbox.redb"); + let key = entry_key("ex", Some("a")); + { + let s = ResultStore::open_or_degrade(path.clone()); + let bytes = ResultStore::encode_result(&result("ex", "a")); + s.complete(key.clone(), "subj".to_string(), bytes) + .await + .unwrap(); + } + let s = ResultStore::open_or_degrade(path); + assert_eq!(s.pending_keys().await.unwrap().len(), 1); + assert!(s.load_payload(key).await.unwrap().is_some()); +} + +#[tokio::test] +async fn recover_distinguishes_started_flag() { + let (s, _d) = store(); + let (k1, r1) = record("ex", "a"); + let (k2, mut r2) = record("ex", "b"); + r2.started = true; + s.journal_batch("ex".to_string(), vec![(k1, r1), (k2, r2)]) + .await + .unwrap(); + + let recovered = s.recover().await.unwrap(); + assert_eq!(recovered, 2); + assert!(!s.has_batch("ex").await.unwrap()); + + let mut errors: Vec = Vec::new(); + for (key, _) in s.pending_keys().await.unwrap() { + let bytes = s.load_payload(key).await.unwrap().unwrap(); + let r: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(r["exit_code"], 85); + errors.push(r["error"].as_str().unwrap().to_string()); + } + errors.sort(); + assert_eq!( + errors, + vec![ERROR_INTERRUPTED.to_string(), ERROR_NOT_STARTED.to_string()] + ); +} + +#[tokio::test] +async fn recover_is_idempotent() { + let (s, _d) = store(); + let (k, r) = record("ex", "a"); + s.journal_batch("ex".to_string(), vec![(k, r)]) + .await + .unwrap(); + assert_eq!(s.recover().await.unwrap(), 1); + assert_eq!(s.recover().await.unwrap(), 0); + assert_eq!(s.pending_keys().await.unwrap().len(), 1); +} + +#[tokio::test] +async fn same_batch_two_scripts_coexist() { + let (s, _d) = store(); + let a = ResultStore::encode_result(&result("ex", "a")); + let b = ResultStore::encode_result(&result("ex", "b")); + s.complete(entry_key("ex", Some("a")), "subj".to_string(), a) + .await + .unwrap(); + s.complete(entry_key("ex", Some("b")), "subj".to_string(), b) + .await + .unwrap(); + assert_eq!(s.pending_keys().await.unwrap().len(), 2); +} + +#[tokio::test] +async fn oversize_result_is_truncated_under_limit() { + let mut r = result("ex", "a"); + r.stdout = "x".repeat(OUTBOX_MAX_PAYLOAD_BYTES + 1024); + r.stderr = "diagnostic".to_string(); + let bytes = ResultStore::encode_result(&r); + assert!(bytes.len() <= OUTBOX_MAX_PAYLOAD_BYTES); + let decoded: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(decoded["error"], TRUNCATION_MARKER); + assert_eq!( + decoded["stderr"], "diagnostic", + "stderr preferred over stdout" + ); +} + +#[tokio::test] +async fn prune_keeps_newest_drops_oldest() { + let (s, _d) = store(); + for i in 0..5 { + let mut r = record("ex", &format!("s{}", i)); + r.1.created_at_secs = i as u64; + let bytes = ResultStore::encode_result(&result("ex", &format!("s{}", i))); + s.complete(r.0, "subj".to_string(), bytes).await.unwrap(); + } + let dropped = s.prune_oldest(2).await.unwrap(); + assert_eq!(dropped, 3); + assert_eq!(s.len().await.unwrap(), 2); +} diff --git a/clients/openframe-client/src/services/token_refresh_run_manager.rs b/clients/openframe-client/src/services/token_refresh_run_manager.rs index 644a53803..ad1df4594 100644 --- a/clients/openframe-client/src/services/token_refresh_run_manager.rs +++ b/clients/openframe-client/src/services/token_refresh_run_manager.rs @@ -1,8 +1,11 @@ +use std::sync::Arc; + use chrono::Utc; use tokio::time::{sleep, timeout, Duration}; use tracing::{debug, error, info, warn}; use crate::services::agent_configuration_service::AgentConfigurationService; +use crate::services::deactivation_service::DeactivationService; use crate::services::AgentAuthService; use crate::utils::jwt; @@ -22,24 +25,44 @@ const REAUTH_TIMEOUT: Duration = Duration::from_secs(30); pub struct TokenRefreshRunManager { auth_service: AgentAuthService, config_service: AgentConfigurationService, + deactivation: Arc, } impl TokenRefreshRunManager { - pub fn new(auth_service: AgentAuthService, config_service: AgentConfigurationService) -> Self { + pub fn new( + auth_service: AgentAuthService, + config_service: AgentConfigurationService, + deactivation: Arc, + ) -> Self { Self { auth_service, config_service, + deactivation, } } pub fn start(&self) { let auth_service = self.auth_service.clone(); let config_service = self.config_service.clone(); + let deactivation = self.deactivation.clone(); info!("Starting proactive token refresh run manager"); tokio::spawn(async move { loop { + // Tenant-gone suspension: this loop is the single backoff probe. Its outcome is + // recorded inside AuthClient (410 -> stay gone / advance uninstall; 2xx -> recover). + if deactivation.is_suspended() { + let wait = deactivation.next_probe_delay().await; + debug!( + "Tenant-gone suspension active; next probe in {}s", + wait.as_secs() + ); + sleep(wait).await; + let _ = timeout(REAUTH_TIMEOUT, auth_service.reauthenticate()).await; + continue; + } + let wait = next_refresh_delay(&config_service).await; if !wait.is_zero() { debug!("Next proactive token refresh in {}s", wait.as_secs()); @@ -63,6 +86,10 @@ impl TokenRefreshRunManager { RETRY_INTERVAL.as_secs() ), } + // Tenant went gone mid-retry — hand control to the backoff probe above. + if deactivation.is_suspended() { + break; + } sleep(RETRY_INTERVAL).await; } } diff --git a/clients/openframe-client/src/services/tool_agent_update_service.rs b/clients/openframe-client/src/services/tool_agent_update_service.rs index b672ace32..4eb08631f 100644 --- a/clients/openframe-client/src/services/tool_agent_update_service.rs +++ b/clients/openframe-client/src/services/tool_agent_update_service.rs @@ -2,8 +2,8 @@ use crate::clients::tool_agent_file_client::ToolAgentFileClient; use crate::models::tool_agent_update_message::{AssetUpdate, ToolAgentUpdateMessage}; use crate::models::{Installation, InstalledAsset, ToolRecordState}; use crate::platform::{ - binary_writer, detect_actual_installation, needs_migration, run_migration, run_update, - DirectoryManager, ToolUpdaterDeps, + binary_writer, clear_aside_binary, detect_actual_installation, needs_migration, run_migration, + run_update, DirectoryManager, ToolUpdaterDeps, }; use crate::services::agent_configuration_service::AgentConfigurationService; use crate::services::tool_run_manager::ToolRunManager; @@ -13,7 +13,7 @@ use crate::services::InstalledToolsService; use crate::services::ToolCommandParamsResolver; use crate::services::ToolKillService; use anyhow::{Context, Result}; -use tracing::{info, warn}; +use tracing::{error, info, warn}; #[derive(Clone)] pub struct ToolAgentUpdateService { @@ -294,10 +294,21 @@ impl ToolAgentUpdateService { // Same-type update let new_installation = run_update(installed_tool, download_config, deps).await?; - installed_tool.version = new_version.to_string(); if let Some(installation) = new_installation { installed_tool.installation = installation; } + + let exec_path = self + .directory_manager + .get_tool_executable_path(tool_agent_id, installed_tool.installation.executable_path()); + if !clear_aside_binary(&exec_path, tool_agent_id).await { + error!(tool_id = %tool_agent_id, + "Pre-update process still holds {}: keeping recorded version {} instead of {} — the new binary is on disk but not running", + exec_path.display(), installed_tool.version, new_version); + return Ok(()); + } + + installed_tool.version = new_version.to_string(); self.installed_tools_service .save(installed_tool.clone()) .await diff --git a/clients/openframe-client/src/services/tool_connection_processing_manager.rs b/clients/openframe-client/src/services/tool_connection_processing_manager.rs index 51ed04fa0..0f6c4e7a9 100644 --- a/clients/openframe-client/src/services/tool_connection_processing_manager.rs +++ b/clients/openframe-client/src/services/tool_connection_processing_manager.rs @@ -1,9 +1,9 @@ use anyhow::{Context, Result}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; use tokio::process::Command; -use tokio::sync::RwLock; +use tokio::sync::{Notify, RwLock}; use tokio::time::{sleep, timeout}; use tracing::{error, info, warn}; @@ -17,12 +17,16 @@ use crate::services::tool_connection_service::ToolConnectionService; use crate::services::tool_run_manager::ToolRunManager; const RETRY_DELAY_SECONDS: u64 = 15; +/// agentId command timeout; the spawned process is killed when it expires. +const AGENT_ID_COMMAND_TIMEOUT_SECONDS: u64 = 15; /// Consecutive agentId-resolution failures tolerated at the normal cadence before the tool /// connection is treated as degraded and the retry loop backs off (e.g. a hung `-nodeid-base64`). const AGENT_ID_MAX_FAST_RETRIES: u32 = 5; /// Back-off delay between agentId attempts once resolution is degraded, so a persistently /// unhealthy agent can't spin a tight 15s loop forever while staying invisible. const AGENT_ID_DEGRADED_BACKOFF_SECONDS: u64 = 300; +/// Cadence of the periodic re-resolve + re-publish, so a mid-run re-key heals within the hour. +const REPUBLISH_INTERVAL_SECONDS: u64 = 3600; // TODO: refactor class #[derive(Clone)] @@ -34,6 +38,7 @@ pub struct ToolConnectionProcessingManager { tool_connection_service: ToolConnectionService, tool_run_manager: ToolRunManager, running_tools: Arc>>, + wake_signals: Arc>>>, } impl ToolConnectionProcessingManager { @@ -53,6 +58,7 @@ impl ToolConnectionProcessingManager { tool_connection_service, tool_run_manager, running_tools: Arc::new(RwLock::new(HashSet::new())), + wake_signals: Arc::new(RwLock::new(HashMap::new())), } } @@ -70,22 +76,16 @@ impl ToolConnectionProcessingManager { return Ok(()); } + // Re-resolve and re-publish on every start: the agent id can change behind our back (tool re-key, db wipe) and the backend updates the mapping only when told. for tool in tools { - if self - .tool_connection_service - .exists_by_tool_agent_id(&tool.tool_agent_id) - .await? - { - info!( - "Tool connection for tool {} already exists - skipping", - tool.tool_id - ); - return Ok(()); + if tool.tool_type.is_empty() { + warn!(tool_agent_id = %tool.tool_agent_id, "Tool has no tool_type - skipping connection publish (the backend rejects empty tool types)"); + continue; } - if self.try_mark_running(&tool.tool_id).await { + if let Some(wake) = self.try_mark_running(&tool.tool_id).await { info!("Processing tool connection for {}", tool.tool_id); - self.process_tool(tool).await?; + self.process_tool(tool, wake).await?; } else { info!( "Connection processing for tool {} is already running - skipping", @@ -98,63 +98,92 @@ impl ToolConnectionProcessingManager { } pub async fn run_new_tool(&self, installed_tool: InstalledTool) -> Result<()> { - if self - .tool_connection_service - .exists_by_tool_agent_id(&installed_tool.tool_agent_id) - .await? - { - info!( - "Tool connection for tool {} already exists - skipping", - installed_tool.tool_id - ); + if installed_tool.tool_type.is_empty() { + warn!(tool_agent_id = %installed_tool.tool_agent_id, "Tool has no tool_type - skipping connection publish (the backend rejects empty tool types)"); return Ok(()); } - if !self.try_mark_running(&installed_tool.tool_id).await { - info!( - "Connection processing for tool {} is already running - skipping", - installed_tool.tool_id - ); - return Ok(()); - } + let wake = match self.try_mark_running(&installed_tool.tool_id).await { + Some(wake) => wake, + None => { + // Reinstall with the loop still alive: wake it so the (possibly new) id publishes now, not at the next hourly tick. + if let Some(wake) = self.wake_signals.read().await.get(&installed_tool.tool_id) { + wake.notify_one(); + info!( + "Connection processing for tool {} is already running - requested immediate re-publish", + installed_tool.tool_id + ); + } else { + info!( + "Connection processing for tool {} is already running - skipping", + installed_tool.tool_id + ); + } + return Ok(()); + } + }; info!( "Processing tool connection for newly installed tool {}", installed_tool.tool_id ); - self.process_tool(installed_tool).await + self.process_tool(installed_tool, wake).await } - async fn try_mark_running(&self, tool_id: &str) -> bool { + /// Marks the tool as running and registers its wake handle under the same guard, so a racing run_new_tool can always nudge a marked tool. + async fn try_mark_running(&self, tool_id: &str) -> Option> { let mut set = self.running_tools.write().await; if set.contains(tool_id) { - false - } else { - set.insert(tool_id.to_string()); - true + return None; } + set.insert(tool_id.to_string()); + let wake = Arc::new(Notify::new()); + self.wake_signals + .write() + .await + .insert(tool_id.to_string(), wake.clone()); + Some(wake) } - pub async fn clear_running_tool(&self, tool_id: &str) { - let mut set = self.running_tools.write().await; - set.remove(tool_id); - } - - async fn process_tool(&self, tool: InstalledTool) -> Result<()> { + async fn process_tool(&self, mut tool: InstalledTool, wake: Arc) -> Result<()> { let params_processor = self.params_processor.clone(); let config_service = self.config_service.clone(); let tool_connection_publisher = self.tool_connection_publisher.clone(); let tool_connection_service = self.tool_connection_service.clone(); let tool_run_manager = self.tool_run_manager.clone(); + let installed_tools_service = self.installed_tools_service.clone(); + let running_tools = self.running_tools.clone(); + let wake_signals = self.wake_signals.clone(); tokio::spawn(async move { // Counts consecutive agentId-resolution failures so a hung agent backs off // (and is reported as degraded) instead of spinning a tight retry loop forever. let mut agent_id_failures: u32 = 0; loop { - while tool_run_manager.is_updating(&tool.tool_agent_id).await { + // Stop if the tool was uninstalled while we were retrying; don't proceed on a failed registry read. + match installed_tools_service + .get_by_tool_agent_id(&tool.tool_agent_id) + .await + { + // Adopt the fresh record so a reinstall/update can't leave the loop resolving with stale config. + Ok(Some(fresh)) if !fresh.tool_type.is_empty() => tool = fresh, + Ok(Some(_)) => {} + Ok(None) => { + info!(tool_id = %tool.tool_id, "Tool no longer installed - stopping connection processing"); + break; + } + Err(e) => { + warn!(tool_id = %tool.tool_id, "Cannot read installed tools registry: {e:#} - retrying"); + sleep(Duration::from_secs(RETRY_DELAY_SECONDS)).await; + continue; + } + } + + // `continue` instead of an inner wait loop so an uninstall during the update window is caught by the registry check above. + if tool_run_manager.is_updating(&tool.tool_agent_id).await { info!(tool_id = %tool.tool_id, "Tool is being updated, deferring node-id resolution..."); sleep(Duration::from_secs(RETRY_DELAY_SECONDS)).await; + continue; } // If tool_agent_id_command_args is empty, use empty string as agent_tool_id @@ -197,9 +226,17 @@ impl ToolConnectionProcessingManager { if !std::path::Path::new(&command_path).exists() { warn!("Executable not found at: {}", command_path); } - // Execute command with a 15-second timeout and capture output - let command_future = Command::new(&command_path).args(&processed_args).output(); - let output = match timeout(Duration::from_secs(15), command_future).await { + // kill_on_drop: a timed-out agent must die with the future, or it leaks and holds the tool's db open + let command_future = Command::new(&command_path) + .args(&processed_args) + .kill_on_drop(true) + .output(); + let output = match timeout( + Duration::from_secs(AGENT_ID_COMMAND_TIMEOUT_SECONDS), + command_future, + ) + .await + { // Command finished within timeout Ok(Ok(out)) => { info!( @@ -216,7 +253,7 @@ impl ToolConnectionProcessingManager { } // Timeout expired Err(_) => { - error!("agentId command timed out after 15 seconds – retrying"); + error!("agentId command timed out after {AGENT_ID_COMMAND_TIMEOUT_SECONDS} seconds – killed it, retrying"); backoff_agent_id_failure(&tool.tool_id, &mut agent_id_failures).await; continue; } @@ -283,8 +320,10 @@ impl ToolConnectionProcessingManager { } info!(tool_id = %tool.tool_id, agent_tool_id = %agent_tool_id, "Tool connection message published successfully and saved"); - // Stop processing after successful publish - break; + tokio::select! { + _ = sleep(Duration::from_secs(REPUBLISH_INTERVAL_SECONDS)) => {} + _ = wake.notified() => info!(tool_id = %tool.tool_id, "Immediate re-publish requested - re-resolving agent id now"), + } } Err(e) => { error!("Failed to get machine_id: {:#}", e); @@ -293,6 +332,10 @@ impl ToolConnectionProcessingManager { } } } + + // Drop the wake entry before the mark so a successor loop's fresh entry can't be clobbered. + wake_signals.write().await.remove(&tool.tool_id); + running_tools.write().await.remove(&tool.tool_id); }); Ok(()) diff --git a/clients/openframe-client/src/services/tool_connection_service.rs b/clients/openframe-client/src/services/tool_connection_service.rs index e4e009755..8db39361b 100644 --- a/clients/openframe-client/src/services/tool_connection_service.rs +++ b/clients/openframe-client/src/services/tool_connection_service.rs @@ -1,6 +1,8 @@ use anyhow::{Context, Result}; use std::fs; use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::Mutex; use crate::models::tool_connection::ToolConnection; use crate::platform::directories::DirectoryManager; @@ -8,6 +10,7 @@ use crate::platform::directories::DirectoryManager; #[derive(Clone)] pub struct ToolConnectionService { file_path: PathBuf, + writer: Arc>, } impl ToolConnectionService { @@ -19,11 +22,15 @@ impl ToolConnectionService { directory_manager .ensure_directories() .with_context(|| "Failed to ensure secured directory exists")?; - Ok(Self { file_path: path }) + Ok(Self { + file_path: path, + writer: Arc::new(Mutex::new(())), + }) } /// Save (upsert) connection pub async fn save(&self, connection: ToolConnection) -> Result<()> { + let _guard = self.writer.lock().await; let mut list = self.get_all().await?; if let Some(existing) = list @@ -38,12 +45,6 @@ impl ToolConnectionService { self.persist(&list).await } - /// Check if a connection exists for given tool_agent_id - pub async fn exists_by_tool_agent_id(&self, id: &str) -> Result { - let list = self.get_all().await?; - Ok(list.iter().any(|c| c.tool_agent_id == id)) - } - pub async fn get_all(&self) -> Result> { if !self.file_path.exists() { return Ok(Vec::new()); @@ -58,6 +59,7 @@ impl ToolConnectionService { /// Delete a tool connection by its tool_agent_id pub async fn delete_by_tool_agent_id(&self, tool_agent_id: &str) -> Result { + let _guard = self.writer.lock().await; let mut list = self.get_all().await?; let initial_len = list.len(); list.retain(|c| c.tool_agent_id != tool_agent_id); diff --git a/clients/openframe-client/src/services/tool_installation_service.rs b/clients/openframe-client/src/services/tool_installation_service.rs index 03eec4949..4355af3f2 100644 --- a/clients/openframe-client/src/services/tool_installation_service.rs +++ b/clients/openframe-client/src/services/tool_installation_service.rs @@ -276,9 +276,6 @@ impl ToolInstallationService { warn!("Failed to remove tool connection: {:#}", e); } - self.tool_connection_processing_manager - .clear_running_tool(&installed_tool.tool_id) - .await; // Do NOT clear tool_run_manager's tracking entry: the existing supervisor loop // resumes with the new binary on its own. Clearing it makes the post-install // run_new_tool spawn a second supervisor, causing two osqueryd to fight over the diff --git a/clients/openframe-client/src/services/tool_restart_service.rs b/clients/openframe-client/src/services/tool_restart_service.rs index bd6bf39da..c3f880937 100644 --- a/clients/openframe-client/src/services/tool_restart_service.rs +++ b/clients/openframe-client/src/services/tool_restart_service.rs @@ -1,3 +1,4 @@ +use crate::config::service_stop::TOOL_RESTART_TIMEOUT_SECS; use crate::models::{Installation, InstalledTool}; use crate::platform::system_service; use crate::services::tool_run_manager::ToolRunManager; @@ -6,6 +7,7 @@ use crate::services::ToolKillService; use anyhow::{Context, Result}; use futures::FutureExt; use std::panic::AssertUnwindSafe; +use std::time::Duration; use tracing::{info, warn}; pub enum RestartOutcome { @@ -68,15 +70,24 @@ impl ToolRestartService { tool_agent_id: tool_agent_id.to_string(), lock_guard: Some(lock_guard), }; - let outcome = AssertUnwindSafe(self.restart_by_tool_agent_id(tool_agent_id)) - .catch_unwind() - .await; + // Hard cap so a wedged OS call can't hold the flag/lock forever and freeze callers (e.g. mesh self-heal). + let outcome = tokio::time::timeout( + Duration::from_secs(TOOL_RESTART_TIMEOUT_SECS), + AssertUnwindSafe(self.restart_by_tool_agent_id(tool_agent_id)).catch_unwind(), + ) + .await; match outcome { - Ok(result) => result, - Err(_) => Err(anyhow::anyhow!( + // May leave the tool stopped; self-heal or server-sent restarts recover it on their next cycle. + Err(_elapsed) => Err(anyhow::anyhow!( + "Restart of tool {} timed out after {}s", + tool_agent_id, + TOOL_RESTART_TIMEOUT_SECS + )), + Ok(Err(_panic)) => Err(anyhow::anyhow!( "Restart panicked for tool {}", tool_agent_id )), + Ok(Ok(result)) => result, } } diff --git a/clients/openframe-client/src/services/tool_run_manager.rs b/clients/openframe-client/src/services/tool_run_manager.rs index 4c0e70ed7..6ac77848b 100644 --- a/clients/openframe-client/src/services/tool_run_manager.rs +++ b/clients/openframe-client/src/services/tool_run_manager.rs @@ -1,7 +1,9 @@ use crate::models::installed_tool::{Installation, InstalledTool, ToolRecordState}; +use crate::platform::system_service; use crate::services::installed_tools_service::InstalledToolsService; use crate::services::tool_command_params_resolver::ToolCommandParamsResolver; use crate::services::tool_kill_service::ToolKillService; +use crate::utils::failure_log_backoff::FailureLogBackoff; use anyhow::{Context, Result}; use std::collections::{HashMap, HashSet}; use std::process::Stdio; @@ -454,6 +456,57 @@ impl ToolRunManager { info!("Tool run manager: shutdown signalled, no new launches will occur"); } + /// Reversibly stop every managed tool (kill processes / stop services) without uninstalling. + /// Used when the tenant is gone, to stop tools hammering their now-unreachable endpoints. + pub async fn stop_all(&self) -> Result<()> { + self.signal_shutdown(); + // Clear supervision so a later restart_all()/run() can relaunch these tools (symmetry + // with restart_all): the shutdown-triggered loop break leaves ids in running_tools. + self.running_tools.write().await.clear(); + let tools = self + .installed_tools_service + .get_all() + .await + .context("Failed to list installed tools for stop_all")?; + for tool in &tools { + if let Err(e) = self + .tool_kill_service + .stop_installed_tool(tool, false) + .await + { + warn!(tool_id = %tool.tool_agent_id, "stop_all: failed to stop tool: {:#}", e); + } + } + info!("Tool run manager: stopped {} tool(s)", tools.len()); + Ok(()) + } + + /// Resume supervision and relaunch every managed tool after a [`stop_all`]. + /// Clears the one-way shutdown flag and the running set, restarts OS-service tools + /// (which `run()` deliberately skips), then re-spawns the standard/GUI supervisors. + pub async fn restart_all(&self) -> Result<()> { + self.shutting_down.store(false, Ordering::Release); + self.running_tools.write().await.clear(); + + match self.installed_tools_service.get_all().await { + Ok(tools) => { + for tool in &tools { + if let Installation::Service { service_name, .. } = &tool.installation { + if let Err(e) = system_service::start_service(service_name).await { + warn!(service = %service_name, "restart_all: failed to start service tool: {:#}", e); + } + } + } + } + Err(e) => warn!( + "restart_all: failed to list installed tools for service restart: {:#}", + e + ), + } + + self.run().await + } + pub async fn mark_client_update_pending(&self) { self.client_update_pending.mark().await; info!("Client update pending: new tool operations will be parked"); @@ -626,6 +679,7 @@ impl ToolRunManager { let mut installation = tool.installation.clone(); tokio::spawn(async move { + let mut launch_backoff = FailureLogBackoff::new(); loop { // Self-update in progress — stop the loop entirely if shutting_down.load(Ordering::Acquire) { @@ -658,15 +712,22 @@ impl ToolRunManager { break; } + let log_attempt = launch_backoff.should_log(); + let processed_args = match params_processor .process(&tool.tool_agent_id, tool.run_command_args.clone()) { Ok(args) => args, Err(e) => { - error!( - "Failed to resolve tool {} run command args: {:#}", - tool.tool_agent_id, e - ); + let failures = launch_backoff.record_failure(log_attempt); + if log_attempt { + error!( + failed_attempts = failures, + "Failed to resolve tool {} run command args: {:#}", + tool.tool_agent_id, + e + ); + } sleep(Duration::from_secs(RETRY_DELAY_SECONDS)).await; continue; } @@ -683,7 +744,7 @@ impl ToolRunManager { .to_string_lossy() .to_string(); - if !std::path::Path::new(&command_path).exists() { + if log_attempt && !std::path::Path::new(&command_path).exists() { warn!("Executable not found at: {}", command_path); } @@ -734,7 +795,9 @@ impl ToolRunManager { launch_as_user, }; - info!(tool_id = %tool.tool_agent_id, "Launching as GuiApp on macOS"); + if log_attempt { + info!(tool_id = %tool.tool_agent_id, "Launching as GuiApp on macOS"); + } if is_process_running(&command_path).await { info!(tool_id = %tool.tool_agent_id, "Already running, skipping launch"); @@ -776,7 +839,9 @@ impl ToolRunManager { match launch_as_user(&command_path, &launch_args, &user).await { Ok(mut child) => { - info!(tool_id = %tool.tool_agent_id, "Launched as user {}, PID: {:?}", user.username, child.id()); + if log_attempt { + info!(tool_id = %tool.tool_agent_id, "Launched as user {}, PID: {:?}", user.username, child.id()); + } if let Some(stdout) = child.stdout.take() { tokio::spawn(async move { @@ -793,17 +858,33 @@ impl ToolRunManager { sleep(Duration::from_secs(3)).await; if is_process_running(&command_path).await { - info!(tool_id = %tool.tool_agent_id, "GuiApp verified running"); + if let Some((failures, failing_for)) = + launch_backoff.record_success() + { + info!(tool_id = %tool.tool_agent_id, failed_attempts = failures, + failing_for_secs = failing_for.as_secs(), + "Tool process started after repeated launch failures"); + } + info!(tool_id = %tool.tool_agent_id, pid = child.id().unwrap_or(0), + user = %user.username, "GuiApp verified running"); running_tools.write().await.remove(&tool.tool_agent_id); return; } - warn!(tool_id = %tool.tool_agent_id, "GuiApp not running after launch, retrying"); + let failures = launch_backoff.record_failure(log_attempt); + if log_attempt { + warn!(tool_id = %tool.tool_agent_id, failed_attempts = failures, + "GuiApp not running after launch, retrying"); + } sleep(Duration::from_secs(RETRY_DELAY_SECONDS)).await; continue; } Err(e) => { - error!(tool_id = %tool.tool_agent_id, "Failed to launch as user: {:#}", e); + let failures = launch_backoff.record_failure(log_attempt); + if log_attempt { + error!(tool_id = %tool.tool_agent_id, failed_attempts = failures, + "Failed to launch as user: {:#}", e); + } sleep(Duration::from_secs(RETRY_DELAY_SECONDS)).await; continue; } @@ -821,7 +902,9 @@ impl ToolRunManager { return; } Installation::Standard { .. } => { - info!(tool_id = %tool.tool_agent_id, "Launching as Standard (managed process)"); + if log_attempt { + info!(tool_id = %tool.tool_agent_id, "Launching as Standard (managed process)"); + } } } @@ -836,10 +919,20 @@ impl ToolRunManager { .stderr(Stdio::piped()) .spawn() { - Ok(child) => child, + Ok(child) => { + if let Some((failures, failing_for)) = launch_backoff.record_success() { + info!(tool_id = %tool.tool_agent_id, failed_attempts = failures, + failing_for_secs = failing_for.as_secs(), + "Tool process started after repeated launch failures"); + } + child + } Err(e) => { - error!(tool_id = %tool.tool_agent_id, error = %e, - "Failed to start tool process - retrying in {} seconds", RETRY_DELAY_SECONDS); + let failures = launch_backoff.record_failure(log_attempt); + if log_attempt { + error!(tool_id = %tool.tool_agent_id, error = %e, failed_attempts = failures, + "Failed to start tool process - retrying in {} seconds", RETRY_DELAY_SECONDS); + } sleep(Duration::from_secs(RETRY_DELAY_SECONDS)).await; continue; } @@ -894,56 +987,5 @@ impl ToolRunManager { } #[cfg(test)] -mod tests { - use super::ClientUpdatePendingFlag; - use std::time::Duration; - - const LONG_TTL: Duration = Duration::from_secs(3600); - - #[tokio::test] - async fn not_pending_before_first_mark() { - let flag = ClientUpdatePendingFlag::default(); - assert!(!flag.is_pending(LONG_TTL).await); - } - - #[tokio::test] - async fn pending_after_mark_within_ttl() { - let flag = ClientUpdatePendingFlag::default(); - flag.mark().await; - assert!(flag.is_pending(LONG_TTL).await); - } - - #[tokio::test] - async fn expired_when_ttl_elapsed() { - let flag = ClientUpdatePendingFlag::default(); - flag.mark().await; - assert!(!flag.is_pending(Duration::ZERO).await); - } - - #[tokio::test] - async fn remark_refreshes_the_ttl() { - let flag = ClientUpdatePendingFlag::default(); - flag.mark().await; - tokio::time::sleep(Duration::from_millis(30)).await; - assert!(!flag.is_pending(Duration::from_millis(10)).await); - flag.mark().await; - assert!(flag.is_pending(Duration::from_millis(10)).await); - } - - #[tokio::test] - async fn clones_share_state() { - let flag = ClientUpdatePendingFlag::default(); - let clone = flag.clone(); - clone.mark().await; - assert!(flag.is_pending(LONG_TTL).await); - } - - #[tokio::test] - async fn clear_releases_the_flag() { - let flag = ClientUpdatePendingFlag::default(); - flag.mark().await; - assert!(flag.is_pending(LONG_TTL).await); - flag.clear().await; - assert!(!flag.is_pending(LONG_TTL).await); - } -} +#[path = "tool_run_manager_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/services/tool_run_manager_tests.rs b/clients/openframe-client/src/services/tool_run_manager_tests.rs new file mode 100644 index 000000000..116fb1a60 --- /dev/null +++ b/clients/openframe-client/src/services/tool_run_manager_tests.rs @@ -0,0 +1,51 @@ +use super::ClientUpdatePendingFlag; +use std::time::Duration; + +const LONG_TTL: Duration = Duration::from_secs(3600); + +#[tokio::test] +async fn not_pending_before_first_mark() { + let flag = ClientUpdatePendingFlag::default(); + assert!(!flag.is_pending(LONG_TTL).await); +} + +#[tokio::test] +async fn pending_after_mark_within_ttl() { + let flag = ClientUpdatePendingFlag::default(); + flag.mark().await; + assert!(flag.is_pending(LONG_TTL).await); +} + +#[tokio::test] +async fn expired_when_ttl_elapsed() { + let flag = ClientUpdatePendingFlag::default(); + flag.mark().await; + assert!(!flag.is_pending(Duration::ZERO).await); +} + +#[tokio::test] +async fn remark_refreshes_the_ttl() { + let flag = ClientUpdatePendingFlag::default(); + flag.mark().await; + tokio::time::sleep(Duration::from_millis(30)).await; + assert!(!flag.is_pending(Duration::from_millis(10)).await); + flag.mark().await; + assert!(flag.is_pending(Duration::from_millis(10)).await); +} + +#[tokio::test] +async fn clones_share_state() { + let flag = ClientUpdatePendingFlag::default(); + let clone = flag.clone(); + clone.mark().await; + assert!(flag.is_pending(LONG_TTL).await); +} + +#[tokio::test] +async fn clear_releases_the_flag() { + let flag = ClientUpdatePendingFlag::default(); + flag.mark().await; + assert!(flag.is_pending(LONG_TTL).await); + flag.clear().await; + assert!(!flag.is_pending(LONG_TTL).await); +} diff --git a/clients/openframe-client/src/services/tool_uninstall_service.rs b/clients/openframe-client/src/services/tool_uninstall_service.rs index c5551ea2a..7592e1f89 100644 --- a/clients/openframe-client/src/services/tool_uninstall_service.rs +++ b/clients/openframe-client/src/services/tool_uninstall_service.rs @@ -1,6 +1,5 @@ use crate::models::{Installation, InstalledTool}; #[cfg(target_os = "windows")] -#[allow(unused_imports)] // retained for windows diagnostics; not currently referenced use crate::platform::file_lock::log_file_lock_info; #[cfg(target_os = "macos")] use crate::platform::remove_app_bundle; @@ -9,9 +8,81 @@ use crate::services::InstalledToolsService; use crate::services::ToolCommandParamsResolver; use crate::services::ToolKillService; use anyhow::{Context, Result}; +use std::process::Stdio; +use std::time::Instant; +use tokio::io::AsyncReadExt; use tokio::process::Command; use tracing::{debug, info, warn}; +const UNINSTALL_COMMAND_TIMEOUT_SECS: u64 = 90; +const OUTPUT_DRAIN_TIMEOUT_SECS: u64 = 5; + +type OutputReader = Option>; + +async fn drain_command_output(stdout: OutputReader, stderr: OutputReader, tool_agent_id: &str) { + for (handle, stream) in [(stdout, "stdout"), (stderr, "stderr")] { + let Some(mut handle) = handle else { continue }; + if tokio::time::timeout( + tokio::time::Duration::from_secs(OUTPUT_DRAIN_TIMEOUT_SECS), + &mut handle, + ) + .await + .is_err() + { + warn!( + "{} of uninstall command for {} still open after {}s (child likely left a process holding the pipe); abandoning it", + stream, tool_agent_id, OUTPUT_DRAIN_TIMEOUT_SECS + ); + handle.abort(); + } + } +} + +async fn stream_command_output(pipe: R, tool_agent_id: String, stream: &'static str) +where + R: AsyncReadExt + Unpin, +{ + let mut pipe = pipe; + let mut chunk = [0u8; 1024]; + let mut line: Vec = Vec::new(); + + loop { + match pipe.read(&mut chunk).await { + Ok(0) => break, + Ok(read) => { + for &byte in &chunk[..read] { + if byte == b'\n' || byte == b'\r' { + emit_command_line(&mut line, &tool_agent_id, stream); + } else { + line.push(byte); + } + } + } + Err(e) => { + warn!( + "Failed reading {} of uninstall command for {}: {}", + stream, tool_agent_id, e + ); + break; + } + } + } + + emit_command_line(&mut line, &tool_agent_id, stream); +} + +fn emit_command_line(line: &mut Vec, tool_agent_id: &str, stream: &'static str) { + if line.is_empty() { + return; + } + let text = String::from_utf8_lossy(line); + let text = text.trim(); + if !text.is_empty() { + info!("[uninstall {} {}] {}", tool_agent_id, stream, text); + } + line.clear(); +} + pub enum UninstallOutcome { Removed, NotInstalled, @@ -190,35 +261,88 @@ impl ToolUninstallService { return Ok(()); } - info!("Running uninstallation command for tool: {}", tool_agent_id); + info!( + "Running uninstallation command for tool: {}: {} {:?}", + tool_agent_id, + agent_path.display(), + processed_args + ); // Execute uninstallation command let mut cmd = Command::new(&agent_path); cmd.args(&processed_args); - - let output = cmd - .output() - .await + cmd.kill_on_drop(true); + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + let started = Instant::now(); + let mut child = cmd + .spawn() + .inspect_err(|_e| { + #[cfg(target_os = "windows")] + log_file_lock_info( + _e, + &agent_path.to_string_lossy(), + "execute uninstallation command", + ); + }) .context("Failed to execute uninstallation command")?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - let stdout = String::from_utf8_lossy(&output.stdout); + let stdout_reader = child.stdout.take().map(|pipe| { + tokio::spawn(stream_command_output( + pipe, + tool_agent_id.to_string(), + "stdout", + )) + }); + let stderr_reader = child.stderr.take().map(|pipe| { + tokio::spawn(stream_command_output( + pipe, + tool_agent_id.to_string(), + "stderr", + )) + }); + + let wait_result = tokio::time::timeout( + tokio::time::Duration::from_secs(UNINSTALL_COMMAND_TIMEOUT_SECS), + child.wait(), + ) + .await; + + let status = match wait_result { + Ok(Ok(status)) => { + drain_command_output(stdout_reader, stderr_reader, tool_agent_id).await; + status + } + Ok(Err(e)) => { + drain_command_output(stdout_reader, stderr_reader, tool_agent_id).await; + return Err(e).context("Failed to wait for uninstallation command"); + } + Err(_) => { + let _ = child.kill().await; + drain_command_output(stdout_reader, stderr_reader, tool_agent_id).await; + return Err(anyhow::anyhow!( + "Uninstallation command for {} timed out after {}s (last output above)", + tool_agent_id, + UNINSTALL_COMMAND_TIMEOUT_SECS + )); + } + }; + if !status.success() { // Fail immediately if uninstall command returns non-zero exit code return Err(anyhow::anyhow!( - "Uninstallation command for {} exited with status: {}\nstdout: {}\nstderr: {}", + "Uninstallation command for {} exited with status: {} after {:?} (output above)", tool_agent_id, - output.status, - stdout, - stderr + status, + started.elapsed() )); } - let stdout = String::from_utf8_lossy(&output.stdout); info!( - "Uninstallation command executed successfully for tool: {}\nstdout: {}", - tool_agent_id, stdout + "Uninstallation command executed successfully for tool: {} in {:?}", + tool_agent_id, + started.elapsed() ); // Cleanup any remaining processes after uninstall command (some tools spawn detached processes) diff --git a/clients/openframe-client/src/utils.rs b/clients/openframe-client/src/utils.rs index 91ce7b5b2..1e2dbe8fa 100644 --- a/clients/openframe-client/src/utils.rs +++ b/clients/openframe-client/src/utils.rs @@ -1,5 +1,7 @@ +pub mod failure_log_backoff; pub mod fs; pub mod jwt; +pub mod timed_permit_pool; #[cfg(target_os = "windows")] pub mod windows_helpers; diff --git a/clients/openframe-client/src/utils/failure_log_backoff.rs b/clients/openframe-client/src/utils/failure_log_backoff.rs new file mode 100644 index 000000000..9f914e638 --- /dev/null +++ b/clients/openframe-client/src/utils/failure_log_backoff.rs @@ -0,0 +1,78 @@ +use std::time::{Duration, Instant}; + +/// Consecutive failures that are all logged before throttling kicks in. +const LOG_EVERY_FAILURES: u64 = 5; +/// Failure-log interval once throttled. +const THROTTLED_INTERVAL: Duration = Duration::from_secs(2 * 60); +/// After failing this long, failure logs slow down to SLOW_INTERVAL. +const SLOW_AFTER: Duration = Duration::from_secs(60 * 60); +/// Failure-log interval for a long-standing failure. +const SLOW_INTERVAL: Duration = Duration::from_secs(60 * 60); + +/// Backs off repeated failure logging (every failure → every 2 min → hourly) without touching the caller's retry cadence; recovery is always reported immediately. +pub struct FailureLogBackoff { + failures: u64, + first_failure: Option, + last_logged: Option, +} + +impl FailureLogBackoff { + pub fn new() -> Self { + Self { + failures: 0, + first_failure: None, + last_logged: None, + } + } + + /// None = log every failure; Some(interval) = log only this often. + fn log_interval(failures: u64, failing_for: Duration) -> Option { + if failures < LOG_EVERY_FAILURES { + None + } else if failing_for < SLOW_AFTER { + Some(THROTTLED_INTERVAL) + } else { + Some(SLOW_INTERVAL) + } + } + + /// Whether this attempt's logs should be emitted; decide before attempting. + pub fn should_log(&self) -> bool { + let (Some(first), Some(last)) = (self.first_failure, self.last_logged) else { + return true; + }; + match Self::log_interval(self.failures, first.elapsed()) { + None => true, + Some(interval) => last.elapsed() >= interval, + } + } + + /// Counts a failure; pass whether the caller actually emitted a log for it. Returns the streak length. + pub fn record_failure(&mut self, logged: bool) -> u64 { + self.failures += 1; + self.first_failure.get_or_insert_with(Instant::now); + if logged { + self.last_logged = Some(Instant::now()); + } + self.failures + } + + /// Some((failures, total failing time)) when a success ends a failure streak; resets state. + pub fn record_success(&mut self) -> Option<(u64, Duration)> { + let streak = self + .first_failure + .map(|first| (self.failures, first.elapsed())); + *self = Self::new(); + streak + } +} + +impl Default for FailureLogBackoff { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +#[path = "failure_log_backoff_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/utils/failure_log_backoff_tests.rs b/clients/openframe-client/src/utils/failure_log_backoff_tests.rs new file mode 100644 index 000000000..bada13c98 --- /dev/null +++ b/clients/openframe-client/src/utils/failure_log_backoff_tests.rs @@ -0,0 +1,75 @@ +use super::*; + +#[test] +fn logs_first_failures_every_time() { + let mut b = FailureLogBackoff::new(); + for _ in 0..LOG_EVERY_FAILURES { + assert!(b.should_log()); + b.record_failure(true); + } +} + +#[test] +fn throttles_after_initial_failures() { + let mut b = FailureLogBackoff::new(); + for _ in 0..LOG_EVERY_FAILURES { + b.record_failure(true); + } + assert!(!b.should_log()); + b.record_failure(false); + assert!(!b.should_log()); +} + +#[test] +fn interval_widens_with_failure_age() { + assert_eq!(FailureLogBackoff::log_interval(0, Duration::ZERO), None); + assert_eq!( + FailureLogBackoff::log_interval(LOG_EVERY_FAILURES - 1, SLOW_AFTER * 2), + None + ); + assert_eq!( + FailureLogBackoff::log_interval(LOG_EVERY_FAILURES, Duration::from_secs(600)), + Some(THROTTLED_INTERVAL) + ); + assert_eq!( + FailureLogBackoff::log_interval(LOG_EVERY_FAILURES, SLOW_AFTER), + Some(SLOW_INTERVAL) + ); +} + +#[test] +fn suppressed_failure_does_not_defer_next_log() { + // Backdated state: mid-streak, last log older than the throttle interval. + let (Some(first), Some(last)) = ( + Instant::now().checked_sub(Duration::from_secs(600)), + Instant::now().checked_sub(THROTTLED_INTERVAL + Duration::from_secs(60)), + ) else { + return; + }; + let mut b = FailureLogBackoff { + failures: LOG_EVERY_FAILURES, + first_failure: Some(first), + last_logged: Some(last), + }; + assert!(b.should_log()); + b.record_failure(false); + assert!( + b.should_log(), + "suppressed failure must not re-arm the throttle" + ); + b.record_failure(true); + assert!(!b.should_log(), "emitted log must re-arm the throttle"); +} + +#[test] +fn success_reports_streak_and_resets() { + let mut b = FailureLogBackoff::new(); + assert_eq!(b.record_success(), None); + for _ in 0..3 { + b.record_failure(true); + } + let (failures, _failing_for) = b.record_success().expect("streak expected"); + assert_eq!(failures, 3); + assert!(b.should_log()); + assert_eq!(b.record_success(), None); +} diff --git a/clients/openframe-client/src/utils/fs.rs b/clients/openframe-client/src/utils/fs.rs index a5fffd85b..fac05e08c 100644 --- a/clients/openframe-client/src/utils/fs.rs +++ b/clients/openframe-client/src/utils/fs.rs @@ -27,28 +27,5 @@ pub fn atomic_write(path: &Path, contents: impl AsRef<[u8]>) -> Result<()> { } #[cfg(all(test, unix))] -mod tests { - use super::*; - use std::os::unix::fs::PermissionsExt; - - #[test] - fn atomic_write_sets_644_and_self_heals() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("token.enc"); - - atomic_write(&path, b"first").unwrap(); - assert_eq!( - std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, - 0o644 - ); - - // A pre-existing owner-only file must be corrected on the next write, not preserved. - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); - atomic_write(&path, b"second").unwrap(); - assert_eq!( - std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, - 0o644 - ); - assert_eq!(std::fs::read(&path).unwrap(), b"second"); - } -} +#[path = "fs_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/utils/fs_tests.rs b/clients/openframe-client/src/utils/fs_tests.rs new file mode 100644 index 000000000..a2fcd0b17 --- /dev/null +++ b/clients/openframe-client/src/utils/fs_tests.rs @@ -0,0 +1,23 @@ +use super::*; +use std::os::unix::fs::PermissionsExt; + +#[test] +fn atomic_write_sets_644_and_self_heals() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("token.enc"); + + atomic_write(&path, b"first").unwrap(); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o644 + ); + + // A pre-existing owner-only file must be corrected on the next write, not preserved. + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + atomic_write(&path, b"second").unwrap(); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o644 + ); + assert_eq!(std::fs::read(&path).unwrap(), b"second"); +} diff --git a/clients/openframe-client/src/utils/jwt.rs b/clients/openframe-client/src/utils/jwt.rs index b5c172267..4cf4b4451 100644 --- a/clients/openframe-client/src/utils/jwt.rs +++ b/clients/openframe-client/src/utils/jwt.rs @@ -22,35 +22,5 @@ pub fn token_exp_unix(token: &str) -> Option { } #[cfg(test)] -mod tests { - use super::*; - - fn make_token(payload: &str) -> String { - format!("header.{}.sig", URL_SAFE_NO_PAD.encode(payload.as_bytes())) - } - - #[test] - fn decodes_exp() { - let token = make_token(r#"{"exp":1700000000,"sub":"machine"}"#); - assert_eq!(token_exp_unix(&token), Some(1700000000)); - } - - #[test] - fn none_without_exp_claim() { - let token = make_token(r#"{"sub":"machine"}"#); - assert_eq!(token_exp_unix(&token), None); - } - - #[test] - fn none_when_malformed() { - assert_eq!(token_exp_unix("not-a-jwt"), None); - assert_eq!(token_exp_unix(""), None); - } - - #[test] - fn none_when_wrong_segment_count() { - let payload = URL_SAFE_NO_PAD.encode(r#"{"exp":1700000000}"#.as_bytes()); - assert_eq!(token_exp_unix(&format!("header.{payload}")), None); - assert_eq!(token_exp_unix(&format!("header.{payload}.sig.extra")), None); - } -} +#[path = "jwt_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/utils/jwt_tests.rs b/clients/openframe-client/src/utils/jwt_tests.rs new file mode 100644 index 000000000..621c8e392 --- /dev/null +++ b/clients/openframe-client/src/utils/jwt_tests.rs @@ -0,0 +1,30 @@ +use super::*; + +fn make_token(payload: &str) -> String { + format!("header.{}.sig", URL_SAFE_NO_PAD.encode(payload.as_bytes())) +} + +#[test] +fn decodes_exp() { + let token = make_token(r#"{"exp":1700000000,"sub":"machine"}"#); + assert_eq!(token_exp_unix(&token), Some(1700000000)); +} + +#[test] +fn none_without_exp_claim() { + let token = make_token(r#"{"sub":"machine"}"#); + assert_eq!(token_exp_unix(&token), None); +} + +#[test] +fn none_when_malformed() { + assert_eq!(token_exp_unix("not-a-jwt"), None); + assert_eq!(token_exp_unix(""), None); +} + +#[test] +fn none_when_wrong_segment_count() { + let payload = URL_SAFE_NO_PAD.encode(r#"{"exp":1700000000}"#.as_bytes()); + assert_eq!(token_exp_unix(&format!("header.{payload}")), None); + assert_eq!(token_exp_unix(&format!("header.{payload}.sig.extra")), None); +} diff --git a/clients/openframe-client/src/utils/timed_permit_pool.rs b/clients/openframe-client/src/utils/timed_permit_pool.rs new file mode 100644 index 000000000..734c72f43 --- /dev/null +++ b/clients/openframe-client/src/utils/timed_permit_pool.rs @@ -0,0 +1,67 @@ +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Result; +use tokio::sync::Semaphore; + +/// Runs blocking closures off-runtime under a timeout, with a permit pool bounding how many threads timed-out (abandoned) calls can leave parked. +pub struct TimedPermitPool { + permits: Arc, + max: usize, +} + +impl TimedPermitPool { + pub fn new(max: usize) -> Self { + Self { + permits: Arc::new(Semaphore::new(max)), + max, + } + } + + /// Fails fast when no permit frees within the timeout; the permit rides inside the closure so it releases only when the blocking call actually returns. + pub async fn call(&self, what: &str, timeout: Duration, f: F) -> Result + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, + { + let permit = match tokio::time::timeout(timeout, self.permits.clone().acquire_owned()).await + { + Err(_elapsed) => { + return Err(anyhow::anyhow!( + "{} not attempted: all {} call slots busy", + what, + self.max + )) + } + Ok(Err(closed)) => { + return Err(anyhow::anyhow!( + "{} not attempted: permit pool closed: {}", + what, + closed + )) + } + Ok(Ok(permit)) => permit, + }; + match tokio::time::timeout( + timeout, + tokio::task::spawn_blocking(move || { + let _permit = permit; + f() + }), + ) + .await + { + Err(_elapsed) => Err(anyhow::anyhow!( + "{} timed out after {}ms", + what, + timeout.as_millis() + )), + Ok(Err(join_err)) => Err(anyhow::anyhow!("{} task failed: {}", what, join_err)), + Ok(Ok(v)) => Ok(v), + } + } +} + +#[cfg(test)] +#[path = "timed_permit_pool_tests.rs"] +mod tests; diff --git a/clients/openframe-client/src/utils/timed_permit_pool_tests.rs b/clients/openframe-client/src/utils/timed_permit_pool_tests.rs new file mode 100644 index 000000000..4d49e7879 --- /dev/null +++ b/clients/openframe-client/src/utils/timed_permit_pool_tests.rs @@ -0,0 +1,90 @@ +use super::*; +use std::thread; + +#[tokio::test] +async fn returns_the_closure_result() { + let pool = TimedPermitPool::new(1); + let v = pool + .call("op", Duration::from_secs(5), || 42) + .await + .unwrap(); + assert_eq!(v, 42); +} + +#[tokio::test] +async fn timeout_fires_on_a_blocked_call() { + let pool = TimedPermitPool::new(1); + let err = pool + .call("op", Duration::from_millis(50), || { + thread::sleep(Duration::from_millis(400)) + }) + .await + .unwrap_err(); + assert!(err.to_string().contains("timed out"), "got: {err}"); +} + +#[tokio::test] +async fn permit_is_held_past_timeout_and_freed_when_the_call_returns() { + let pool = TimedPermitPool::new(1); + let err = pool + .call("op", Duration::from_millis(50), || { + thread::sleep(Duration::from_millis(400)) + }) + .await + .unwrap_err(); + assert!(err.to_string().contains("timed out"), "got: {err}"); + + // The abandoned call still holds the only permit, so the next call must fail fast. + let err = pool + .call("op", Duration::from_millis(50), || ()) + .await + .unwrap_err(); + assert!(err.to_string().contains("call slots busy"), "got: {err}"); + + // Once the blocked closure finishes, the permit frees and the pool recovers. + tokio::time::sleep(Duration::from_millis(600)).await; + pool.call("op", Duration::from_secs(5), || ()) + .await + .unwrap(); +} + +#[tokio::test] +async fn call_over_capacity_fails_fast_while_slots_are_busy() { + let pool = Arc::new(TimedPermitPool::new(4)); + let (tx, rx) = std::sync::mpsc::channel::<()>(); + let rx = Arc::new(std::sync::Mutex::new(rx)); + + let mut holders = Vec::new(); + for _ in 0..4 { + let pool = pool.clone(); + let rx = rx.clone(); + holders.push(tokio::spawn(async move { + pool.call("holder", Duration::from_secs(10), move || { + let _ = rx.lock().unwrap().recv(); + }) + .await + })); + } + // Give the four holders time to occupy every slot. + tokio::time::sleep(Duration::from_millis(200)).await; + + let err = pool + .call("fifth", Duration::from_millis(50), || ()) + .await + .unwrap_err(); + assert!( + err.to_string().contains("all 4 call slots busy"), + "got: {err}" + ); + + // Release the holders; every call must complete and the pool must be usable again. + for _ in 0..4 { + tx.send(()).unwrap(); + } + for h in holders { + h.await.unwrap().unwrap(); + } + pool.call("after", Duration::from_secs(5), || ()) + .await + .unwrap(); +}